In this paper, we propose Latent Relation Language Models (LRLMs), a class of language models that parameterizes the joint distribution over the words in a document and the entities that occur therein via knowledge graph relations. This model has a number of attractive properties: it not only improves language modeling performance, but is also able to annotate the posterior probability of entity spans for a given text through relations. Experiments demonstrate empirical improvements over both a word-based baseline language model and a previous approach that incorporates knowledge graph information. Qualitative analysis further demonstrates the proposed model's ability to learn to predict appropriate relations in context.
@article{arxiv.1908.07690,
title = {Latent Relation Language Models},
author = {Hiroaki Hayashi and Zecong Hu and Chenyan Xiong and Graham Neubig},
journal= {arXiv preprint arXiv:1908.07690},
year = {2019}
}