This paper describes a language representation model which combines the Bidirectional Encoder Representations from Transformers (BERT) learning mechanism described in Devlin et al. (2018) with a generalization of the Universal Transformer model described in Dehghani et al. (2018). We further improve this model by adding a latent variable that represents the persona and topics of interests of the writer for each training example. We also describe a simple method to improve the usefulness of our language representation for solving problems in a specific domain at the expense of its ability to generalize to other fields. Finally, we release a pre-trained language representation model for social texts that was trained on 100 million tweets.
@article{arxiv.1905.06638,
title = {Latent Universal Task-Specific BERT},
author = {Alon Rozental and Zohar Kelrich and Daniel Fleischer},
journal= {arXiv preprint arXiv:1905.06638},
year = {2019}
}