Pretrained language models are promising particularly for low-resource languages as they only require unlabelled data. However, training existing models requires huge amounts of compute, while pretrained cross-lingual models often underperform on low-resource languages. We propose Multi-lingual language model Fine-Tuning (MultiFiT) to enable practitioners to train and fine-tune language models efficiently in their own language. In addition, we propose a zero-shot method using an existing pretrained cross-lingual model. We evaluate our methods on two widely used cross-lingual classification datasets where they outperform models pretrained on orders of magnitude more data and compute. We release all models and code.
@article{arxiv.1909.04761,
title = {MultiFiT: Efficient Multi-lingual Language Model Fine-tuning},
author = {Julian Martin Eisenschlos and Sebastian Ruder and Piotr Czapla and Marcin Kardas and Sylvain Gugger and Jeremy Howard},
journal= {arXiv preprint arXiv:1909.04761},
year = {2020}
}