Multilingual pretrained language models (such as multilingual BERT) have achieved impressive results for cross-lingual transfer. However, due to the constant model capacity, multilingual pre-training usually lags behind the monolingual competitors. In this work, we present two approaches to improve zero-shot cross-lingual classification, by transferring the knowledge from monolingual pretrained models to multilingual ones. Experimental results on two cross-lingual classification benchmarks show that our methods outperform vanilla multilingual fine-tuning.
@article{arxiv.1911.03913,
title = {Can Monolingual Pretrained Models Help Cross-Lingual Classification?},
author = {Zewen Chi and Li Dong and Furu Wei and Xian-Ling Mao and Heyan Huang},
journal= {arXiv preprint arXiv:1911.03913},
year = {2019}
}