Multilingual pre-trained language models have demonstrated impressive (zero-shot) cross-lingual transfer abilities, however, their performance is hindered when the target language has distant typology from source languages or when pre-training data is limited in size. In this paper, we propose XLM-P, which contextually retrieves prompts as flexible guidance for encoding instances conditionally. Our XLM-P enables (1) lightweight modeling of language-invariant and language-specific knowledge across languages, and (2) easy integration with other multilingual pre-training methods. On the tasks of XTREME including text classification, sequence labeling, question answering, and sentence retrieval, both base- and large-size language models pre-trained with our proposed method exhibit consistent performance improvement. Furthermore, it provides substantial advantages for low-resource languages in unsupervised sentence retrieval and for target languages that differ greatly from the source language in cross-lingual transfer.
@article{arxiv.2306.07610,
title = {Soft Language Clustering for Multilingual Model Pre-training},
author = {Jiali Zeng and Yufan Jiang and Yongjing Yin and Yi Jing and Fandong Meng and Binghuai Lin and Yunbo Cao and Jie Zhou},
journal= {arXiv preprint arXiv:2306.07610},
year = {2023}
}