Despite their popularity in non-English NLP, multilingual language models often underperform monolingual ones due to inter-language competition for model parameters. We propose Cross-lingual Expert Language Models (X-ELM), which mitigate this competition by independently training language models on subsets of the multilingual corpus. This process specializes X-ELMs to different languages while remaining effective as a multilingual ensemble. Our experiments show that when given the same compute budget, X-ELM outperforms jointly trained multilingual models across all considered languages and that these gains transfer to downstream tasks. X-ELM provides additional benefits over performance improvements: new experts can be iteratively added, adapting X-ELM to new languages without catastrophic forgetting. Furthermore, training is asynchronous, reducing the hardware requirements for multilingual training and democratizing multilingual modeling.
@article{arxiv.2401.10440,
title = {Breaking the Curse of Multilinguality with Cross-lingual Expert Language Models},
author = {Terra Blevins and Tomasz Limisiewicz and Suchin Gururangan and Margaret Li and Hila Gonen and Noah A. Smith and Luke Zettlemoyer},
journal= {arXiv preprint arXiv:2401.10440},
year = {2024}
}