We consider the problem of scaling automated suggested replies for Outlook email system to multiple languages. Faced with increased compute requirements and low resources for language expansion, we build a single universal model for improving the quality and reducing run-time costs of our production system. However, restricted data movement across regional centers prevents joint training across languages. To this end, we propose a multi-task continual learning framework, with auxiliary tasks and language adapters to learn universal language representation across regions. The experimental results show positive cross-lingual transfer across languages while reducing catastrophic forgetting across regions. Our online results on real user traffic show significant gains in CTR and characters saved, as well as 65% training cost reduction compared with per-language models. As a consequence, we have scaled the feature in multiple languages including low-resource markets.
@article{arxiv.2106.02232,
title = {Language Scaling for Universal Suggested Replies Model},
author = {Qianlan Ying and Payal Bajaj and Budhaditya Deb and Yu Yang and Wei Wang and Bojia Lin and Milad Shokouhi and Xia Song and Yang Yang and Daxin Jiang},
journal= {arXiv preprint arXiv:2106.02232},
year = {2021}
}