While large language models (LLMs) are empowered with broad knowledge, their task-specific performance is often suboptimal. It necessitates fine-tuning LLMs with task-specific data, but such data may be inaccessible due to privacy concerns. In this paper, we propose a novel approach to enhance LLMs with smaller language models (SLMs) that are trained on clients using their private task-specific data. To enable mutual enhancement between LLMs and SLMs, we propose CrossLM, where the SLMs promote the LLM to generate task-specific high-quality data, and both the LLM and SLMs are enhanced with the generated data. We evaluate CrossLM using publicly accessible language models across a range of benchmark tasks. The results demonstrate that CrossLM significantly enhances the task-specific performance of SLMs on clients and the LLM on the cloud server simultaneously while preserving the LLM's generalization capability.
@article{arxiv.2312.05842,
title = {Mutual Enhancement of Large and Small Language Models with Cross-Silo Knowledge Transfer},
author = {Yongheng Deng and Ziqing Qiao and Ju Ren and Yang Liu and Yaoxue Zhang},
journal= {arXiv preprint arXiv:2312.05842},
year = {2023}
}