Recently, various merging methods have been proposed to build a multi-task model from task-specific finetuned models without retraining. However, existing methods suffer from a large performance deterioration compared to using multiple task-specific models. In this paper, we propose to inject task-specific knowledge into the merged model and design two parameter-efficient approaches (BYOM-FFT and BYOM-LoRA) to Build Your Own Multi-task model. BYOM-FFT is for merging fully finetuned models, while BYOM-LoRA is for LoRA-finetuned models. Both methods are data-free and computation-efficient. Extensive experiments on computer vision and natural language processing tasks show that the proposed BYOM methods outperform existing merging methods by a large margin. Moreover, BYOM-FFT is general and can be integrated into existing merging methods to further boost performance.
@article{arxiv.2310.01886,
title = {BYOM: Building Your Own Multi-Task Model For Free},
author = {Weisen Jiang and Baijiong Lin and Han Shi and Yu Zhang and Zhenguo Li and James T. Kwok},
journal= {arXiv preprint arXiv:2310.01886},
year = {2024}
}