Model merging has emerged as a cost-efficient approximation to multitask learning. Among merging strategies, task arithmetic is notable for its simplicity and effectiveness. In this work, we provide a theoretical motivation for task vectors by highlighting that, under single-epoch full-batch gradient descent, they are equivalent to multitask gradients. This insight leads us to reinterpret model merging as a single step in an iterative procedure that Alternates between Tuning and Merging (ATM). We propose two applications of ATM: (1) as an alternative to multitask learning in scenarios where data sharing is restricted (e.g., federated settings), and (2) as a lightweight refinement step to improve existing model merging methods using a small validation set. Experiments across diverse vision tasks demonstrate the effectiveness of ATM.
@article{arxiv.2411.03055,
title = {ATM: Improving Model Merging by Alternating Tuning and Merging},
author = {Luca Zhou and Daniele Solombrino and Donato Crisostomi and Maria Sofia Bucarelli and Fabrizio Silvestri and Emanuele Rodolà},
journal= {arXiv preprint arXiv:2411.03055},
year = {2025}
}