Despite the rapid advancements in large language model (LLM) development, fine-tuning them for specific tasks often results in the catastrophic forgetting of their general, language-based reasoning abilities. This work investigates and addresses this challenge in the context of the Generative Retrieval (GenRetrieval) task. During GenRetrieval fine-tuning, we find this forgetting occurs rapidly and correlates with the distance between the fine-tuned and original model parameters. Given these observations, we propose ORBIT, a novel approach that actively tracks the distance between fine-tuned and initial model weights, and uses a weight averaging strategy to constrain model drift during GenRetrieval fine-tuning when this inter-model distance exceeds a maximum threshold. Our results show that ORBIT retains substantial text and retrieval performance by outperforming both common continual learning baselines and related regularization methods that also employ weight averaging.
@article{arxiv.2605.12419,
title = {ORBIT: Preserving Foundational Language Capabilities in GenRetrieval via Origin-Regulated Merging},
author = {Neha Verma and Nikhil Mehta and Shao-Chuan Wang and Naijing Zhang and Alicia Tsai and Li Wei and Lukasz Heldt and Lichan Hong and Ed Chi and Xinyang Yi},
journal= {arXiv preprint arXiv:2605.12419},
year = {2026}
}