Conventional fine-tuning on domain-specific datasets can inadvertently alter a model's pretrained multimodal priors, leading to reduced generalization. To address this, we propose Chain-of-Adaptation (CoA), an adaptation framework designed to integrate domain knowledge while maintaining the model's inherent reasoning and perceptual capabilities. CoA introduces a structured reasoning format that enhances domain alignment without sacrificing general multimodal competence by reinforcement learning. Experiments on standard surgical benchmarks, under both in-distribution and out-of-distribution settings, demonstrate that CoA achieves higher accuracy, stronger generalization, and more stable behavior than supervised fine-tuning. Furthermore, ablation studies confirm that CoA effectively preserves the model's core visual-language abilities, providing a reliable pathway for domain specialization in VLMs.
@article{arxiv.2603.20116,
title = {Chain-of-Adaptation: Surgical Vision-Language Adaptation with Reinforcement Learning},
author = {Jiajie Li and Chenhui Xu and Meihuan Liu and Jinjun Xiong},
journal= {arXiv preprint arXiv:2603.20116},
year = {2026}
}