With context windows of millions of tokens, Long-Context Language Models (LCLMs) can encode entire document collections, offering a strong alternative to conventional retrieval-augmented generation (RAG). However, it remains unclear whether fine-tuning strategies can improve long-context performance and translate to greater robustness under KV-cache compression techniques. In this work, we investigate which training strategies most effectively enhance LCLMs' ability to identify and use relevant information, as well as enhancing their robustness under KV-cache compression. Our experiments show substantial in-domain improvements, achieving gains of up to +20 points over the base model. However, out-of-domain generalization remains task dependent with large variance -- LCLMs excels on finance questions (+9 points), while RAG shows stronger performance on multiple-choice questions (+6 points) over the baseline models. Finally, we show that our fine-tuning approaches bring moderate improvements in robustness under KV-cache compression, with gains varying across tasks.
@article{arxiv.2601.18527,
title = {Exploring Fine-Tuning for In-Context Retrieval and Efficient KV-Caching in Long-Context Language Models},
author = {Francesco Maria Molfese and Momchil Hardalov and Rexhina Blloshmi and Bill Byrne and Adrià de Gispert},
journal= {arXiv preprint arXiv:2601.18527},
year = {2026}
}
Comments
European Chapter of the Association for Computational Linguistics EACL 2026