Long-context understanding has emerged as a critical capability for large language models (LLMs). However, evaluating this ability remains challenging. We present SCALAR, a benchmark designed to assess citation-grounded long-context reasoning in academic writing. SCALAR leverages academic papers and their citation structure to automatically generate high-quality ground-truth labels without human annotation. It features controllable difficulty levels and a dynamic updating mechanism that mitigates data contamination. The benchmark includes two tasks: a multiple-choice QA format and a cloze-style citation prediction. We evaluate a range of state-of-the-art LLMs and find that the multiple-choice task effectively distinguishes model capabilities. While human experts achieve over 90% accuracy, most models struggle. The cloze-style task is even more challenging, with no model exceeding 50% accuracy. SCALAR provides a domain-grounded, continuously updating framework for tracking progress in citation-based long-context understanding.
@article{arxiv.2502.13753,
title = {SCALAR: Scientific Citation-based Live Assessment of Long-context Academic Reasoning},
author = {Renxi Wang and Honglin Mu and Liqun Ma and Lizhi Lin and Yunlong Feng and Timothy Baldwin and Xudong Han and Haonan Li},
journal= {arXiv preprint arXiv:2502.13753},
year = {2026}
}