We introduce AInsteinBench, a large-scale benchmark for evaluating whether large language model (LLM) agents can operate as scientific computing development agents within real research software ecosystems. Unlike existing scientific reasoning benchmarks which focus on conceptual knowledge, or software engineering benchmarks that emphasize generic feature implementation and issue resolving, AInsteinBench evaluates models in end-to-end scientific development settings grounded in production-grade scientific repositories. The benchmark consists of tasks derived from maintainer-authored pull requests across six widely used scientific codebases, spanning quantum chemistry, quantum computing, molecular dynamics, numerical relativity, fluid dynamics, and cheminformatics. All benchmark tasks are carefully curated through multi-stage filtering and expert review to ensure scientific challenge, adequate test coverage, and well-calibrated difficulty. By leveraging evaluation in executable environments, scientifically meaningful failure modes, and test-driven verification, AInsteinBench measures a model's ability to move beyond surface-level code generation toward the core competencies required for computational scientific research.
@article{arxiv.2512.21373,
title = {AInsteinBench: Benchmarking Coding Agents on Scientific Repositories},
author = {Titouan Duston and Shuo Xin and Yang Sun and Daoguang Zan and Aoyan Li and Shulin Xin and Kai Shen and Yixiao Chen and Qiming Sun and Ge Zhang and Jiashuo Liu and Huan Zhou and Jingkai Liu and Zhichen Pu and Yuanheng Wang and Bo-Xuan Ge and Xin Tong and Fei Ye and Zhi-Chao Zhao and Wen-Biao Han and Zhoujian Cao and Yueran Zhao and Weiluo Ren and Qingshen Long and Yuxiao Liu and Anni Huang and Yidi Du and Yuanyuan Rong and Jiahao Peng},
journal= {arXiv preprint arXiv:2512.21373},
year = {2025}
}