During migration across instruction set architectures (ISAs), software package build repair is a critical task for ensuring the reliability of software deployment and the stability of modern operating systems. While Large Language Models (LLMs) have shown promise in tackling this challenge, prior work has primarily focused on single instruction set architecture (ISA) and homogeneous programming languages. To address this limitation, we introduce a new benchmark designed for software package build repair across diverse architectures and languages. Comprising 268 real-world software package build failures, the benchmark provides a standardized evaluation pipeline. We evaluate six state-of-the-art LLMs on the benchmark, and the results show that cross-ISA software package repair remains difficult and requires further advances. By systematically exposing this challenge, the benchmark establishes a foundation for advancing future methods aimed at improving software portability and bridging architectural gaps.
@article{arxiv.2601.12927,
title = {A Benchmark for Language Models in Real-World System Building},
author = {Weilin Jin and Chenyu Zhao and Zeshun Huang and Chaoyun Zhang and Qingwei Lin and Chetan Bansal and Saravan Rajmohan and Shenglin Zhang and Yongqian Sun and Dan Pei and Yifan Wu and Tong Jia and Ying Li and Zhonghai Wu and Minghua Ma},
journal= {arXiv preprint arXiv:2601.12927},
year = {2026}
}