Large Language Models have advanced automated software development, however, it remains a challenge to correctly infer dependencies, namely, identifying the internal components and external packages required for a repository to successfully run. Existing studies highlight that dependency-related issues cause over 40\% of observed runtime errors on the generated repository. To address this, we introduce DI-BENCH, a large-scale benchmark and evaluation framework specifically designed to assess LLMs' capability on dependency inference. The benchmark features 581 repositories with testing environments across Python, C#, Rust, and JavaScript. Extensive experiments with textual and execution-based metrics reveal that the current best-performing model achieves only a 42.9% execution pass rate, indicating significant room for improvement. DI-BENCH establishes a new viewpoint for evaluating LLM performance on repositories, paving the way for more robust end-to-end software synthesis.
@article{arxiv.2501.13699,
title = {DI-BENCH: Benchmarking Large Language Models on Dependency Inference with Testable Repositories at Scale},
author = {Linghao Zhang and Junhao Wang and Shilin He and Chaoyun Zhang and Yu Kang and Bowen Li and Jiaheng Wen and Chengxing Xie and Maoquan Wang and Yufan Huang and Elsie Nallipogu and Qingwei Lin and Yingnong Dang and Saravan Rajmohan and Dongmei Zhang and Qi Zhang},
journal= {arXiv preprint arXiv:2501.13699},
year = {2025}
}