The rapid adoption of AI agents across domains has made systematic evaluation crucial for ensuring their usefulness and successful production deployment. Evaluation of AI agents typically involves using a fixed set of benchmarks and computing multiple evaluation metrics for the agent. While sufficient for simple coding tasks, these benchmarks fall short for enterprise-scale agents, where services and requirements evolve continuously and ground-truth examples are sparse. We propose a process of benchmark generation that helps evolve the benchmarks as the requirements change and perform robust evaluation of evolving AI agents. We instantiate this approach for a case study of service migration from one deployment platform to another at a large public enterprise. Our approach relies on semi-structured documents where developers express the high-level intent, and uses state-of-the-art LLMs to generate benchmarks from just a small number of such documents. Overall, this process results in a maintainable evaluation framework, enabling rapid feedback on agent performance and facilitating targeted improvements.
@article{arxiv.2511.10049,
title = {Continuous Benchmark Generation for Evaluating Enterprise-scale LLM Agents},
author = {Divyanshu Saxena and Rishikesh Maurya and Xiaoxuan Ou and Gagan Somashekar and Shachee Mishra Gupta and Arun Iyer and Yu Kang and Chetan Bansal and Aditya Akella and Saravan Rajmohan},
journal= {arXiv preprint arXiv:2511.10049},
year = {2025}
}