Static benchmarks for RAG systems often suffer from rapid saturation and require significant manual effort to maintain robustness. To address this, we present IRB, a framework for automatically generating benchmarks to evaluate the factuality of RAG systems. IRB employs a structured generation pipeline utilizing \textit{factual scaffold} and \textit{algorithmic scaffold}. We utilize IRB to construct a benchmark and evaluate frontier LLMs and retrievers. Our results demonstrate that IRB poses a significant challenge for frontier LLMs in the closed-book setting. Furthermore, our evaluation suggests that reasoning LLMs are more reliable, and that improving the retrieval component may yield more cost-effective gains in RAG system correctness than scaling the generator.
@article{arxiv.2602.08070,
title = {IRB: Automated Generation of Robust Factuality Benchmarks},
author = {Lam Thanh Do and Bhagyashree Taleka and Hozaifa Ammar Bhutta and Vikram Sharma Mailthody and Kevin Chen-Chuan Chang and Wen-mei Hwu},
journal= {arXiv preprint arXiv:2602.08070},
year = {2026}
}