Generation of citation-backed reports is a primary use case for retrieval-augmented generation (RAG) systems. While open-source evaluation tools exist for various RAG tasks, tools designed for report generation are lacking. Accordingly, we introduce Auto-ARGUE, a robust LLM-based implementation of the recently proposed ARGUE framework for report generation evaluation. We present analysis of Auto-ARGUE on the report generation pilot task from the TREC 2024 NeuCLIR track and on two tasks from the TREC 2024 RAG track, showing good system-level correlations with human judgments. Additionally, we release ARGUE-Viz, a web app for visualization and fine-grained analysis of Auto-ARGUE judgments and scores.
@article{arxiv.2509.26184,
title = {Auto-ARGUE: LLM-Based Report Generation Evaluation},
author = {William Walden and Marc Mason and Orion Weller and Laura Dietz and John Conroy and Neil Molino and Hannah Recknor and Bryan Li and Gabrielle Kaili-May Liu and Yu Hou and Dawn Lawrie and James Mayfield and Eugene Yang},
journal= {arXiv preprint arXiv:2509.26184},
year = {2026}
}