This paper investigates defenses for LLM-based evaluation systems against prompt injection. We formalize a class of threats called blind attacks, where a candidate answer is crafted independently of the true answer to deceive the evaluator. To counter such attacks, we propose a framework that augments Standard Evaluation (SE) with Counterfactual Evaluation (CFE), which re-evaluates the submission against a deliberately false ground-truth answer. An attack is detected if the system validates an answer under both standard and counterfactual conditions. Experiments show that while standard evaluation is highly vulnerable, our SE+CFE framework significantly improves security by boosting attack detection with minimal performance trade-offs.
@article{arxiv.2507.23453,
title = {Counterfactual Evaluation for Blind Attack Detection in LLM-based Evaluation Systems},
author = {Lijia Liu and Takumi Kondo and Kyohei Atarashi and Koh Takeuchi and Jiyi Li and Shigeru Saito and Hisashi Kashima},
journal= {arXiv preprint arXiv:2507.23453},
year = {2025}
}