Existing XAI metrics measure faithfulness for a single model, ignoring model multiplicity where near-optimal classifiers rely on different or spurious acoustic cues. In noisy farm environments, stationary artifacts such as ventilation noise can produce explanations that are faithful yet unreliable, as masking-based metrics fail to penalize redundant shortcuts. We propose AGRI-Fidelity, a reliability-oriented evaluation framework for listenable explanations in poultry disease detection without spatial ground truth. The method combines cross-model consensus with cyclic temporal permutation to construct null distributions and compute a False Discovery Rate (FDR), suppressing stationary artifacts while preserving time-localized bioacoustic markers. Across real and controlled datasets, AGRI-Fidelity effectively provides reliability-aware discrimination for all data points versus masking-based metrics.
@article{arxiv.2603.18247,
title = {AGRI-Fidelity: Evaluating the Reliability of Listenable Explanations for Poultry Disease Detection},
author = {Sindhuja Madabushi and Arda Dogan and Jonathan Liu and Dian Chen and Dong S. Ha and Sook Shin and Sam H. Noh and Jin-Hee Cho},
journal= {arXiv preprint arXiv:2603.18247},
year = {2026}
}