The performance of AI models on safety benchmarks does not indicate their real-world performance after deployment. This opaqueness of AI models impedes existing regulatory frameworks constituted on benchmark performance, leaving them incapable of mitigating ongoing real-world harm. The problem stems from a fundamental challenge in AI interpretability, which seems to be overlooked by regulators and decision makers. We propose a simple, realistic and readily usable regulatory framework which does not rely on benchmarks, and call for interdisciplinary collaboration to find new ways to address this crucial problem.
@article{arxiv.2501.15693,
title = {Beyond Benchmarks: On The False Promise of AI Regulation},
author = {Gabriel Stanovsky and Renana Keydar and Gadi Perl and Eliya Habba},
journal= {arXiv preprint arXiv:2501.15693},
year = {2025}
}