We revisit benchmarks for differentially private image classification. We suggest a comprehensive set of benchmarks, allowing researchers to evaluate techniques for differentially private machine learning in a variety of settings, including with and without additional data, in convex settings, and on a variety of qualitatively different datasets. We further test established techniques on these benchmarks in order to see which ideas remain effective in different settings. Finally, we create a publicly available leader board for the community to track progress in differentially private machine learning.
@article{arxiv.2601.17189,
title = {Rethinking Benchmarks for Differentially Private Image Classification},
author = {Sabrina Mokhtari and Sara Kodeiri and Shubhankar Mohapatra and Florian Tramèr and Gautam Kamath},
journal= {arXiv preprint arXiv:2601.17189},
year = {2026}
}