Large Multimodal Models (LMMs) exhibit major shortfalls when interpreting images and, by some measures, have poorer spatial cognition than small children or animals. Despite this, they attain high scores on many popular visual benchmarks, with headroom rapidly eroded by an ongoing surge of model progress. To address this, there is a pressing need for difficult benchmarks that remain relevant for longer. We take this idea to its limit by introducing ZeroBench-a lightweight visual reasoning benchmark that is entirely impossible for contemporary frontier LMMs. Our benchmark consists of 100 manually curated questions and 334 less difficult subquestions. We evaluate 20 LMMs on ZeroBench, all of which score 0.0%, and rigorously analyse the errors. To encourage progress in visual understanding, we publicly release ZeroBench.
@article{arxiv.2502.09696,
title = {ZeroBench: An Impossible Visual Benchmark for Contemporary Large Multimodal Models},
author = {Jonathan Roberts and Mohammad Reza Taesiri and Ansh Sharma and Akash Gupta and Samuel Roberts and Ioana Croitoru and Simion-Vlad Bogolin and Jialu Tang and Florian Langer and Vyas Raina and Vatsal Raina and Hanyi Xiong and Vishaal Udandarao and Jingyi Lu and Shiyang Chen and Sam Purkis and Tianshuo Yan and Wenye Lin and Gyungin Shin and Qiaochu Yang and Anh Totti Nguyen and David I. Atkinson and Aaditya Baranwal and Alexandru Coca and Mikah Dang and Sebastian Dziadzio and Jakob D. Kunz and Kaiqu Liang and Alexander Lo and Brian Pulfer and Steven Walton and Charig Yang and Kai Han and Samuel Albanie},
journal= {arXiv preprint arXiv:2502.09696},
year = {2025}
}