In an ideal design pipeline, user interface (UI) design is intertwined with user research to validate decisions, yet studies are often resource-constrained during early exploration. Recent advances in multimodal large language models (MLLMs) offer a promising opportunity to act as early evaluators, helping designers narrow options before formal testing. Unlike prior work that emphasizes user behavior in narrow domains such as e-commerce with metrics like clicks or conversions, we focus on subjective user evaluations across varied interfaces. We investigate whether MLLMs can mimic human preferences when evaluating individual UIs and comparing them. Using data from a crowdsourcing platform, we benchmark GPT-4o, Claude, and Llama across 30 interfaces and examine alignment with human judgments on multiple UI factors. Our results show that MLLMs approximate human preferences on some dimensions but diverge on others, underscoring both their potential and limitations in supplementing early UX research.
@article{arxiv.2510.08783,
title = {MLLM as a UI Judge: Benchmarking Multimodal LLMs for Predicting Human Perception of User Interfaces},
author = {Reuben A. Luera and Ryan Rossi and Franck Dernoncourt and Samyadeep Basu and Sungchul Kim and Subhojyoti Mukherjee and Puneet Mathur and Ruiyi Zhang and Jihyung Kil and Nedim Lipka and Seunghyun Yoon and Jiuxiang Gu and Zichao Wang and Cindy Xiong Bearfield and Branislav Kveton},
journal= {arXiv preprint arXiv:2510.08783},
year = {2025}
}