We introduce LLM SELECTOR, the first framework for active model selection of Large Language Models (LLMs). Unlike prior evaluation and benchmarking approaches that rely on fully annotated datasets, LLM SELECTOR efficiently identifies the best LLM with limited annotations. In particular, for any given task, LLM SELECTOR adaptively selects a small set of queries to annotate that are most informative about the best model for the task. To further reduce annotation cost, we leverage a judge-based oracle annotation model. Through extensive experiments on 6 benchmarks with 151 LLMs, we show that LLM SELECTOR reduces annotation costs by up to 59.62% when selecting the best and near-best LLM for the task.
@article{arxiv.2510.09418,
title = {Active Model Selection for Large Language Models},
author = {Yavuz Durmazkeser and Patrik Okanovic and Andreas Kirsch and Torsten Hoefler and Nezihe Merve Gürel},
journal= {arXiv preprint arXiv:2510.09418},
year = {2025}
}