Model routing is a simple technique for reducing the inference cost of large language models (LLMs), wherein one maintains a pool of candidate LLMs, and learns to route each prompt to the smallest feasible LLM. Existing works focus on learning a router for a fixed pool of LLMs. In this paper, we consider the problem of dynamic routing, where new, previously unobserved LLMs are available at test time. We propose UniRoute, a new approach to this problem that relies on representing each LLM as a feature vector, derived based on predictions on a set of representative prompts. Based on this, we detail two effective instantiations of UniRoute, relying on cluster-based routing and a learned cluster map respectively. We show that these are estimates of a theoretically optimal routing rule, and quantify their errors via an excess risk bound. Experiments on a range of public benchmarks show the effectiveness of UniRoute in routing amongst more than 30 unseen LLMs.
@article{arxiv.2502.08773,
title = {Universal Model Routing for Efficient LLM Inference},
author = {Wittawat Jitkrittum and Harikrishna Narasimhan and Ankit Singh Rawat and Jeevesh Juneja and Congchao Wang and Zifeng Wang and Alec Go and Chen-Yu Lee and Pradeep Shenoy and Rina Panigrahy and Aditya Krishna Menon and Sanjiv Kumar},
journal= {arXiv preprint arXiv:2502.08773},
year = {2025}
}