The deployment of Large Language Models (LLMs) and Large Multimodal Models (LMMs) on mobile devices has gained significant attention due to the benefits of enhanced privacy, stability, and personalization. However, the hardware constraints of mobile devices necessitate the use of models with fewer parameters and model compression techniques like quantization. Currently, there is limited understanding of quantization's impact on various task performances, including LLM tasks, LMM tasks, and, critically, trust and safety. There is a lack of adequate tools for systematically testing these models on mobile devices. To address these gaps, we introduce MobileAIBench, a comprehensive benchmarking framework for evaluating mobile-optimized LLMs and LMMs. MobileAIBench assesses models across different sizes, quantization levels, and tasks, measuring latency and resource consumption on real devices. Our two-part open-source framework includes a library for running evaluations on desktops and an iOS app for on-device latency and hardware utilization measurements. Our thorough analysis aims to accelerate mobile AI research and deployment by providing insights into the performance and feasibility of deploying LLMs and LMMs on mobile platforms.
@article{arxiv.2406.10290,
title = {MobileAIBench: Benchmarking LLMs and LMMs for On-Device Use Cases},
author = {Rithesh Murthy and Liangwei Yang and Juntao Tan and Tulika Manoj Awalgaonkar and Yilun Zhou and Shelby Heinecke and Sachin Desai and Jason Wu and Ran Xu and Sarah Tan and Jianguo Zhang and Zhiwei Liu and Shirley Kokane and Zuxin Liu and Ming Zhu and Huan Wang and Caiming Xiong and Silvio Savarese},
journal= {arXiv preprint arXiv:2406.10290},
year = {2024}
}