Large Language Models (LLMs) have demonstrated impressive quality when applied to predictive tasks such as relevance ranking and semantic search. However, deployment of such LLMs remains prohibitively expensive for industry applications with strict latency and throughput requirements. In this work, we present lessons and efficiency insights from developing a purely text-based decoder-only Small Language Model (SLM) for a semantic search application at LinkedIn. Particularly, we discuss model compression techniques such as pruning that allow us to reduce the model size by up to 40% while maintaining the accuracy. Additionally, we present context compression techniques that allow us to reduce the input context length by up to 10x with minimal loss of accuracy. Finally, we present practical lessons from optimizing the serving infrastructure for deploying such a system on GPUs at scale, serving millions of requests per second. Taken together, this allows us to increase our system's throughput by 10x in a real-world deployment, while meeting our quality bar.
@article{arxiv.2510.22101,
title = {Scaling Up Efficient Small Language Models Serving and Deployment for Semantic Job Search},
author = {Kayhan Behdin and Qingquan Song and Sriram Vasudevan and Jian Sheng and Xiaojing Ma and Z Zhou and Chuanrui Zhu and Guoyao Li and Chanh Nguyen and Sayan Ghosh and Hejian Sang and Ata Fatahi Baarzi and Sundara Raman Ramachandran and Xiaoqing Wang and Qing Lan and Vinay Y S and Qi Guo and Caleb Johnson and Zhipeng Wang and Fedor Borisyuk},
journal= {arXiv preprint arXiv:2510.22101},
year = {2025}
}