Large language models (LLMs) have achieved significant progress in natural language processing but face challenges in deployment due to high memory and computational requirements. Weight quantization is a common approach to address these issues, yet achieving effective low-bit compression remains challenging. This paper presents LCD, which unifies the learning of clustering-based quantization within a knowledge distillation framework. Using carefully designed optimization techniques, LCD preserves LLM performance even at ultra-low bit widths of 2-3 bits. Additionally, LCD compresses activations through smoothing and accelerates inference with a LUT-based design. Experimental results show that LCD outperforms existing methods and delivers up to a 6.2x speedup in inference. Notably, LCD is shown to be more cost-effective, making it a practical solution for real-world applications.
@article{arxiv.2506.12038,
title = {LCD: Advancing Extreme Low-Bit Clustering for Large Language Models via Knowledge Distillation},
author = {Fangxin Liu and Ning Yang and Junping Zhao and Tao Yang and Haibing Guan and Li Jiang},
journal= {arXiv preprint arXiv:2506.12038},
year = {2025}
}