Given the exceptional performance of proprietary large language models (LLMs) like GPT-4, recent research has increasingly focused on boosting the capabilities of smaller models through knowledge distillation (KD) from these powerful yet black-box teachers. While leveraging the high-quality outputs of these teachers is advantageous, the inaccessibility of their internal states often limits effective knowledge transfer. To overcome this limitation, we introduce Proxy-KD, a novel method that uses a proxy model to facilitate the efficient transfer of knowledge from black-box LLMs to smaller models. Our experiments show that Proxy-KD not only enhances the performance of KD from black-box teacher models but also surpasses traditional white-box KD techniques.~This approach presents a compelling new avenue for distilling knowledge from advanced LLMs.
@article{arxiv.2401.07013,
title = {Knowledge Distillation of Black-Box Large Language Models},
author = {Hongzhan Chen and Ruijun Chen and Yuqi Yi and Xiaojun Quan and Chenliang Li and Ming Yan and Ji Zhang},
journal= {arXiv preprint arXiv:2401.07013},
year = {2024}
}