We introduce an Analyze-Revise-Finetune (ARF) pipeline that enables smaller open-source language models (LLMs) to surpass substantially larger proprietary models in customer service summarization tasks. The pipeline first analyzes and categorizes common errors in summaries produced by a teacher model (GPT-3.5), then performs a targeted revision using a compact editor model (Llama 3.1 70B) to generate high-quality, refined training data. Fine-tuning smaller student models (e.g., Llama 3.1 8B, QWen3 4B) on this refined data resulted in superior summarization performance compared to GPT-3.5. The ARF pipeline improves cost efficiency and data privacy while maintaining competitive accuracy, illustrating a generalizable framework for enhancing open-source LLMs across diverse downstream applications.
@article{arxiv.2511.03005,
title = {Error-Aware Knowledge Distillation via Targeted Revision for Customer-Service Summarization},
author = {Hee-Jin Lee and Zhen Guo and Luchao Jin and Morteza Moazami Goudarzi},
journal= {arXiv preprint arXiv:2511.03005},
year = {2026}
}