While large models such as GPT-3 demonstrate exceptional performance in zeroshot and fewshot summarization tasks, their extensive serving and fine-tuning costs hinder their utilization in various applications. Conversely, previous studies have found that although automatic metrics tend to favor smaller fine-tuned models, the quality of the summaries they generate is inferior to that of larger models like GPT-3 when assessed by human evaluators. To address this issue, we propose InheritSumm, a versatile and compact summarization model derived from GPT-3.5 through distillation. InheritSumm not only exhibits comparable zeroshot and fewshot summarization capabilities to GPT-3.5 but is also sufficiently compact for fine-tuning purposes. Experimental results demonstrate that InheritSumm achieves similar or superior performance to GPT-3.5 in zeroshot and fewshot settings. Furthermore, it outperforms the previously established best small models in both prefix-tuning and full-data fine-tuning scenarios.
@article{arxiv.2305.13083,
title = {InheritSumm: A General, Versatile and Compact Summarizer by Distilling from GPT},
author = {Yichong Xu and Ruochen Xu and Dan Iter and Yang Liu and Shuohang Wang and Chenguang Zhu and Michael Zeng},
journal= {arXiv preprint arXiv:2305.13083},
year = {2023}
}