Recent explorations of large-scale pre-trained language models (PLMs) have revealed the power of PLMs with huge amounts of parameters, setting off a wave of training ever-larger PLMs. However, it requires tremendous computational resources to train a large-scale PLM, which may be practically unaffordable. In addition, existing large-scale PLMs are mainly trained from scratch individually, ignoring that many well-trained PLMs are available. To this end, we explore the question how could existing PLMs benefit training large-scale PLMs in future. Specifically, we introduce a pre-training framework named "knowledge inheritance" (KI) and explore how could knowledge distillation serve as auxiliary supervision during pre-training to efficiently learn larger PLMs. Experimental results demonstrate the superiority of KI in training efficiency. We also conduct empirical analyses to explore the effects of teacher PLMs' pre-training settings, including model architecture, pre-training data, etc. Finally, we show that KI could be applied to domain adaptation and knowledge transfer.
@article{arxiv.2105.13880,
title = {Knowledge Inheritance for Pre-trained Language Models},
author = {Yujia Qin and Yankai Lin and Jing Yi and Jiajie Zhang and Xu Han and Zhengyan Zhang and Yusheng Su and Zhiyuan Liu and Peng Li and Maosong Sun and Jie Zhou},
journal= {arXiv preprint arXiv:2105.13880},
year = {2022}
}