The semi-supervised learning (SSL) strategy in lightweight models requires reducing annotated samples and facilitating cost-effective inference. However, the constraint on model parameters, imposed by the scarcity of training labels, limits the SSL performance. In this paper, we introduce PS-NET, a novel framework tailored for semi-supervised text mining with lightweight models. PS-NET incorporates online distillation to train lightweight student models by imitating the Teacher model. It also integrates an ensemble of student peers that collaboratively instruct each other. Additionally, PS-NET implements a constant adversarial perturbation schema to further self-augmentation by progressive generalizing. Our PS-NET, equipped with a 2-layer distilled BERT, exhibits notable performance enhancements over SOTA lightweight SSL frameworks of FLiText and DisCo in SSL text classification with extremely rare labelled data.
@article{arxiv.2412.00883,
title = {Lightweight Contenders: Navigating Semi-Supervised Text Mining through Peer Collaboration and Self Transcendence},
author = {Qianren Mao and Weifeng Jiang and Junnan Liu and Chenghua Lin and Qian Li and Xianqing Wen and Jianxin Li and Jinhu Lu},
journal= {arXiv preprint arXiv:2412.00883},
year = {2024}
}