Transfer learning techniques are particularly useful in NLP tasks where a sizable amount of high-quality annotated data is difficult to obtain. Current approaches directly adapt a pre-trained language model (LM) on in-domain text before fine-tuning to downstream tasks. We show that extending the vocabulary of the LM with domain-specific terms leads to further gains. To a bigger effect, we utilize structure in the unlabeled data to create auxiliary synthetic tasks, which helps the LM transfer to downstream tasks. We apply these approaches incrementally on a pre-trained Roberta-large LM and show considerable performance gain on three tasks in the IT domain: Extractive Reading Comprehension, Document Ranking and Duplicate Question Detection.
@article{arxiv.2010.05904,
title = {Multi-Stage Pre-training for Low-Resource Domain Adaptation},
author = {Rong Zhang and Revanth Gangi Reddy and Md Arafat Sultan and Vittorio Castelli and Anthony Ferritto and Radu Florian and Efsun Sarioglu Kayi and Salim Roukos and Avirup Sil and Todd Ward},
journal= {arXiv preprint arXiv:2010.05904},
year = {2020}
}