Language models like BERT and SpanBERT pretrained on open-domain data have obtained impressive gains on various NLP tasks. In this paper, we probe the effectiveness of domain-adaptive pretraining objectives on downstream tasks. In particular, three objectives, including a novel objective focusing on modeling predicate-argument relations, are evaluated on two challenging dialogue understanding tasks. Experimental results demonstrate that domain-adaptive pretraining with proper objectives can significantly improve the performance of a strong baseline on these tasks, achieving the new state-of-the-art performances.
@article{arxiv.2105.13665,
title = {Domain-Adaptive Pretraining Methods for Dialogue Understanding},
author = {Han Wu and Kun Xu and Linfeng Song and Lifeng Jin and Haisong Zhang and Linqi Song},
journal= {arXiv preprint arXiv:2105.13665},
year = {2021}
}