Fine-tuning language models in a downstream task is the standard approach for many state-of-the-art methodologies in the field of NLP. However, when the distribution between the source task and target task drifts, \textit{e.g.}, conversational environments, these gains tend to be diminished. This article proposes a sequence of pre-training steps (a curriculum) guided by "data hacking" and grammar analysis that allows further gradual adaptation between pre-training distributions. In our experiments, we acquire a considerable improvement from our method compared to other known pre-training approaches for the MultiWoZ task.
@article{arxiv.2308.01849,
title = {Curricular Transfer Learning for Sentence Encoded Tasks},
author = {Jader Martins Camboim de Sá and Matheus Ferraroni Sanches and Rafael Roque de Souza and Júlio Cesar dos Reis and Leandro Aparecido Villas},
journal= {arXiv preprint arXiv:2308.01849},
year = {2023}
}