This empirical study analyzes the effects of the pre-training corpus on the quality of learned transformer representations. We focus on the representation quality induced solely through pre-training. Our experiments show that pre-training on a small, specialized corpus can yield effective representations, and that the success of combining a generic and a specialized corpus depends on the distributional similarity between the target task and the specialized corpus.
Cite
@article{arxiv.2505.24455,
title = {Domain Pre-training Impact on Representations},
author = {Cesar Gonzalez-Gutierrez and Ariadna Quattoni},
journal= {arXiv preprint arXiv:2505.24455},
year = {2025}
}