In NLP classification tasks where little labeled data exists, domain fine-tuning of transformer models on unlabeled data is an established approach. In this paper we have two aims. (1) We describe our observations from fine-tuning the Finnish BERT model on Finnish medical text data. (2) We report on our attempts to predict the benefit of domain-specific pre-training of Finnish BERT from observing the geometry of embedding changes due to domain fine-tuning. Our driving motivation is the common\situation in healthcare AI where we might experience long delays in acquiring datasets, especially with respect to labels.
@article{arxiv.2604.14815,
title = {Domain Fine-Tuning FinBERT on Finnish Histopathological Reports: Train-Time Signals and Downstream Correlations},
author = {Rami Luisto and Liisa Petäinen and Tommi Grönholm and Jan Böhm and Maarit Ahtiainen and Tomi Lilja and Ilkka Pölönen and Sami Äyrämö},
journal= {arXiv preprint arXiv:2604.14815},
year = {2026}
}