With the growing amount of text in health data, there have been rapid advances in large pre-trained models that can be applied to a wide variety of biomedical tasks with minimal task-specific modifications. Emphasizing the cost of these models, which renders technical replication challenging, this paper summarizes experiments conducted in replicating BioBERT and further pre-training and careful fine-tuning in the biomedical domain. We also investigate the effectiveness of domain-specific and domain-agnostic pre-trained models across downstream biomedical NLP tasks. Our finding confirms that pre-trained models can be impactful in some downstream NLP tasks (QA and NER) in the biomedical domain; however, this improvement may not justify the high cost of domain-specific pre-training.
@article{arxiv.2012.15419,
title = {An Experimental Evaluation of Transformer-based Language Models in the Biomedical Domain},
author = {Paul Grouchy and Shobhit Jain and Michael Liu and Kuhan Wang and Max Tian and Nidhi Arora and Hillary Ngai and Faiza Khan Khattak and Elham Dolatabadi and Sedef Akinli Kocak},
journal= {arXiv preprint arXiv:2012.15419},
year = {2021}
}