While large language models a la BERT are used ubiquitously in NLP, pretraining them is considered a luxury that only a few well-funded industry labs can afford. How can one train such models with a more modest budget? We present a recipe for pretraining a masked language model in 24 hours using a single low-end deep learning server. We demonstrate that through a combination of software optimizations, design choices, and hyperparameter tuning, it is possible to produce models that are competitive with BERT-base on GLUE tasks at a fraction of the original pretraining cost.
@article{arxiv.2104.07705,
title = {How to Train BERT with an Academic Budget},
author = {Peter Izsak and Moshe Berchansky and Omer Levy},
journal= {arXiv preprint arXiv:2104.07705},
year = {2021}
}