Training large neural language models on large datasets is resource- and time-intensive. These requirements create a barrier to entry, where those with fewer resources cannot build competitive models. This paper presents various techniques for making it possible to (a) train a large language model using resources that a modest research lab might have, and (b) train it in a reasonable amount of time. We provide concrete recommendations for practitioners, which we illustrate with a case study: a T5 model for Danish, the first for this language.
@article{arxiv.2208.12097,
title = {Training a T5 Using Lab-sized Resources},
author = {Manuel R. Ciosici and Leon Derczynski},
journal= {arXiv preprint arXiv:2208.12097},
year = {2022}
}