We demonstrate that transformers obtain impressive performance even when some of the layers are randomly initialized and never updated. Inspired by old and well-established ideas in machine learning, we explore a variety of non-linear "reservoir" layers interspersed with regular transformer layers, and show improvements in wall-clock compute time until convergence, as well as overall performance, on various machine translation and (masked) language modelling tasks.
@article{arxiv.2012.15045,
title = {Reservoir Transformers},
author = {Sheng Shen and Alexei Baevski and Ari S. Morcos and Kurt Keutzer and Michael Auli and Douwe Kiela},
journal= {arXiv preprint arXiv:2012.15045},
year = {2021}
}