Conventional stochastic rounding (CSR) is widely employed in the training of neural networks (NNs), showing promising training results even in low-precision computations. We introduce an improved stochastic rounding method, that is simple and efficient. The proposed method succeeds in training NNs with 16-bit fixed-point numbers and provides faster convergence and higher classification accuracy than both CSR and deterministic rounding-to-the-nearest method.
@article{arxiv.2103.13445,
title = {A Simple and Efficient Stochastic Rounding Method for Training Neural Networks in Low Precision},
author = {Lu Xia and Martijn Anthonissen and Michiel Hochstenbach and Barry Koren},
journal= {arXiv preprint arXiv:2103.13445},
year = {2021}
}