It has been shown that injecting noise into the neural network weights during the training process leads to a better generalization of the resulting model. Noise injection in the distributed setup is a straightforward technique and it represents a promising approach to improve the locally trained models. We investigate the effects of noise injection into the neural networks during a decentralized training process. We show both theoretically and empirically that noise injection has no positive effect in expectation on linear models, though. However for non-linear neural networks we empirically show that noise injection substantially improves model quality helping to reach a generalization ability of a local model close to the serial baseline.
@article{arxiv.1809.10678,
title = {Introducing Noise in Decentralized Training of Neural Networks},
author = {Linara Adilova and Nathalie Paul and Peter Schlicht},
journal= {arXiv preprint arXiv:1809.10678},
year = {2018}
}