This work aims to assess the state of the art of data parallel deep neural network training, trying to identify potential research tracks to be exploited for performance improvement. Beside, it presents a design for a practical C++ library dedicated at implementing and unifying the current state of the art methodologies for parallel training in a performance-conscious framework, allowing the user to explore novel strategies without departing significantly from its usual work-flow.
@article{arxiv.1806.09528,
title = {Pushing the boundaries of parallel Deep Learning -- A practical approach},
author = {Paolo Viviani and Maurizio Drocco and Marco Aldinucci},
journal= {arXiv preprint arXiv:1806.09528},
year = {2018}
}