Distributed training techniques have been widely deployed in large-scale deep neural networks (DNNs) training on dense-GPU clusters. However, on public cloud clusters, due to the moderate inter-connection bandwidth between instances, traditional state-of-the-art distributed training systems cannot scale well in training large-scale models. In this paper, we propose a new computing and communication efficient top-k sparsification communication library for distributed training. To further improve the system scalability, we optimize I/O by proposing a simple yet efficient multi-level data caching mechanism and optimize the update operation by introducing a novel parallel tensor operator. Experimental results on a 16-node Tencent Cloud cluster (each node with 8 Nvidia Tesla V100 GPUs) show that our system achieves 25%-40% faster than existing state-of-the-art systems on CNNs and Transformer. We finally break the record on DAWNBench on training ResNet-50 to 93% top-5 accuracy on ImageNet.
@article{arxiv.2010.10458,
title = {Towards Scalable Distributed Training of Deep Learning on Public Cloud Clusters},
author = {Shaohuai Shi and Xianhao Zhou and Shutao Song and Xingyao Wang and Zilin Zhu and Xue Huang and Xinan Jiang and Feihu Zhou and Zhenyu Guo and Liqiang Xie and Rui Lan and Xianbin Ouyang and Yan Zhang and Jieqian Wei and Jing Gong and Weiliang Lin and Ping Gao and Peng Meng and Xiaomin Xu and Chenyang Guo and Bo Yang and Zhibo Chen and Yongjian Wu and Xiaowen Chu},
journal= {arXiv preprint arXiv:2010.10458},
year = {2020}
}