Training machine learning models in parallel is an increasingly important workload. We accelerate distributed parallel training by designing a communication primitive that uses a programmable switch dataplane to execute a key step of the training process. Our approach, SwitchML, reduces the volume of exchanged data by aggregating the model updates from multiple workers in the network. We co-design the switch processing with the end-host protocols and ML frameworks to provide an efficient solution that speeds up training by up to 5.5× for a number of real-world benchmark models.
@article{arxiv.1903.06701,
title = {Scaling Distributed Machine Learning with In-Network Aggregation},
author = {Amedeo Sapio and Marco Canini and Chen-Yu Ho and Jacob Nelson and Panos Kalnis and Changhoon Kim and Arvind Krishnamurthy and Masoud Moshref and Dan R. K. Ports and Peter Richtárik},
journal= {arXiv preprint arXiv:1903.06701},
year = {2020}
}