Training and deploying large-scale machine learning models is time-consuming, requires significant distributed computing infrastructures, and incurs high operational costs. Our analysis, grounded in real-world large model training on datacenter-scale infrastructures, reveals that 14~32% of all GPU hours are spent on communication with no overlapping computation. To minimize this outstanding communication latency and other inherent at-scale inefficiencies, we introduce an agile performance modeling framework, MAD-Max. This framework is designed to optimize parallelization strategies and facilitate hardware-software co-design opportunities. Through the application of MAD-Max to a suite of real-world large-scale ML models on state-of-the-art GPU clusters, we showcase potential throughput enhancements of up to 2.24x for pre-training and up to 5.2x for inference scenarios, respectively.
@article{arxiv.2310.02784,
title = {MAD Max Beyond Single-Node: Enabling Large Machine Learning Model Acceleration on Distributed Systems},
author = {Samuel Hsia and Alicia Golden and Bilge Acun and Newsha Ardalani and Zachary DeVito and Gu-Yeon Wei and David Brooks and Carole-Jean Wu},
journal= {arXiv preprint arXiv:2310.02784},
year = {2024}
}