Simulation offers unique values for both enumeration and extrapolation purposes, and is becoming increasingly important for managing the massive machine learning (ML) clusters and large-scale distributed training jobs. In this paper, we build Echo to tackle three key challenges in large-scale training simulation: (1) tracing the runtime training workloads at each device in an ex-situ fashion so we can use a single device to obtain the actual execution graphs of 1K-GPU training, (2) accurately estimating the collective communication without high overheads of discrete-event based network simulation, and (3) accounting for the interference-induced computation slowdown from overlapping communication and computation kernels on the same device. Echo delivers on average 8% error in training step -- roughly 3x lower than state-of-the-art simulators -- for GPT-175B on a 96-GPU H800 cluster with 3D parallelism on Megatron-LM under 2 minutes.
@article{arxiv.2412.12487,
title = {Echo: Simulating Distributed Training At Scale},
author = {Yicheng Feng and Yuetao Chen and Kaiwen Chen and Jingzong Li and Tianyuan Wu and Peng Cheng and Chuan Wu and Wei Wang and Tsung-Yi Ho and Hong Xu},
journal= {arXiv preprint arXiv:2412.12487},
year = {2024}
}