In this work, we introduce a novel approach for optimizing LLM training by adjusting learning rates across weights of different components in Transformer models. Traditional methods often apply a uniform learning rate across all network layers, potentially overlooking the unique dynamics of each part. Remarkably, our introduced relative learning rates, RLRS, method accelerates the training process by up to 23%, particularly in complex models such as Mixture of Experts (MoE). Hyperparameters of RLRS can be efficiently tuned on smaller models and then effectively reused on models up to 27× larger. This simple and effective method results in a substantial reduction in training time and computational resources, offering a practical and scalable solution for optimizing large-scale neural networks.
@article{arxiv.2507.03526,
title = {Decoupled Relative Learning Rate Schedules},
author = {Jan Ludziejewski and Jan Małaśnicki and Maciej Pióro and Michał Krutul and Kamil Ciebiera and Maciej Stefaniak and Jakub Krajewski and Piotr Sankowski and Marek Cygan and Kamil Adamczewski and Sebastian Jaszczur},
journal= {arXiv preprint arXiv:2507.03526},
year = {2025}
}