In this paper, we present a scalable distributed implementation of the Sampled Limited-memory Symmetric Rank-1 (S-LSR1) algorithm. First, we show that a naive distributed implementation of S-LSR1 requires multiple rounds of expensive communications at every iteration and thus is inefficient. We then propose DS-LSR1, a communication-efficient variant that: (i) drastically reduces the amount of data communicated at every iteration, (ii) has favorable work-load balancing across nodes, and (iii) is matrix-free and inverse-free. The proposed method scales well in terms of both the dimension of the problem and the number of data points. Finally, we illustrate the empirical performance of DS-LSR1 on a standard neural network training task.
@article{arxiv.1905.13096,
title = {Scaling Up Quasi-Newton Algorithms: Communication Efficient Distributed SR1},
author = {Majid Jahani and Mohammadreza Nazari and Sergey Rusakov and Albert S. Berahas and Martin Takáč},
journal= {arXiv preprint arXiv:1905.13096},
year = {2020}
}