We compute the minimum-norm weights of a deep linear ResNet, and find that the inductive bias of this architecture lies between minimizing nuclear norm and rank. This implies that, with appropriate hyperparameters, deep nonlinear ResNets have an inductive bias towards minimizing bottleneck rank.
@article{arxiv.2501.19149,
title = {On the inductive bias of infinite-depth ResNets and the bottleneck rank},
author = {Enric Boix-Adsera},
journal= {arXiv preprint arXiv:2501.19149},
year = {2025}
}