This paper presents a simple yet principled approach to boosting the robustness of the residual network (ResNet) that is motivated by the dynamical system perspective. Namely, a deep neural network can be interpreted using a partial differential equation, which naturally inspires us to characterize ResNet by an explicit Euler method. Our analytical studies reveal that the step factor h in the Euler method is able to control the robustness of ResNet in both its training and generalization. Specifically, we prove that a small step factor h can benefit the training robustness for back-propagation; from the view of forward-propagation, a small h can aid in the robustness of the model generalization. A comprehensive empirical evaluation on both vision CIFAR-10 and text AG-NEWS datasets confirms that a small h aids both the training and generalization robustness.
@article{arxiv.1902.10887,
title = {Towards Robust ResNet: A Small Step but A Giant Leap},
author = {Jingfeng Zhang and Bo Han and Laura Wynter and Kian Hsiang Low and Mohan Kankanhalli},
journal= {arXiv preprint arXiv:1902.10887},
year = {2019}
}