We investigate the parameter-space geometry of recurrent neural networks (RNNs), and develop an adaptation of path-SGD optimization method, attuned to this geometry, that can learn plain RNNs with ReLU activations. On several datasets that require capturing long-term dependency structure, we show that path-SGD can significantly improve trainability of ReLU RNNs compared to RNNs trained with SGD, even with various recently suggested initialization schemes.
@article{arxiv.1605.07154,
title = {Path-Normalized Optimization of Recurrent Neural Networks with ReLU Activations},
author = {Behnam Neyshabur and Yuhuai Wu and Ruslan Salakhutdinov and Nathan Srebro},
journal= {arXiv preprint arXiv:1605.07154},
year = {2016}
}