We propose a unified framework for neural net normalization, regularization and optimization, which includes Path-SGD and Batch-Normalization and interpolates between them across two different dimensions. Through this framework we investigate issue of invariance of the optimization, data dependence and the connection with natural gradients.
@article{arxiv.1511.06747,
title = {Data-Dependent Path Normalization in Neural Networks},
author = {Behnam Neyshabur and Ryota Tomioka and Ruslan Salakhutdinov and Nathan Srebro},
journal= {arXiv preprint arXiv:1511.06747},
year = {2016}
}