We present the surprising result that randomly initialized neural networks are good feature extractors in expectation. These random features correspond to finite-sample realizations of what we call Neural Network Prior Kernel (NNPK), which is inherently infinite-dimensional. We conduct ablations across multiple architectures of varying sizes as well as initializations and activation functions. Our analysis suggests that certain structures that manifest in a trained model are already present at initialization. Therefore, NNPK may provide further insight into why neural networks are so effective in learning such structures.
@article{arxiv.2202.06438,
title = {Learning from Randomly Initialized Neural Network Features},
author = {Ehsan Amid and Rohan Anil and Wojciech Kotłowski and Manfred K. Warmuth},
journal= {arXiv preprint arXiv:2202.06438},
year = {2022}
}