We describe a general framework for online adaptation of optimization hyperparameters by `hot swapping' their values during learning. We investigate this approach in the context of adaptive learning rate selection using an explore-exploit strategy from the multi-armed bandit literature. Experiments on a benchmark neural network show that the hot swapping approach leads to consistently better solutions compared to well-known alternatives such as AdaDelta and stochastic gradient with exhaustive hyperparameter search.
Cite
@article{arxiv.1412.6599,
title = {Hot Swapping for Online Adaptation of Optimization Hyperparameters},
author = {Kevin Bache and Dennis DeCoste and Padhraic Smyth},
journal= {arXiv preprint arXiv:1412.6599},
year = {2015}
}