While existing work on neural architecture search (NAS) tunes hyperparameters in a separate post-processing step, we demonstrate that architectural choices and other hyperparameter settings interact in a way that can render this separation suboptimal. Likewise, we demonstrate that the common practice of using very few epochs during the main NAS and much larger numbers of epochs during a post-processing step is inefficient due to little correlation in the relative rankings for these two training regimes. To combat both of these problems, we propose to use a recent combination of Bayesian optimization and Hyperband for efficient joint neural architecture and hyperparameter search.
@article{arxiv.1807.06906,
title = {Towards Automated Deep Learning: Efficient Joint Neural Architecture and Hyperparameter Search},
author = {Arber Zela and Aaron Klein and Stefan Falkner and Frank Hutter},
journal= {arXiv preprint arXiv:1807.06906},
year = {2018}
}