This paper aims at enlarging the problem of Neural Architecture Search (NAS) from Single-Path and Multi-Path Search to automated Mixed-Path Search. In particular, we model the NAS problem as a sparse supernet using a new continuous architecture representation with a mixture of sparsity constraints. The sparse supernet enables us to automatically achieve sparsely-mixed paths upon a compact set of nodes. To optimize the proposed sparse supernet, we exploit a hierarchical accelerated proximal gradient algorithm within a bi-level optimization framework. Extensive experiments on Convolutional Neural Network and Recurrent Neural Network search demonstrate that the proposed method is capable of searching for compact, general and powerful neural architectures.
@article{arxiv.2007.16112,
title = {Neural Architecture Search as Sparse Supernet},
author = {Yan Wu and Aoming Liu and Zhiwu Huang and Siwei Zhang and Luc Van Gool},
journal= {arXiv preprint arXiv:2007.16112},
year = {2021}
}