Neural Networks design is a complex and often daunting task, particularly for resource-constrained scenarios typical of mobile-sized models. Neural Architecture Search is a promising approach to automate this process, but existing competitive methods require large training time and computational resources to generate accurate models. To overcome these limits, this paper contributes with: i) a novel training-free metric, named Entropic Score, to estimate model expressivity through the aggregated element-wise entropy of its activations; ii) a cyclic search algorithm to separately yet synergistically search model size and topology. Entropic Score shows remarkable ability in searching for the topology of the network, and a proper combination with LogSynflow, to search for model size, yields superior capability to completely design high-performance Hybrid Transformers for edge applications in less than 1 GPU hour, resulting in the fastest and most accurate NAS method for ImageNet classification.
@article{arxiv.2310.04179,
title = {Entropic Score metric: Decoupling Topology and Size in Training-free NAS},
author = {Niccolò Cavagnero and Luca Robbiano and Francesca Pistilli and Barbara Caputo and Giuseppe Averta},
journal= {arXiv preprint arXiv:2310.04179},
year = {2023}
}