Faster inference of deep learning models is highly demanded on edge devices and even servers, for both financial and environmental reasons. To address this issue, we propose SoftNeuro, a novel, high-performance inference framework with efficient performance tuning. The key idea is to separate algorithmic routines from network layers. Our framework maximizes the inference performance by profiling various routines for each layer and selecting the fastest path. To efficiently find the best path, we propose a routine-selection algorithm based on dynamic programming. Experiments show that the proposed framework achieves both fast inference and efficient tuning.
@article{arxiv.2110.06037,
title = {SoftNeuro: Fast Deep Inference using Multi-platform Optimization},
author = {Masaki Hilaga and Yasuhiro Kuroda and Hitoshi Matsuo and Tatsuya Kawaguchi and Gabriel Ogawa and Hiroshi Miyake and Yusuke Kozawa},
journal= {arXiv preprint arXiv:2110.06037},
year = {2021}
}