As opportunities for AI-assisted healthcare grow steadily, model deployment faces challenges due to the specific characteristics of the industry. The configuration choice for a production device can impact model performance while influencing operational costs. Moreover, in healthcare some situations might require fast, but not real time, inference. We study different configurations and conduct a cost-performance analysis to determine the optimized hardware for the deployment of a model subject to healthcare domain constraints. We observe that a naive performance comparison may not lead to an optimal configuration selection. In fact, given realistic domain constraints, CPU execution might be preferable to GPU accelerators. Hence, defining beforehand precise expectations for model deployment is crucial.
@article{arxiv.1910.03060,
title = {Impact of Inference Accelerators on hardware selection},
author = {Dibyajyoti Pati and Caroline Favart and Purujit Bahl and Vivek Soni and Yun-chan Tsai and Michael Potter and Jiahui Guan and Xiaomeng Dong and V. Ratna Saripalli},
journal= {arXiv preprint arXiv:1910.03060},
year = {2019}
}