The rise of Internet of Things (IoT) devices in the physical world necessitates voice-based interfaces capable of handling complex user experiences. While modern Large Language Models (LLMs) already demonstrate strong tool-usage capabilities, modeling real-world IoT devices presents a difficult, understudied challenge which combines modeling spatiotemporal constraints with speech inputs, dynamic state tracking, and mixed-initiative interaction patterns. We introduce MIST (the Multimodal Interactive Speech-based Tool-calling Dataset), a synthetic multi-turn, voice-driven code generation task that operates over IoT devices. We find that there is a significant gap between open- and closed-weight multimodal LLMs on MIST, and that even frontier closed-weight LLMs have substantial headroom. We release MIST and an extensible data generation framework to build related datasets in order to facilitate research on mixed-initiative voice assistants which reason about physical world constraints.
@article{arxiv.2605.06897,
title = {MIST: Multimodal Interactive Speech-based Tool-calling Conversational Assistants for Smart Homes},
author = {Maximillian Chen and Xuanming Zhang and Michael Peng and Zhou Yu and Alexandros Papangelis and Yohan Jo},
journal= {arXiv preprint arXiv:2605.06897},
year = {2026}
}