Recent breakthroughs in multi-talker ASR (MT-ASR) and speaker diarization (SD) rely on synthetic data to mitigate the scarcity of large-scale conversational recordings, yet the impact of specific simulation choices remains poorly understood. To mind the gap between simulated mixtures and real-world interactions, we present a study of synthetic data generation for leading MT-ASR (DiCoW) and SD (Sortformer) systems. By introducing FastMSS, a highly efficient open-source simulator, we analyze turn-taking dynamics, source domain, acoustic augmentation, and data mixing strategies. Our findings reveal that optimal simulation recipes are highly task-dependent: increasing speech overlap benefits ASR but degrades diarization. Furthermore, broad source diversity consistently outperforms exact domain matching. Ultimately, synthetic-only training approaches real-data baselines, and combining simulated data with real recordings yields substantial gains over real-only training across both tasks.
@article{arxiv.2605.15442,
title = {Mind the Gap: Impact of Synthetic Conversational Data on Multi-Talker ASR and Speaker Diarization},
author = {Alexander Polok and Ivan Medennikov and Jan Černocký and Shinji Watanabe and Lukáš Burget and Samuele Cornell},
journal= {arXiv preprint arXiv:2605.15442},
year = {2026}
}