Accurate segmentation of articulatory structures in real-time MRI (rtMRI) remains challenging, as existing methods rely primarily on visual cues and overlook complementary information from synchronized speech signals. We propose VocSegMRI, a multimodal framework integrating video, audio, and phonological inputs via cross-attention fusion and a contrastive learning objective that improves cross-modal alignment and segmentation precision. Evaluated on USC-75 and further validated via zero-shot transfer on USC-TIMIT, VocSegMRI outperforms unimodal and multimodal baselines, with ablations confirming the contribution of each component.
@article{arxiv.2509.13767,
title = {VocSegMRI: Multimodal Learning for Precise Vocal Tract Segmentation in Real-time MRI},
author = {Daiqi Liu and Johannes Enk and Maureen Stone and Fangxu Xing and Tomás Arias-Vergara and Jerry L. Prince and Jana Hutter and Jonghye Woo and Andreas Maier and Paula Andrea Pérez-Toro},
journal= {arXiv preprint arXiv:2509.13767},
year = {2026}
}