Decades of research indicate that emotion recognition is more effective when drawing information from multiple modalities. But what if some modalities are sometimes missing? To address this problem, we propose a novel Transformer-based architecture for recognizing valence and arousal in a time-continuous manner even with missing input modalities. We use a coupling of cross-attention and self-attention mechanisms to emphasize relationships between modalities during time and enhance the learning process on weak salient inputs. Experimental results on the Ulm-TSST dataset show that our model exhibits an improvement of the concordance correlation coefficient evaluation of 37% when predicting arousal values and 30% when predicting valence values, compared to a late-fusion baseline approach.
@article{arxiv.2311.10119,
title = {Accommodating Missing Modalities in Time-Continuous Multimodal Emotion Recognition},
author = {Juan Vazquez-Rodriguez and Grégoire Lefebvre and Julien Cumin and James L. Crowley},
journal= {arXiv preprint arXiv:2311.10119},
year = {2023}
}