The inherent challenge of multimodal fusion is to precisely capture the cross-modal correlation and flexibly conduct cross-modal interaction. To fully release the value of each modality and mitigate the influence of low-quality multimodal data, dynamic multimodal fusion emerges as a promising learning paradigm. Despite its widespread use, theoretical justifications in this field are still notably lacking. Can we design a provably robust multimodal fusion method? This paper provides theoretical understandings to answer this question under a most popular multimodal fusion framework from the generalization perspective. We proceed to reveal that several uncertainty estimation solutions are naturally available to achieve robust multimodal fusion. Then a novel multimodal fusion framework termed Quality-aware Multimodal Fusion (QMF) is proposed, which can improve the performance in terms of classification accuracy and model robustness. Extensive experimental results on multiple benchmarks can support our findings.
@article{arxiv.2306.02050,
title = {Provable Dynamic Fusion for Low-Quality Multimodal Data},
author = {Qingyang Zhang and Haitao Wu and Changqing Zhang and Qinghua Hu and Huazhu Fu and Joey Tianyi Zhou and Xi Peng},
journal= {arXiv preprint arXiv:2306.02050},
year = {2023}
}