In the medical multi-modal frameworks, the alignment of cross-modality features presents a significant challenge. However, existing works have learned features that are implicitly aligned from the data, without considering the explicit relationships in the medical context. This data-reliance may lead to low generalization of the learned alignment relationships. In this work, we propose the Eye-gaze Guided Multi-modal Alignment (EGMA) framework to harness eye-gaze data for better alignment of medical visual and textual features. We explore the natural auxiliary role of radiologists' eye-gaze data in aligning medical images and text, and introduce a novel approach by using eye-gaze data, collected synchronously by radiologists during diagnostic evaluations. We conduct downstream tasks of image classification and image-text retrieval on four medical datasets, where EGMA achieved state-of-the-art performance and stronger generalization across different datasets. Additionally, we explore the impact of varying amounts of eye-gaze data on model performance, highlighting the feasibility and utility of integrating this auxiliary data into multi-modal alignment framework.
@article{arxiv.2403.12416,
title = {Eye-gaze Guided Multi-modal Alignment for Medical Representation Learning},
author = {Chong Ma and Hanqi Jiang and Wenting Chen and Yiwei Li and Zihao Wu and Xiaowei Yu and Zhengliang Liu and Lei Guo and Dajiang Zhu and Tuo Zhang and Dinggang Shen and Tianming Liu and Xiang Li},
journal= {arXiv preprint arXiv:2403.12416},
year = {2024}
}