There is growing interest in applying AI to radiology report generation, particularly for chest X-rays (CXRs). This paper investigates whether incorporating pixel-level information through segmentation masks can improve fine-grained image interpretation of multimodal large language models (MLLMs) for radiology report generation. We introduce MAIRA-Seg, a segmentation-aware MLLM framework designed to utilize semantic segmentation masks alongside CXRs for generating radiology reports. We train expert segmentation models to obtain mask pseudolabels for radiology-specific structures in CXRs. Subsequently, building on the architectures of MAIRA, a CXR-specialised model for report generation, we integrate a trainable segmentation tokens extractor that leverages these mask pseudolabels, and employ mask-aware prompting to generate draft radiology reports. Our experiments on the publicly available MIMIC-CXR dataset show that MAIRA-Seg outperforms non-segmentation baselines. We also investigate set-of-marks prompting with MAIRA and find that MAIRA-Seg consistently demonstrates comparable or superior performance. The results confirm that using segmentation masks enhances the nuanced reasoning of MLLMs, potentially contributing to better clinical outcomes.
@article{arxiv.2411.11362,
title = {MAIRA-Seg: Enhancing Radiology Report Generation with Segmentation-Aware Multimodal Large Language Models},
author = {Harshita Sharma and Valentina Salvatelli and Shaury Srivastav and Kenza Bouzid and Shruthi Bannur and Daniel C. Castro and Maximilian Ilse and Sam Bond-Taylor and Mercy Prasanna Ranjit and Fabian Falck and Fernando Pérez-García and Anton Schwaighofer and Hannah Richardson and Maria Teodora Wetscherek and Stephanie L. Hyland and Javier Alvarez-Valle},
journal= {arXiv preprint arXiv:2411.11362},
year = {2024}
}