Recent multimodal large language models (MLLMs) have demonstrated significant potential in open-ended conversation, generating more accurate and personalized responses. However, their abilities to memorize, recall, and reason in sustained interactions within real-world scenarios remain underexplored. This paper introduces MMRC, a Multi-Modal Real-world Conversation benchmark for evaluating six core open-ended abilities of MLLMs: information extraction, multi-turn reasoning, information update, image management, memory recall, and answer refusal. With data collected from real-world scenarios, MMRC comprises 5,120 conversations and 28,720 corresponding manually labeled questions, posing a significant challenge to existing MLLMs. Evaluations on 20 MLLMs in MMRC indicate an accuracy drop during open-ended interactions. We identify four common failure patterns: long-term memory degradation, inadequacies in updating factual knowledge, accumulated assumption of error propagation, and reluctance to say no. To mitigate these issues, we propose a simple yet effective NOTE-TAKING strategy, which can record key information from the conversation and remind the model during its responses, enhancing conversational capabilities. Experiments across six MLLMs demonstrate significant performance improvements.
@article{arxiv.2502.11903,
title = {MMRC: A Large-Scale Benchmark for Understanding Multimodal Large Language Model in Real-World Conversation},
author = {Haochen Xue and Feilong Tang and Ming Hu and Yexin Liu and Qidong Huang and Yulong Li and Chengzhi Liu and Zhongxing Xu and Chong Zhang and Chun-Mei Feng and Yutong Xie and Imran Razzak and Zongyuan Ge and Jionglong Su and Junjun He and Yu Qiao},
journal= {arXiv preprint arXiv:2502.11903},
year = {2025}
}