Social understanding abilities are crucial for multimodal large language models (MLLMs) to interpret human social interactions. We introduce Social Caption, a framework grounded in interaction theory to evaluate social understanding abilities of MLLMs along three dimensions: Social Inference (SI), the ability to make accurate inferences about interactions; Holistic Social Analysis (HSA), the ability to generate comprehensive descriptions of interactions; Directed Social Analysis (DSA), the ability to extract relevant social information from interactions. We analyze factors influencing model performance in social understanding, such as scale, architectural design, and spoken context. Experiments with MLLM judges contribute insights about scaling automated evaluation of multimodal social understanding.
@article{arxiv.2601.14569,
title = {Social Caption: Evaluating Social Understanding in Multimodal Models},
author = {Bhaavanaa Thumu and Leena Mathur and Youssouf Kebe and Louis-Philippe Morency},
journal= {arXiv preprint arXiv:2601.14569},
year = {2026}
}