Memory has become an increasingly important component of agentic systems, as these systems are expected to reason over long-term experience. However, prior work has largely focused on unimodal memory, leaving multimodal memory relatively underexplored despite its central role in real-world applications. Compared with unimodal settings, multimodal memory introduces additional challenges, including heterogeneous input integration, person-centric information alignment, and evidence aggregation across different granularities. We present PyraVid, a hierarchical multimodal memory framework inspired by Event Segmentation Theory from cognitive science. PyraVid organizes long videos into a coarse-to-fine pyramid structure, enabling structured memory access and effective evidence aggregation. It further supports structure-guided memory expansion with pruning, allowing the retrieval of related events with strong causal connectivity but low semantic similarity while reducing noise. Experiments on multiple long-video understanding benchmarks show that PyraVid consistently improves performance across datasets, model scales, and question types, highlighting the effectiveness of hierarchical multimodal memory for long-horizon reasoning.
@article{arxiv.2605.17065,
title = {PyraVid: Hierarchical Multimodal Memory for Long-Horizon Video Reasoning},
author = {Sikuan Yan and Sicheng Dong and Haotong Wang and Ercong Nie and Yilun Liu and Jinhe Bi and Yingjie Xu and Susanna Schwarzmann and Riccardo Trivisonno and Volker Tresp and Yunpu Ma},
journal= {arXiv preprint arXiv:2605.17065},
year = {2026}
}