The recent growth of web video sharing platforms has increased the demand for systems that can efficiently browse, retrieve and summarize video content. Query-aware multi-video summarization is a promising technique that caters to this demand. In this work, we introduce a novel Query-Aware Hierarchical Pointer Network for Multi-Video Summarization, termed DeepQAMVS, that jointly optimizes multiple criteria: (1) conciseness, (2) representativeness of important query-relevant events and (3) chronological soundness. We design a hierarchical attention model that factorizes over three distributions, each collecting evidence from a different modality, followed by a pointer network that selects frames to include in the summary. DeepQAMVS is trained with reinforcement learning, incorporating rewards that capture representativeness, diversity, query-adaptability and temporal coherence. We achieve state-of-the-art results on the MVS1K dataset, with inference time scaling linearly with the number of input video frames.
@article{arxiv.2105.06441,
title = {DeepQAMVS: Query-Aware Hierarchical Pointer Networks for Multi-Video Summarization},
author = {Safa Messaoud and Ismini Lourentzou and Assma Boughoula and Mona Zehni and Zhizhen Zhao and Chengxiang Zhai and Alexander G. Schwing},
journal= {arXiv preprint arXiv:2105.06441},
year = {2021}
}