Multimodal ML models can process data in multiple modalities (e.g., video, images, audio, text) and are useful for video content analysis in a variety of problems (e.g., object detection, scene understanding). In this paper, we focus on the problem of video categorization by using a multimodal approach. We have developed a novel temporal coherence-based regularization approach, which applies to different types of models (e.g., RNN, NetVLAD, Transformer). We demonstrate through experiments how our proposed multimodal video categorization models with temporal coherence out-perform strong state-of-the-art baseline models.
@article{arxiv.2002.03844,
title = {Exploiting Temporal Coherence for Multi-modal Video Categorization},
author = {Palash Goyal and Saurabh Sahu and Shalini Ghosh and Chul Lee},
journal= {arXiv preprint arXiv:2002.03844},
year = {2020}
}