In this paper, we study the problem of weakly-supervised temporal grounding of sentence in video. Specifically, given an untrimmed video and a query sentence, our goal is to localize a temporal segment in the video that semantically corresponds to the query sentence, with no reliance on any temporal annotation during training. We propose a two-stage model to tackle this problem in a coarse-to-fine manner. In the coarse stage, we first generate a set of fixed-length temporal proposals using multi-scale sliding windows, and match their visual features against the sentence features to identify the best-matched proposal as a coarse grounding result. In the fine stage, we perform a fine-grained matching between the visual features of the frames in the best-matched proposal and the sentence features to locate the precise frame boundary of the fine grounding result. Comprehensive experiments on the ActivityNet Captions dataset and the Charades-STA dataset demonstrate that our two-stage model achieves compelling performance.
@article{arxiv.2001.09308,
title = {Look Closer to Ground Better: Weakly-Supervised Temporal Grounding of Sentence in Video},
author = {Zhenfang Chen and Lin Ma and Wenhan Luo and Peng Tang and Kwan-Yee K. Wong},
journal= {arXiv preprint arXiv:2001.09308},
year = {2020}
}