In this work, we address two coupled tasks of gaze prediction and action recognition in egocentric videos by exploring their mutual context. Our assumption is that in the procedure of performing a manipulation task, what a person is doing determines where the person is looking at, and the gaze point reveals gaze and non-gaze regions which contain important and complementary information about the undergoing action. We propose a novel mutual context network (MCN) that jointly learns action-dependent gaze prediction and gaze-guided action recognition in an end-to-end manner. Experiments on public egocentric video datasets demonstrate that our MCN achieves state-of-the-art performance of both gaze prediction and action recognition.
@article{arxiv.1901.01874,
title = {Mutual Context Network for Jointly Estimating Egocentric Gaze and Actions},
author = {Yifei Huang and Zhenqiang Li and Minjie Cai and Yoichi Sato},
journal= {arXiv preprint arXiv:1901.01874},
year = {2023}
}