Modern optical flow methods are often composed of a cascade of many independent steps or formulated as a black box neural network that is hard to interpret and analyze. In this work we seek for a plain, interpretable, but learnable solution. We propose a novel inpainting based algorithm that approaches the problem in three steps: feature selection and matching, selection of supporting points and energy based inpainting. To facilitate the inference we propose an optimization layer that allows to backpropagate through 10K iterations of a first-order method without any numerical or memory problems. Compared to recent state-of-the-art networks, our modular CNN is very lightweight and competitive with other, more involved, inpainting based methods.
@article{arxiv.1811.03721,
title = {Learning Energy Based Inpainting for Optical Flow},
author = {Christoph Vogel and Patrick Knöbelreiter and Thomas Pock},
journal= {arXiv preprint arXiv:1811.03721},
year = {2018}
}