We propose a probabilistic video model, the Video Pixel Network (VPN), that estimates the discrete joint distribution of the raw pixel values in a video. The model and the neural architecture reflect the time, space and color structure of video tensors and encode it as a four-dimensional dependency chain. The VPN approaches the best possible performance on the Moving MNIST benchmark, a leap over the previous state of the art, and the generated videos show only minor deviations from the ground truth. The VPN also produces detailed samples on the action-conditional Robotic Pushing benchmark and generalizes to the motion of novel objects.
@article{arxiv.1610.00527,
title = {Video Pixel Networks},
author = {Nal Kalchbrenner and Aaron van den Oord and Karen Simonyan and Ivo Danihelka and Oriol Vinyals and Alex Graves and Koray Kavukcuoglu},
journal= {arXiv preprint arXiv:1610.00527},
year = {2016}
}