We present a novel approach to generating photo-realistic images of a face with accurate lip sync, given an audio input. By using a recurrent neural network, we achieved mouth landmarks based on audio features. We exploited the power of conditional generative adversarial networks to produce highly-realistic face conditioned on a set of landmarks. These two networks together are capable of producing a sequence of natural faces in sync with an input audio track.
@article{arxiv.1803.07461,
title = {Speech-Driven Facial Reenactment Using Conditional Generative Adversarial Networks},
author = {Seyed Ali Jalalifar and Hosein Hasani and Hamid Aghajan},
journal= {arXiv preprint arXiv:1803.07461},
year = {2018}
}