Audiovisual speech recognition (AVSR) is a method to alleviate the adverse effect of noise in the acoustic signal. Leveraging recent developments in deep neural network-based speech recognition, we present an AVSR neural network architecture which is trained end-to-end, without the need to separately model the process of decision fusion as in conventional (e.g. HMM-based) systems. The fusion system outperforms single-modality recognition under all noise conditions. Investigation of the saliency of the input features shows that the neural network automatically adapts to different noise levels in the acoustic signal.
@article{arxiv.1804.11127,
title = {Investigations on End-to-End Audiovisual Fusion},
author = {Michael Wand and Ngoc Thang Vu and Juergen Schmidhuber},
journal= {arXiv preprint arXiv:1804.11127},
year = {2018}
}