Speech-driven facial animation requires accurate correspondence between acoustic signals and facial motion, especially for articulation-related mouth movements. However, directly mapping speech audio to facial coefficients often overlooks the linguistic and phonetic structure underlying speech production. In this paper, we propose AudioFace, a language-assisted framework for speech-driven blendshape generation that treats mouth-related facial coefficient prediction as a structured generation problem guided by linguistic and articulatory information. Instead of relying solely on acoustic features, our method leverages the prior knowledge of multimodal large language models and introduces transcript- and phoneme-level cues to bridge speech signals with interpretable facial actions. Extensive experiments show that AudioFace achieves superior performance across multiple evaluation metrics, validating the effectiveness of language-assisted and multimodal-prior-guided speech-driven facial animation.
@article{arxiv.2605.07478,
title = {AudioFace: Language-Assisted Speech-Driven Facial Animation with Multimodal Language Models},
author = {Kai Zheng and Zejian Kang and Rui Mao and Hongyuan Zou and Yuanchen Fei and Xuanyang Xu and Xiangru Huang},
journal= {arXiv preprint arXiv:2605.07478},
year = {2026}
}