Advances in Deep Learning have made possible reliable landmark tracking of human bodies and faces that can be used for a variety of tasks. We test a recent Computer Vision solution, MediaPipe Holistic (MPH), to find out if its tracking of the facial features is reliable enough for a linguistic analysis of data from sign languages, and compare it to an older solution (OpenFace, OF). We use an existing data set of sentences in Kazakh-Russian Sign Language and a newly created small data set of videos with head tilts and eyebrow movements. We find that MPH does not perform well enough for linguistic analysis of eyebrow movement - but in a different way from OF, which is also performing poorly without correction. We reiterate a previous proposal to train additional correction models to overcome these limitations.
@article{arxiv.2403.10367,
title = {Testing MediaPipe Holistic for Linguistic Analysis of Nonmanual Markers in Sign Languages},
author = {Anna Kuznetsova and Vadim Kimmelman},
journal= {arXiv preprint arXiv:2403.10367},
year = {2024}
}