We present a comprehensive study on meaningfully evaluating sign language utterances in the form of human skeletal poses. The study covers keypoint distance-based, embedding-based, and back-translation-based metrics. We show tradeoffs between different metrics in different scenarios through automatic meta-evaluation of sign-level retrieval and a human correlation study of text-to-pose translation across different sign languages. Our findings and the open-source pose-evaluation toolkit provide a practical and reproducible way of developing and evaluating sign language translation or generation systems.
@article{arxiv.2510.07453,
title = {Meaningful Pose-Based Sign Language Evaluation},
author = {Zifan Jiang and Colin Leong and Amit Moryossef and Anne Göhring and Annette Rios and Oliver Cory and Maksym Ivashechkin and Neha Tarigopula and Biao Zhang and Rico Sennrich and Sarah Ebling},
journal= {arXiv preprint arXiv:2510.07453},
year = {2025}
}