Simultaneous speech translation (SimulST) systems must balance translation quality with response time, making latency measurement crucial for evaluating their real-world performance. However, there has been a longstanding belief that current metrics yield unrealistically high latency measurements in unsegmented streaming settings. In this paper, we investigate this phenomenon, revealing its root cause in a fundamental misconception underlying existing latency evaluation approaches. We demonstrate that this issue affects not only streaming but also segment-level latency evaluation across different metrics. Furthermore, we propose a modification to correctly measure computation-aware latency for SimulST systems, addressing the limitations present in existing metrics.
@article{arxiv.2410.16011,
title = {CA*: Addressing Evaluation Pitfalls in Computation-Aware Latency for Simultaneous Speech Translation},
author = {Xi Xu and Wenda Xu and Siqi Ouyang and Lei Li},
journal= {arXiv preprint arXiv:2410.16011},
year = {2024}
}