Neural metrics trained on human evaluations of MT tend to correlate well with human judgments, but their behavior is not fully understood. In this paper, we perform a controlled experiment and compare a baseline metric that has not been trained on human evaluations (Prism) to a trained version of the same metric (Prism+FT). Surprisingly, we find that Prism+FT becomes more robust to machine-translated references, which are a notorious problem in MT evaluation. This suggests that the effects of metric training go beyond the intended effect of improving overall correlation with human judgments.
@article{arxiv.2312.00536,
title = {Trained MT Metrics Learn to Cope with Machine-translated References},
author = {Jannis Vamvas and Tobias Domhan and Sony Trenous and Rico Sennrich and Eva Hasler},
journal= {arXiv preprint arXiv:2312.00536},
year = {2023}
}