This paper investigates the reference-less evaluation of machine translation for low-resource language pairs, known as quality estimation (QE). Segment-level QE is a challenging cross-lingual language understanding task that provides a quality score (0-100) to the translated output. We comprehensively evaluate large language models (LLMs) in zero/few-shot scenarios and perform instruction fine-tuning using a novel prompt based on annotation guidelines. Our results indicate that prompt-based approaches are outperformed by the encoder-based fine-tuned QE models. Our error analysis reveals tokenization issues, along with errors due to transliteration and named entities, and argues for refinement in LLM pre-training for cross-lingual tasks. We release the data, and models trained publicly for further research.
@article{arxiv.2501.04473,
title = {When LLMs Struggle: Reference-less Translation Evaluation for Low-resource Languages},
author = {Archchana Sindhujan and Diptesh Kanojia and Constantin Orasan and Shenbin Qian},
journal= {arXiv preprint arXiv:2501.04473},
year = {2025}
}