Despite Large Language Models (LLMs) demonstrating superior translation performance and long-context capabilities, evaluation methodologies remain constrained to sentence-level assessment due to dataset limitations, token number restrictions in metrics, and rigid sentence boundary requirements. We introduce SEGALE, an evaluation scheme that extends existing automatic metrics to long-document translation by treating documents as continuous text and applying sentence segmentation and alignment methods. Our approach enables previously unattainable document-level evaluation, handling translations of arbitrary length generated with document-level prompts while accounting for under-/over-translations and varied sentence boundaries. Experiments show our scheme significantly outperforms existing long-form document evaluation schemes, while being comparable to evaluations performed with groundtruth sentence alignments. Additionally, we apply our scheme to book-length texts and newly demonstrate that many open-weight LLMs fail to effectively translate documents at their reported maximum context lengths.
@article{arxiv.2509.17249,
title = {Extending Automatic Machine Translation Evaluation to Book-Length Documents},
author = {Kuang-Da Wang and Shuoyang Ding and Chao-Han Huck Yang and Ping-Chun Hsieh and Wen-Chih Peng and Vitaly Lavrukhin and Boris Ginsburg},
journal= {arXiv preprint arXiv:2509.17249},
year = {2025}
}