This study investigates ChatGPT for Japanese-English translation, exploring simple and enhanced prompts and comparing against commercially available translation engines. Performing both automatic and MQM-based human evaluations, we found that document-level translation outperforms sentence-level translation for ChatGPT. On the other hand, we were not able to determine if enhanced prompts performed better than simple prompts in our experiments. We also discovered that ChatGPT-3.5 was preferred by automatic evaluation, but a tradeoff exists between accuracy (ChatGPT-3.5) and fluency (ChatGPT-4). Lastly, ChatGPT yields competitive results against two widely-known translation systems.
@article{arxiv.2510.08042,
title = {ChatGPT as a Translation Engine: A Case Study on Japanese-English},
author = {Vincent Michael Sutanto and Giovanni Gatti De Giacomo and Toshiaki Nakazawa and Masaru Yamada},
journal= {arXiv preprint arXiv:2510.08042},
year = {2025}
}