We investigate the effectiveness of large language models (LLMs) for text diacritization in two typologically distinct languages: Arabic and Yoruba. To enable a rigorous evaluation, we introduce a novel multilingual dataset MultiDiac, with diverse samples that capture a range of diacritic ambiguities. We evaluate 12 LLMs varying in size, accessibility, and language coverage, and benchmark them against 4 specialized diacritization models. Additionally, we fine-tune four small open-source models using LoRA for Yoruba. Our results show that many off-the-shelf LLMs outperform specialized diacritization models, but smaller models suffer from hallucinations. We find that fine-tuning on a small dataset can help improve diacritization performance and reduce hallucinations for Yoruba.
@article{arxiv.2506.11602,
title = {Are LLMs Good Text Diacritizers? An Arabic and Yoruba Case Study},
author = {Hawau Olamide Toyin and Samar Mohamed Magdy and Hanan Aldarmaki},
journal= {arXiv preprint arXiv:2506.11602},
year = {2026}
}