Multilingual Large Language Models (LLMs) often provide suboptimal performance on low-resource languages like Urdu. This paper introduces UrduLLaMA 1.0, a model derived from the open-source Llama-3.1-8B-Instruct architecture and continually pre-trained on 128 million Urdu tokens, capturing the rich diversity of the language. To enhance instruction-following and translation capabilities, we leverage Low-Rank Adaptation (LoRA) to fine tune the model on 41,000 Urdu instructions and approximately 50,000 English-Urdu translation pairs. Evaluation across three machine translation datasets demonstrates significant performance improvements compared to state-of-the-art (SOTA) models, establishing a new benchmark for Urdu LLMs. These findings underscore the potential of targeted adaptation strategies with limited data and computational resources to address the unique challenges of low-resource languages.
@article{arxiv.2502.16961,
title = {UrduLLaMA 1.0: Dataset Curation, Preprocessing, and Evaluation in Low-Resource Settings},
author = {Layba Fiaz and Munief Hassan Tahir and Sana Shams and Sarmad Hussain},
journal= {arXiv preprint arXiv:2502.16961},
year = {2025}
}