Language Models (LMs) are typically tuned with human preferences to produce helpful responses, but the impact of preference tuning on the ability to handle culturally diverse queries remains understudied. In this paper, we systematically analyze how native human cultural preferences can be incorporated into the preference learning process to train more culturally aware LMs. We introduce \textbf{CARE}, a multilingual resource containing 3,490 culturally specific questions and 31.7k responses with human judgments. We demonstrate how a modest amount of high-quality native preferences improves cultural awareness across various LMs, outperforming larger generic preference data. Our analyses reveal that models with stronger initial cultural performance benefit more from alignment, leading to gaps among models developed in different regions with varying access to culturally relevant data. CARE is publicly available at https://github.com/Guochry/CARE.
@article{arxiv.2504.05154,
title = {CARE: Multilingual Human Preference Learning for Cultural Awareness},
author = {Geyang Guo and Tarek Naous and Hiromi Wakaki and Yukiko Nishimura and Yuki Mitsufuji and Alan Ritter and Wei Xu},
journal= {arXiv preprint arXiv:2504.05154},
year = {2025}
}