Large Audio Language Models (LALMs) have emerged as powerful tools for speech-related tasks but remain underexplored for fine-tuning, especially with limited speech data. To bridge this gap, we systematically examine how different fine-tuning schemes including text-only, direct mixing, and curriculum learning affect spoken language understanding (SLU), focusing on scenarios where text-label pairs are abundant while paired speech-label data are limited. Results show that LALMs already achieve competitive performance with text-only fine-tuning, highlighting their strong generalization ability. Adding even small amounts of speech data (2-5%) yields substantial further gains, with curriculum learning particularly effective under scarce data. In cross-lingual SLU, combining source-language speech data with target-language text and minimal target-language speech data enables effective adaptation. Overall, this study provides practical insights into the LALM fine-tuning under realistic data constraints.
@article{arxiv.2509.15389,
title = {Exploring Fine-Tuning of Large Audio Language Models for Spoken Language Understanding under Limited Speech Data},
author = {Youngwon Choi and Jaeyoon Jung and Hyeonyu Kim and Huu-Kim Nguyen and Hwayeon Kim},
journal= {arXiv preprint arXiv:2509.15389},
year = {2026}
}