Although the advancements of pre-trained Large Language Models have significantly accelerated recent progress in NLP, their ever-increasing size poses significant challenges for conventional fine-tuning, especially in memory-intensive tasks. We investigate the potential of Parameter-Efficient Fine-Tuning, focusing on Low-Rank Adaptation (LoRA), in the domain of multilingual summarization, a task that is both challenging (due to typically long inputs), and relatively unexplored. We conduct an extensive study across different data availability scenarios, including high- and low-data settings, and cross-lingual transfer, leveraging models of different sizes. Our findings reveal that LoRA is competitive with full fine-tuning when trained with high quantities of data, and excels in low-data scenarios and cross-lingual transfer. We also study different strategies for few-shot cross-lingual transfer, finding that continued LoRA tuning outperforms full fine-tuning and the dynamic composition of language-specific LoRA modules.
@article{arxiv.2311.08572,
title = {Low-Rank Adaptation for Multilingual Summarization: An Empirical Study},
author = {Chenxi Whitehouse and Fantine Huot and Jasmijn Bastings and Mostafa Dehghani and Chu-Cheng Lin and Mirella Lapata},
journal= {arXiv preprint arXiv:2311.08572},
year = {2024}
}