Whisper, despite being trained on 680K hours of web-scaled audio data, faces difficulty in recognising rare words like domain-specific terms, with a solution being contextual biasing through prompting. To improve upon this method, in this paper, we propose a supervised learning strategy to fine-tune Whisper for contextual biasing instruction. We demonstrate that by using only 670 hours of Common Voice English set for fine-tuning, our model generalises to 11 diverse open-source English datasets, achieving a 45.6% improvement in recognition of rare words and 60.8% improvement in recognition of words unseen during fine-tuning over the baseline method. Surprisingly, our model's contextual biasing ability generalises even to languages unseen during fine-tuning.
@article{arxiv.2502.11572,
title = {Improving Rare-Word Recognition of Whisper in Zero-Shot Settings},
author = {Yash Jogi and Vaibhav Aggarwal and Shabari S Nair and Yash Verma and Aayush Kubba},
journal= {arXiv preprint arXiv:2502.11572},
year = {2025}
}