Recognition of personalized content remains a challenge in end-to-end speech recognition. We explore three novel approaches that use personalized content in a neural rescoring step to improve recognition: gazetteers, prompting, and a cross-attention based encoder-decoder model. We use internal de-identified en-US data from interactions with a virtual voice assistant supplemented with personalized named entities to compare these approaches. On a test set with personalized named entities, we show that each of these approaches improves word error rate by over 10%, against a neural rescoring baseline. We also show that on this test set, natural language prompts can improve word error rate by 7% without any training and with a marginal loss in generalization. Overall, gazetteers were found to perform the best with a 10% improvement in word error rate (WER), while also improving WER on a general test set by 1%.
@article{arxiv.2307.06832,
title = {Personalization for BERT-based Discriminative Speech Recognition Rescoring},
author = {Jari Kolehmainen and Yile Gu and Aditya Gourav and Prashanth Gurunath Shivakumar and Ankur Gandhe and Ariya Rastrow and Ivan Bulyko},
journal= {arXiv preprint arXiv:2307.06832},
year = {2023}
}