Code-switching describes the practice of using more than one language in the same sentence. In this study, we investigate how to optimize a neural transducer based bilingual automatic speech recognition (ASR) model for code-switching speech. Focusing on the scenario where the ASR model is trained without supervised code-switching data, we found that semi-supervised training and synthetic code-switched data can improve the bilingual ASR system on code-switching speech. We analyze how each of the neural transducer's encoders contributes towards code-switching performance by measuring encoder-specific recall values, and evaluate our English/Mandarin system on the ASCEND data set. Our final system achieves 25% mixed error rate (MER) on the ASCEND English/Mandarin code-switching test set -- reducing the MER by 2.1% absolute compared to the previous literature -- while maintaining good accuracy on the monolingual test sets.
@article{arxiv.2210.12214,
title = {Optimizing Bilingual Neural Transducer with Synthetic Code-switching Text Generation},
author = {Thien Nguyen and Nathalie Tran and Liuhui Deng and Thiago Fraga da Silva and Matthew Radzihovsky and Roger Hsiao and Henry Mason and Stefan Braun and Erik McDermott and Dogan Can and Pawel Swietojanski and Lyan Verwimp and Sibel Oyman and Tresi Arvizo and Honza Silovsky and Arnab Ghoshal and Mathieu Martel and Bharat Ram Ambati and Mohamed Ali},
journal= {arXiv preprint arXiv:2210.12214},
year = {2022}
}