Recently, speech-text pre-training methods have shown remarkable success in many speech and natural language processing tasks. However, most previous pre-trained models are usually tailored for one or two specific tasks, but fail to conquer a wide range of speech-text tasks. In addition, existing speech-text pre-training methods fail to explore the contextual information within a dialogue to enrich utterance representations. In this paper, we propose Speech-text dialog Pre-training for spoken dialog understanding with ExpliCiT cRoss-Modal Alignment (SPECTRA), which is the first-ever speech-text dialog pre-training model. Concretely, to consider the temporality of speech modality, we design a novel temporal position prediction task to capture the speech-text alignment. This pre-training task aims to predict the start and end time of each textual word in the corresponding speech waveform. In addition, to learn the characteristics of spoken dialogs, we generalize a response selection task from textual dialog pre-training to speech-text dialog pre-training scenarios. Experimental results on four different downstream speech-text tasks demonstrate the superiority of SPECTRA in learning speech-text alignment and multi-turn dialog context.
@article{arxiv.2305.11579,
title = {Speech-Text Dialog Pre-training for Spoken Dialog Understanding with Explicit Cross-Modal Alignment},
author = {Tianshu Yu and Haoyu Gao and Ting-En Lin and Min Yang and Yuchuan Wu and Wentao Ma and Chao Wang and Fei Huang and Yongbin Li},
journal= {arXiv preprint arXiv:2305.11579},
year = {2023}
}