We introduce dGSLM, the first "textless" model able to generate audio samples of naturalistic spoken dialogues. It uses recent work on unsupervised spoken unit discovery coupled with a dual-tower transformer architecture with cross-attention trained on 2000 hours of two-channel raw conversational audio (Fisher dataset) without any text or labels. We show that our model is able to generate speech, laughter and other paralinguistic signals in the two channels simultaneously and reproduces more naturalistic and fluid turn-taking compared to a text-based cascaded model.
@article{arxiv.2203.16502,
title = {Generative Spoken Dialogue Language Modeling},
author = {Tu Anh Nguyen and Eugene Kharitonov and Jade Copet and Yossi Adi and Wei-Ning Hsu and Ali Elkahky and Paden Tomasello and Robin Algayres and Benoit Sagot and Abdelrahman Mohamed and Emmanuel Dupoux},
journal= {arXiv preprint arXiv:2203.16502},
year = {2022}
}