Embedding-based retrieval models have made significant strides in retrieval-augmented generation (RAG) techniques for text and multimodal large language models (LLMs) applications. However, when it comes to speech larage language models (SLLMs), these methods are limited to a two-stage process, where automatic speech recognition (ASR) is combined with text-based retrieval. This sequential architecture suffers from high latency and error propagation. To address these limitations, we propose a unified embedding framework that eliminates the need for intermediate text representations. Specifically, the framework includes separate speech and text encoders, followed by a shared scaling layer that maps both modalities into a common embedding space. Our model reduces pipeline latency by 50\% while achieving higher retrieval accuracy compared to traditional two-stage methods. We also provide a theoretical analysis of the challenges inherent in end-to-end speech retrieval and introduce architectural principles for effective speech-to-document matching. Extensive experiments demonstrate the robustness of our approach across diverse acoustic conditions and speaker variations, paving the way for a new paradigm in multimodal SLLMs retrieval systems.
@article{arxiv.2502.02603,
title = {SEAL: Speech Embedding Alignment Learning for Speech Large Language Model with Retrieval-Augmented Generation},
author = {Chunyu Sun and Bingyu Liu and Zhichao Cui and Junhan Shi and Anbin Qi and Tian-hao Zhang and Dinghao Zhou and Lewei Lu},
journal= {arXiv preprint arXiv:2502.02603},
year = {2025}
}