Recent advances in speech language models (LLMs) have extended textual LLMs to the speech domain, but balancing speech understanding and generation remains challenging, especially with codec-based representations. We propose a continual pre-training (CPT) framework that adapts a textual LLM to handle codec-discretized speech, mitigating modality mismatch and preserving linguistic reasoning. Our unified model supports both understanding and generation, achieving strong results across ASR, TTS, S2T-Trans, and S2S-Trans. Notably, we present the first end-to-end, single-pass S2S-Trans system using only neural codec tokens, without intermediate transcriptions, translations, or semantic tokens. CPT proves essential for cross-modal alignment and task generalization, making it a powerful tool for building robust, unified speech LLMs.
@article{arxiv.2502.16897,
title = {Balancing Speech Understanding and Generation Using Continual Pre-training for Codec-based Speech LLM},
author = {Jiatong Shi and Chunlei Zhang and Jinchuan Tian and Junrui Ni and Hao Zhang and Shinji Watanabe and Dong Yu},
journal= {arXiv preprint arXiv:2502.16897},
year = {2025}
}