While Transformer has become the de-facto standard for speech, modeling upon the fine-grained frame-level features remains an open challenge of capturing long-distance dependencies and distributing the attention weights. We propose \textit{Progressive Down-Sampling} (PDS) which gradually compresses the acoustic features into coarser-grained units containing more complete semantic information, like text-level representation. In addition, we develop a representation fusion method to alleviate information loss that occurs inevitably during high compression. In this way, we compress the acoustic features into 1/32 of the initial length while achieving better or comparable performances on the speech recognition task. And as a bonus, it yields inference speedups ranging from 1.20× to 1.47×. By reducing the modeling burden, we also achieve competitive results when training on the more challenging speech translation task.
@article{arxiv.2305.17356,
title = {Bridging the Granularity Gap for Acoustic Modeling},
author = {Chen Xu and Yuhao Zhang and Chengbo Jiao and Xiaoqian Liu and Chi Hu and Xin Zeng and Tong Xiao and Anxiang Ma and Huizhen Wang and JingBo Zhu},
journal= {arXiv preprint arXiv:2305.17356},
year = {2023}
}