Transformer-based acoustic modeling has achieved great suc-cess for both hybrid and sequence-to-sequence speech recogni-tion. However, it requires access to the full sequence, and thecomputational cost grows quadratically with respect to the in-put sequence length. These factors limit its adoption for stream-ing applications. In this work, we proposed a novel augmentedmemory self-attention, which attends on a short segment of theinput sequence and a bank of memories. The memory bankstores the embedding information for all the processed seg-ments. On the librispeech benchmark, our proposed methodoutperforms all the existing streamable transformer methods bya large margin and achieved over 15% relative error reduction,compared with the widely used LC-BLSTM baseline. Our find-ings are also confirmed on some large internal datasets.
@article{arxiv.2005.08042,
title = {Streaming Transformer-based Acoustic Models Using Self-attention with Augmented Memory},
author = {Chunyang Wu and Yongqiang Wang and Yangyang Shi and Ching-Feng Yeh and Frank Zhang},
journal= {arXiv preprint arXiv:2005.08042},
year = {2020}
}