The Transformer architecture model, based on self-attention and multi-head attention, has achieved remarkable success in offline end-to-end Automatic Speech Recognition (ASR). However, self-attention and multi-head attention cannot be easily applied for streaming or online ASR. For self-attention in Transformer ASR, the softmax normalization function-based attention mechanism makes it impossible to highlight important speech information. For multi-head attention in Transformer ASR, it is not easy to model monotonic alignments in different heads. To overcome these two limits, we integrate sparse attention and monotonic attention into Transformer-based ASR. The sparse mechanism introduces a learned sparsity scheme to enable each self-attention structure to fit the corresponding head better. The monotonic attention deploys regularization to prune redundant heads for the multi-head attention structure. The experiments show that our method can effectively improve the attention mechanism on widely used benchmarks of speech recognition.
@article{arxiv.2209.15176,
title = {Adaptive Sparse and Monotonic Attention for Transformer-based Automatic Speech Recognition},
author = {Chendong Zhao and Jianzong Wang and Wen qi Wei and Xiaoyang Qu and Haoqian Wang and Jing Xiao},
journal= {arXiv preprint arXiv:2209.15176},
year = {2022}
}