This work aims to improve the efficiency of vision transformers (ViT). While ViTs use computationally expensive self-attention operations in every layer, we identify that these operations are highly correlated across layers -- a key redundancy that causes unnecessary computations. Based on this observation, we propose SkipAt, a method to reuse self-attention computation from preceding layers to approximate attention at one or more subsequent layers. To ensure that reusing self-attention blocks across layers does not degrade the performance, we introduce a simple parametric function, which outperforms the baseline transformer's performance while running computationally faster. We show the effectiveness of our method in image classification and self-supervised learning on ImageNet-1K, semantic segmentation on ADE20K, image denoising on SIDD, and video denoising on DAVIS. We achieve improved throughput at the same-or-higher accuracy levels in all these tasks.
@article{arxiv.2301.02240,
title = {Skip-Attention: Improving Vision Transformers by Paying Less Attention},
author = {Shashanka Venkataramanan and Amir Ghodrati and Yuki M. Asano and Fatih Porikli and Amirhossein Habibian},
journal= {arXiv preprint arXiv:2301.02240},
year = {2023}
}