The key-value (KV) cache in the tensor version of transformers presents a significant bottleneck during inference. While previous work analyzes the fundamental space complexity barriers in standard attention mechanisms [Haris and Onak, 2025], our work generalizes the space complexity barriers result to tensor attention version. Our theoretical contributions rely on a reduction from communication complexity and deduce the memory lower bound for tensor-structured attention mechanisms when d=Ω(logn). Furthermore, we introduce two types of tensor attention cache and present a trade-off between time and memory for two scenarios. Overall, our work provides a theoretical foundation for us to understand the time-memory tradeoff of KV-Cache compression in tensor attention decoding and offers more perspectives in developing more memory-efficient tensor attention Transformer architectures.
@article{arxiv.2503.11108,
title = {Time and Memory Trade-off of KV-Cache Compression in Tensor Transformer Decoding},
author = {Yifang Chen and Xiaoyu Li and Yingyu Liang and Zhenmei Shi and Zhao Song and Yu Tian},
journal= {arXiv preprint arXiv:2503.11108},
year = {2025}
}