Understanding why Transformers perform so well remains challenging due to their non-convex optimization landscape. In this work, we analyze a shallow Transformer with m independent heads trained by projected gradient descent in the kernel regime. Our analysis reveals two main findings: (i) the width required for nonasymptotic guarantees scales only logarithmically with the sample size n, and (ii) the optimization error is independent of the sequence length T. This contrasts sharply with recurrent architectures, where the optimization error can grow exponentially with T. The trade-off is memory: to keep the full context, the Transformer's memory requirement grows with the sequence length. We validate our theoretical results numerically in a teacher-student setting and compare Transformers with recurrent architectures on an autoregressive task.
@article{arxiv.2601.16514,
title = {Finite-Time Analysis of Gradient Descent for Shallow Transformers},
author = {Enes Arda and Semih Cayci and Atilla Eryilmaz},
journal= {arXiv preprint arXiv:2601.16514},
year = {2026}
}