Speculative Decoding (SD) is a popular lossless technique for accelerating the inference of Large Language Models (LLMs). We show that the decoding speed of SD frameworks with static draft structures can be significantly improved by incorporating context-aware adaptive draft structures. However, current studies on adaptive draft structures are limited by their performance, modeling approaches, and applicability. In this paper, we introduce AdaEAGLE, the first SD framework that explicitly models adaptive draft structures. AdaEAGLE leverages the Lightweight Draft Length Predictor (LDLP) module to explicitly predict the optimal number of draft tokens during inference to guide the draft model. It achieves comparable speedup results without manual thresholds and allows for deeper, more specialized optimizations. Moreover, together with threshold-based strategies, AdaEAGLE achieves a 1.62× speedup over the vanilla AR decoding and outperforms fixed-length SotA baseline while maintaining output quality.
@article{arxiv.2412.18910,
title = {AdaEAGLE: Optimizing Speculative Decoding via Explicit Modeling of Adaptive Draft Structures},
author = {Situo Zhang and Hankun Wang and Da Ma and Zichen Zhu and Lu Chen and Kunyao Lan and Kai Yu},
journal= {arXiv preprint arXiv:2412.18910},
year = {2024}
}