Federated inference enhances LLM performance in edge computing through weighted averaging of distributed model predictions. However, autoregressive LLM inference requires frequent full-model forward passes across workers, severely limiting decoding throughput. Distributed deployment further aggravates this due to a communication bottleneck: each worker must transmit full token probability distributions per draft token, dominating end-to-end latency. To address these challenges, we introduce speculative decoding to enable parallel LLM processing and propose a top-K compressed transmission scheme with two server-side reconstruction strategies. We theoretically analyze the robustness of our method in terms of local reconstruction error, aggregation bias, and acceptance-rate bias, and derive corresponding bounds. Experiments demonstrate that our scheme achieves high generation fidelity while significantly reducing communication overhead.
@article{arxiv.2604.25777,
title = {SpecFed: Accelerating Federated LLM Inference with Speculative Decoding and Compressed Transmission},
author = {Ce Zheng and Xinghan Wang and Jiahong Ning and Yuxuan Shi and Ning Huang and Tingting Yang},
journal= {arXiv preprint arXiv:2604.25777},
year = {2026}
}
Comments
IEEE International Symposium on Information Theory (ISIT), 2026