Generative recommendation (GR) offers superior modeling capabilities but suffers from prohibitive inference costs due to the repeated encoding of long user histories. While cross-request Key-Value (KV) cache reuse presents a significant optimization opportunity, the massive scale of individual user states creates a storage explosion that far exceeds physical GPU limits. We propose MTServe, a hierarchical cache management system that virtualizes GPU memory by leveraging host RAM as a scalable backup store. To bridge the I/O gap between tiers, MTServe introduces a suite of system-level optimizations, including a hybrid storage layout, an asynchronous data transfer pipeline, and a locality-driven replacement policy. On both public and production datasets, MTServe delivers up to 3.1* speedup while maintaining near-perfect hit ratios (>98.5%).
@article{arxiv.2604.22881,
title = {MTServe: Efficient Serving for Generative Recommendation Models with Hierarchical Caches},
author = {Xin Wang and Chi Ma and Shaobin Chen and Pu Wang and Menglei Zhou and Junyi Qiu and Qiaorui Chen and Jiayu Sun and Shijie Liu and Zehuan Wang and Lei Yu and Chuan Liu and Fei Jiang and Wei Lin and Hao Wang and Jiawei Jiang and Xiao Yan},
journal= {arXiv preprint arXiv:2604.22881},
year = {2026}
}