To deploy LLMs on resource-contained platforms such as mobile robots and smartphones, non-transformers LLMs have achieved major breakthroughs. Recently, a novel RNN-based LLM family, Repentance Weighted Key Value (RWKV) has shown strong computational efficiency; nevertheless, RWKV models still have high parameter counts which limited their deployment. In this paper, we propose a suite of compression techniques, ranging from model architecture optimizations to post-training compression, tailored to the RWKV architecture. Combined, our techniques reduce the memory footprint of RWKV models by 3.4x -- 5x with only negligible degradation in accuracy; compared to transformer LLMs with similar accuracy, our models require 4x less memory footprint.
@article{arxiv.2412.10856,
title = {RWKV-edge: Deeply Compressed RWKV for Resource-Constrained Devices},
author = {Wonkyo Choe and Yangfeng Ji and Felix Xiaozhu Lin},
journal= {arXiv preprint arXiv:2412.10856},
year = {2025}
}