Reinforcement Learning (RL) is pivotal for enhancing Large Language Model (LLM) reasoning, yet mainstream algorithms such as GRPO and DAPO remain constrained by a coarse-grained credit assignment paradigm, where all tokens within the same response receive the identical reward. In this paper, we propose Dynamic Entropy Weighting, systematically define entropy-based weight ratios ∑k=1nHk,tHi,t and similar variants to redistribute rewards and get fine-grained rewards through two new algorithms: Group Token Policy Optimization (GTPO), which assigns an entropy-weighted reward to each token and synthesizes token-specific advantage function to drive the model toward optimal path, and the analogous algorithm Sequence-Level GRPO (GRPO-S), which extends this design to the sequence level and exhibits superior stability in long Chain-of-Thought (CoT) reasoning tasks.
@article{arxiv.2508.04349,
title = {GTPO and GRPO-S: Token and Sequence-Level Reward Shaping with Policy Entropy},
author = {Hongze Tan and Zihan Wang and Jianfei Pan and Jinghao Lin and Hao Wang and Yifan Wu and Tao Chen and Zhihang Zheng and Zhihao Tang and Haihua Yang},
journal= {arXiv preprint arXiv:2508.04349},
year = {2026}
}