Reinforcement Learning with Verifiable Rewards (RLVR) is a central paradigm for turning large language models (LLMs) into reliable problem solvers, especially in logic-heavy domains. Despite its empirical success, it remains unclear whether RLVR elicits novel capabilities or merely sharpens the distribution over existing knowledge. We study this by formalizing over-sharpening, a phenomenon where the policy collapses onto limited modes, suppressing valid alternatives. At a high level, we discover finite-batch updates intrinsically bias learning toward sampled modes, triggering a collapse that propagates globally via semantic coupling. To mitigate this, we propose inverse-success advantage calibration to prioritize difficult queries and distribution-level calibration to diversify sampling via a memory network. Empirical evaluations validate that our strategies can effectively improve generalization.
@article{arxiv.2601.15609,
title = {When Sharpening Becomes Collapse: Sampling Bias and Semantic Coupling in RL with Verifiable Rewards},
author = {Mingyuan Fan and Weiguang Han and Daixin Wang and Cen Chen and Zhiqiang Zhang and Jun Zhou},
journal= {arXiv preprint arXiv:2601.15609},
year = {2026}
}