GRPO-style reinforcement learning (RL)-based LLM fine-tuning algorithms have recently gained popularity. Relying on heuristic trust-region approximations, however, they can lead to brittle optimization behavior, as global importance-ratio clipping and group-wise normalization fail to regulate samples whose importance ratios fall outside the clipping range. We propose Query-Adaptive Trust-Region policy Optimization (QUATRO), which directly enforces trust-region constraints through a principled optimization. This yields a clear and interpretable objective that enables explicit control over policy updates and stable, entropy-controlled optimization, with a stabilizer terms arising intrinsically from the exact trust-region formulation. Empirically verified on diverse mathematical reasoning benchmarks, QUATRO shows stable training under increased policy staleness and aggressive learning rates, maintaining well-controlled entropy throughout training.
@article{arxiv.2602.04620,
title = {QUATRO: Query-Adaptive Trust Region Policy Optimization for LLM Fine-tuning},
author = {Doyeon Lee and Eunyi Lyou and Hyunsoo Cho and Sookyung Kim and Joonseok Lee and Jaemoo Choi},
journal= {arXiv preprint arXiv:2602.04620},
year = {2026}
}