Reinforcement learning with offline data suffers from Q-value extrapolation errors. To address this issue, we first demonstrate that linear extrapolation of the Q-function beyond the data range is particularly problematic. To mitigate this, we propose guiding the gradual decrease of Q-values outside the data range, which is achieved through reward scaling with layer normalization (RS-LN) and a penalization mechanism for infeasible actions (PA). By combining RS-LN and PA, we develop a new algorithm called PARS. We evaluate PARS across a range of tasks, demonstrating superior performance compared to state-of-the-art algorithms in both offline training and online fine-tuning on the D4RL benchmark, with notable success in the challenging AntMaze Ultra task.
@article{arxiv.2507.08761,
title = {Penalizing Infeasible Actions and Reward Scaling in Reinforcement Learning with Offline Data},
author = {Jeonghye Kim and Yongjae Shin and Whiyoung Jung and Sunghoon Hong and Deunsol Yoon and Youngchul Sung and Kanghoon Lee and Woohyung Lim},
journal= {arXiv preprint arXiv:2507.08761},
year = {2025}
}