Hindsight experience replay (HER) accelerates off-policy reinforcement learning algorithms for environments that emit sparse rewards by modifying the goal of the episode post-hoc to be some state achieved during the episode. Because post-hoc modification of the observed goal violates the assumptions of on-policy algorithms, HER is not typically applied to on-policy algorithms. Here, we show that HER can dramatically accelerate proximal policy optimization (PPO), an on-policy reinforcement learning algorithm, when tested on a custom predator-prey environment.
@article{arxiv.2410.22524,
title = {Hindsight Experience Replay Accelerates Proximal Policy Optimization},
author = {Douglas C. Crowder and Darrien M. McKenzie and Matthew L. Trappett and Frances S. Chance},
journal= {arXiv preprint arXiv:2410.22524},
year = {2024}
}