Deploying Reinforcement Learning (RL) agents to solve real-world applications often requires satisfying complex system constraints. Often the constraint thresholds are incorrectly set due to the complex nature of a system or the inability to verify the thresholds offline (e.g, no simulator or reasonable offline evaluation procedure exists). This results in solutions where a task cannot be solved without violating the constraints. However, in many real-world cases, constraint violations are undesirable yet they are not catastrophic, motivating the need for soft-constrained RL approaches. We present a soft-constrained RL approach that utilizes meta-gradients to find a good trade-off between expected return and minimizing constraint violations. We demonstrate the effectiveness of this approach by showing that it consistently outperforms the baselines across four different MuJoCo domains.
@article{arxiv.2010.06324,
title = {Balancing Constraints and Rewards with Meta-Gradient D4PG},
author = {Dan A. Calian and Daniel J. Mankowitz and Tom Zahavy and Zhongwen Xu and Junhyuk Oh and Nir Levine and Timothy Mann},
journal= {arXiv preprint arXiv:2010.06324},
year = {2020}
}