One of the main challenges in reinforcement learning (RL) is generalisation. In typical deep RL methods this is achieved by approximating the optimal value function with a low-dimensional representation using a deep network. While this approach works well in many domains, in domains where the optimal value function cannot easily be reduced to a low-dimensional representation, learning can be very slow and unstable. This paper contributes towards tackling such challenging domains, by proposing a new method, called Hybrid Reward Architecture (HRA). HRA takes as input a decomposed reward function and learns a separate value function for each component reward function. Because each component typically only depends on a subset of all features, the corresponding value function can be approximated more easily by a low-dimensional representation, enabling more effective learning. We demonstrate HRA on a toy-problem and the Atari game Ms. Pac-Man, where HRA achieves above-human performance.
@article{arxiv.1706.04208,
title = {Hybrid Reward Architecture for Reinforcement Learning},
author = {Harm van Seijen and Mehdi Fatemi and Joshua Romoff and Romain Laroche and Tavian Barnes and Jeffrey Tsang},
journal= {arXiv preprint arXiv:1706.04208},
year = {2017}
}