Recent work in hierarchical reinforcement learning has shown success in scaling to billions of timesteps when learning over a set of predefined option reward functions. We show that, instead of using a single reward function per option, the reward functions can be effectively used to induce a space of behaviours, by letting the controller specify linear combinations over reward functions, allowing a more expressive set of policies to be represented. We call this method Hierarchical Behaviour Spaces (HBS). We evaluate HBS on the NetHack Learning Environment, demonstrating strong performance. We conduct a series of experiments and determine that, perhaps going against conventional wisdom, the benefits of hierarchy in our method come from increased exploration rather than long term reasoning.
@article{arxiv.2604.24558,
title = {Hierarchical Behaviour Spaces},
author = {Michael Tryfan Matthews and Anssi Kanervisto and Jakob Foerster and Pierluca D'Oro and Scott Fujimoto and Mikael Henaff},
journal= {arXiv preprint arXiv:2604.24558},
year = {2026}
}