In hierarchical reinforcement learning a major challenge is determining appropriate low-level policies. We propose an unsupervised learning scheme, based on asymmetric self-play from Sukhbaatar et al. (2018), that automatically learns a good representation of sub-goals in the environment and a low-level policy that can execute them. A high-level policy can then direct the lower one by generating a sequence of continuous sub-goal vectors. We evaluate our model using Mazebase and Mujoco environments, including the challenging AntGather task. Visualizations of the sub-goal embeddings reveal a logical decomposition of tasks within the environment. Quantitatively, our approach obtains compelling performance gains over non-hierarchical approaches.
@article{arxiv.1811.09083,
title = {Learning Goal Embeddings via Self-Play for Hierarchical Reinforcement Learning},
author = {Sainbayar Sukhbaatar and Emily Denton and Arthur Szlam and Rob Fergus},
journal= {arXiv preprint arXiv:1811.09083},
year = {2018}
}