This paper extends off-policy reinforcement learning to the multi-agent case in which a set of networked agents communicating with their neighbors according to a time-varying graph collaboratively evaluates and improves a target policy while following a distinct behavior policy. To this end, the paper develops a multi-agent version of emphatic temporal difference learning for off-policy policy evaluation, and proves convergence under linear function approximation. The paper then leverages this result, in conjunction with a novel multi-agent off-policy policy gradient theorem and recent work in both multi-agent on-policy and single-agent off-policy actor-critic methods, to develop and give convergence guarantees for a new multi-agent off-policy actor-critic algorithm.
@article{arxiv.1903.06372,
title = {A Multi-Agent Off-Policy Actor-Critic Algorithm for Distributed Reinforcement Learning},
author = {Wesley Suttle and Zhuoran Yang and Kaiqing Zhang and Zhaoran Wang and Tamer Basar and Ji Liu},
journal= {arXiv preprint arXiv:1903.06372},
year = {2019}
}