In sequence generation task, many works use policy gradient for model optimization to tackle the intractable backpropagation issue when maximizing the non-differentiable evaluation metrics or fooling the discriminator in adversarial learning. In this paper, we replace policy gradient with proximal policy optimization (PPO), which is a proved more efficient reinforcement learning algorithm, and propose a dynamic approach for PPO (PPO-dynamic). We demonstrate the efficacy of PPO and PPO-dynamic on conditional sequence generation tasks including synthetic experiment and chit-chat chatbot. The results show that PPO and PPO-dynamic can beat policy gradient by stability and performance.
@article{arxiv.1808.07982,
title = {Proximal Policy Optimization and its Dynamic Version for Sequence Generation},
author = {Yi-Lin Tuan and Jinzhi Zhang and Yujia Li and Hung-yi Lee},
journal= {arXiv preprint arXiv:1808.07982},
year = {2018}
}