Recent advances in reinforcement learning (RL) algorithms aim to enhance the performance of language models at scale. Yet, there is a noticeable absence of a cost-effective and standardized testbed tailored to evaluating and comparing these algorithms. To bridge this gap, we present a generalized version of the 24-Puzzle: the (N,K)-Puzzle, which challenges language models to reach a target value K with N integers. We evaluate the effectiveness of established RL algorithms such as Proximal Policy Optimization (PPO), alongside novel approaches like Identity Policy Optimization (IPO) and Direct Policy Optimization (DPO).
@article{arxiv.2403.07191,
title = {$\mathbf{(N,K)}$-Puzzle: A Cost-Efficient Testbed for Benchmarking Reinforcement Learning Algorithms in Generative Language Model},
author = {Yufeng Zhang and Liyu Chen and Boyi Liu and Yingxiang Yang and Qiwen Cui and Yunzhe Tao and Hongxia Yang},
journal= {arXiv preprint arXiv:2403.07191},
year = {2024}
}