This paper investigates search in model-based reinforcement learning (RL). Conventional wisdom holds that long-term predictions and compounding errors are the primary obstacles for model-based RL. We challenge this view, showing that search is not a drop-in replacement for a learned policy. Surprisingly, we find that search can harm performance even when the model is highly accurate. Instead, we show that mitigating overestimation bias matters more than improving model or value function accuracy. Building on this insight, we identify that taking the minimum over an ensemble of value functions effectively addresses this bias and enables effective search, achieving state-of-the-art performance across multiple popular benchmark domains.
@article{arxiv.2601.21306,
title = {The Surprising Difficulty of Search in Model-Based Reinforcement Learning},
author = {Wei-Di Chang and Mikael Henaff and Brandon Amos and Gregory Dudek and Scott Fujimoto},
journal= {arXiv preprint arXiv:2601.21306},
year = {2026}
}