This paper establishes that an MDP with a unique optimal policy and ergodic associated transition matrix ensures the convergence of various versions of the Value Iteration algorithm at a geometric rate that exceeds the discount factor {\gamma} for both discounted and average-reward criteria.
@article{arxiv.2406.09592,
title = {On Value Iteration Convergence in Connected MDPs},
author = {Arsenii Mustafin and Alex Olshevsky and Ioannis Ch. Paschalidis},
journal= {arXiv preprint arXiv:2406.09592},
year = {2024}
}