Comparison-based preference learning has become central to the alignment of AI models with human preferences. However, these methods may behave counterintuitively. After empirically observing that, when accounting for a preference for response y over z, the model may actually decrease the probability (and reward) of generating y (an observation also made by others), this paper investigates the root causes of (non) monotonicity, for a general comparison-based preference learning framework that subsumes Direct Preference Optimization (DPO), Generalized Preference Optimization (GPO) and Generalized Bradley-Terry (GBT). Under mild assumptions, we prove that such methods still satisfy what we call local pairwise monotonicity. We also provide a bouquet of formalizations of monotonicity, and identify sufficient conditions for their guarantee, thereby providing a toolbox to evaluate how prone learning models are to monotonicity violations. These results clarify the limitations of current methods and provide guidance for developing more trustworthy preference learning algorithms.
@article{arxiv.2506.08998,
title = {On Monotonicity in AI Alignment},
author = {Gilles Bareilles and Julien Fageot and Lê-Nguyên Hoang and Peva Blanchard and Wassim Bouaziz and Sébastien Rouault and El-Mahdi El-Mhamdi},
journal= {arXiv preprint arXiv:2506.08998},
year = {2025}
}