We present a method and dataset for fine-tuning language models with preference supervision using feedback-driven improvement chains. Given a model response, an annotator provides fine-grained feedback by marking ``liked'' and ``disliked'' spans and specifying what they liked or disliked about them. The base model then rewrites the disliked spans accordingly, proceeding from left to right, forming a sequence of incremental improvements. We construct preference pairs for direct alignment from each adjacent step in the chain, enabling the model to learn from localized, targeted edits. We find that our approach outperforms direct alignment methods based on standard A/B preference ranking or full contrastive rewrites, demonstrating that structured, revision-based supervision leads to more efficient and effective preference tuning.
@article{arxiv.2512.23693,
title = {Fine-Tuning LLMs with Fine-Grained Human Feedback on Text Spans},
author = {Sky CH-Wang and Justin Svegliato and Helen Appel and Jason Eisner},
journal= {arXiv preprint arXiv:2512.23693},
year = {2025}
}