Video Large Language Models (VLMs) have achieved remarkable success in video understanding, but the significant computational cost from processing dense frames severely limits their practical application. Existing methods alleviate this by selecting keyframes, but their greedy decision-making, combined with a decoupled evaluation of relevance and diversity, often falls into local optima and results in erroneously selecting irrelevant noise frames. To address these challenges, we propose GIFT: Global Irreplaceability Frame Targeting, a novel training-free framework that selects frames by assessing their intrinsic irreplaceability. Specifically, we first introduce Directed Diversity to quantify a frame's uniqueness conditioned on relevance, which allows us to formulate a unified irreplaceability score. Subsequently, our Budget-Aware Refinement strategy employs a adaptive iterative process that first secures a core set of frames with the highest irreplaceability, and then shifts its priority to building crucial temporal context around these selections as the budget expands. Extensive experiments demonstrate that GIFT achieves a maximum average improvement of 12.5% across long-form video benchmarks on LLaVA-Video-7B compared to uniform sampling.
@article{arxiv.2603.25072,
title = {GIFT: Global Irreplaceability Frame Targeting for Efficient Video Understanding},
author = {Junpeng Ma and Sashuai Zhou and Guanghao Li and Xin Gao and Yue Cao and Hengyu Zeng and Yuxiang Yan and Zhibin Wang and Jun Song and Bo Zheng and Shanghang Zhang and Jian Pu},
journal= {arXiv preprint arXiv:2603.25072},
year = {2026}
}