Autoregressive models have become the de facto choice for sequence generation tasks, but standard approaches treat digits as independent tokens and apply cross-entropy loss, overlooking the coherent structure of numerical sequences. This paper introduces Numerical Token Integrity Loss (NTIL) to address this gap. NTIL operates at two levels: (1) token-level, where it extends the Earth Mover's Distance (EMD) to preserve ordinal relationships between numerical values, and (2) sequence-level, where it penalizes the overall discrepancy between the predicted and actual sequences. This dual approach improves numerical prediction and integrates effectively with LLMs/MLLMs. Extensive experiments show significant performance improvements with NTIL.
@article{arxiv.2505.13077,
title = {Advancing Sequential Numerical Prediction in Autoregressive Models},
author = {Xiang Fei and Jinghui Lu and Qi Sun and Hao Feng and Yanjie Wang and Wei Shi and An-Lan Wang and Jingqun Tang and Can Huang},
journal= {arXiv preprint arXiv:2505.13077},
year = {2025}
}