The majority of text is stored in UTF-8, which must be validated on ingestion. We present the lookup algorithm, which outperforms UTF-8 validation routines used in many libraries and languages by more than 10 times using commonly available SIMD instructions. To ensure reproducibility, our work is freely available as open source software.
@article{arxiv.2010.03090,
title = {Validating UTF-8 In Less Than One Instruction Per Byte},
author = {John Keiser and Daniel Lemire},
journal= {arXiv preprint arXiv:2010.03090},
year = {2026}
}