Masked Image Modeling (MIM) has become a ubiquitous self-supervised vision paradigm. In this work, we show that MIM objectives cause the learned representations to retain non-semantic information, which ultimately hurts performance during inference. We introduce a model-agnostic score for semantic invariance using Principal Component Analysis (PCA) on real and synthetic non-semantic images. Based on this score, we propose a simple method, Semantically Orthogonal Artifact Projection (SOAP), to directly suppress non-semantic information in patch representations, leading to consistent improvements in zero-shot performance across various MIM-based models. SOAP is a post-hoc suppression method, requires zero training, and can be attached to any model as a single linear head.
@article{arxiv.2604.00172,
title = {Suppressing Non-Semantic Noise in Masked Image Modeling Representations},
author = {Martine Hjelkrem-Tan and Marius Aasan and Rwiddhi Chakraborty and Gabriel Y. Arteaga and Changkyu Choi and Adín Ramírez Rivera},
journal= {arXiv preprint arXiv:2604.00172},
year = {2026}
}