Learned representations are often invariant to rotational transformations, leaving individual dimensions non-identifiable and interchangeable. We study how Matryoshka Representation Learning (MRL) induces a task-aligned privileged basis distinct from variance-based or regularizer-induced orderings. In the linear setting, we prove that full-prefix MRL recovers the ordered principal directions, and can be computed efficiently using shared statistics. Empirically, we demonstrate that MRL yields consistent per-dimension structure aligned with task signal, where coordinate magnitude reflects informativeness.
@article{arxiv.2605.09160,
title = {Objective-Specific Privileged Bases via Full-Prefix Matryoshka Learning},
author = {Arghamitra Talukder and Philippe Chlenski and Itsik Pe'er},
journal= {arXiv preprint arXiv:2605.09160},
year = {2026}
}