Self-supervised methods received tremendous attention thanks to their seemingly heuristic approach to learning representations that respect the semantics of the data without any apparent supervision in the form of labels. A growing body of literature is already being published in an attempt to build a coherent and theoretically grounded understanding of the workings of a zoo of losses used in modern self-supervised representation learning methods. In this paper, we attempt to provide an understanding from the perspective of a Laplace operator and connect the inductive bias stemming from the augmentation process to a low-rank matrix completion problem. To this end, we leverage the results from low-rank matrix completion to provide theoretical analysis on the convergence of modern SSL methods and a key property that affects their downstream performance.
@article{arxiv.2305.19818,
title = {Spectal Harmonics: Bridging Spectral Embedding and Matrix Completion in Self-Supervised Learning},
author = {Marina Munkhoeva and Ivan Oseledets},
journal= {arXiv preprint arXiv:2305.19818},
year = {2023}
}