Joint Embedding Predictive Architectures (JEPA) are a novel self supervised training technique that have shown recent promise across domains. We introduce BERT-JEPA (BEPA), a training paradigm that adds a JEPA training objective to BERT-style models, working to combat a collapsed [CLS] embedding space and turning it into a language-agnostic space. This new structure leads to increased performance across multilingual benchmarks.
Cite
@article{arxiv.2601.00366,
title = {BERT-JEPA: Reorganizing CLS Embeddings for Language-Invariant Semantics},
author = {Taj Gillin and Adam Lalani and Kenneth Zhang and Marcel Mateos Salles},
journal= {arXiv preprint arXiv:2601.00366},
year = {2026}
}