Remote sensing imagery is dense with objects and contextual visual information. There is a recent trend to combine paired satellite images and text captions for pretraining performant encoders for downstream tasks. However, while contrastive image-text methods like CLIP enable vision-language alignment and zero-shot classification ability, vision-only downstream performance tends to degrade compared to image-only pretraining, such as MAE. In this paper, we propose FLAVARS, a pretraining method that combines the best of both contrastive learning and masked modeling, along with geospatial alignment via contrastive location encoding. We find that FLAVARS significantly outperforms a baseline of SkyCLIP for vision-only tasks such as KNN classification and semantic segmentation, +6\% mIOU on SpaceNet1, while retaining the ability to perform zero-shot classification, unlike MAE pretrained methods.
@article{arxiv.2501.08490,
title = {FLAVARS: A Multimodal Foundational Language and Vision Alignment Model for Remote Sensing},
author = {Isaac Corley and Simone Fobi Nsutezo and Anthony Ortiz and Caleb Robinson and Rahul Dodhia and Juan M. Lavista Ferres and Peyman Najafirad},
journal= {arXiv preprint arXiv:2501.08490},
year = {2025}
}