Can we train a single transformer model capable of processing multiple modalities and datasets, whilst sharing almost all of its learnable parameters? We present PolyViT, a model trained on image, audio and video which answers this question. By co-training different tasks on a single modality, we are able to improve the accuracy of each individual task and achieve state-of-the-art results on 5 standard video- and audio-classification datasets. Co-training PolyViT on multiple modalities and tasks leads to a model that is even more parameter-efficient, and learns representations that generalize across multiple domains. Moreover, we show that co-training is simple and practical to implement, as we do not need to tune hyperparameters for each combination of datasets, but can simply adapt those from standard, single-task training.
@article{arxiv.2111.12993,
title = {PolyViT: Co-training Vision Transformers on Images, Videos and Audio},
author = {Valerii Likhosherstov and Anurag Arnab and Krzysztof Choromanski and Mario Lucic and Yi Tay and Adrian Weller and Mostafa Dehghani},
journal= {arXiv preprint arXiv:2111.12993},
year = {2021}
}