We present the training recipe and results of scaling up PaLI-X, a multilingual vision and language model, both in terms of size of the components and the breadth of its training task mixture. Our model achieves new levels of performance on a wide-range of varied and complex tasks, including multiple image-based captioning and question-answering tasks, image-based document understanding and few-shot (in-context) learning, as well as object detection, video question answering, and video captioning. PaLI-X advances the state-of-the-art on most vision-and-language benchmarks considered (25+ of them). Finally, we observe emerging capabilities, such as complex counting and multilingual object detection, tasks that are not explicitly in the training mix.
@article{arxiv.2305.18565,
title = {PaLI-X: On Scaling up a Multilingual Vision and Language Model},
author = {Xi Chen and Josip Djolonga and Piotr Padlewski and Basil Mustafa and Soravit Changpinyo and Jialin Wu and Carlos Riquelme Ruiz and Sebastian Goodman and Xiao Wang and Yi Tay and Siamak Shakeri and Mostafa Dehghani and Daniel Salz and Mario Lucic and Michael Tschannen and Arsha Nagrani and Hexiang Hu and Mandar Joshi and Bo Pang and Ceslee Montgomery and Paulina Pietrzyk and Marvin Ritter and AJ Piergiovanni and Matthias Minderer and Filip Pavetic and Austin Waters and Gang Li and Ibrahim Alabdulmohsin and Lucas Beyer and Julien Amelot and Kenton Lee and Andreas Peter Steiner and Yang Li and Daniel Keysers and Anurag Arnab and Yuanzhong Xu and Keran Rong and Alexander Kolesnikov and Mojtaba Seyedhosseini and Anelia Angelova and Xiaohua Zhai and Neil Houlsby and Radu Soricut},
journal= {arXiv preprint arXiv:2305.18565},
year = {2023}
}