We present Any-Modality Augmented Language Model (AnyMAL), a unified model that reasons over diverse input modality signals (i.e. text, image, video, audio, IMU motion sensor), and generates textual responses. AnyMAL inherits the powerful text-based reasoning abilities of the state-of-the-art LLMs including LLaMA-2 (70B), and converts modality-specific signals to the joint textual space through a pre-trained aligner module. To further strengthen the multimodal LLM's capabilities, we fine-tune the model with a multimodal instruction set manually collected to cover diverse topics and tasks beyond simple QAs. We conduct comprehensive empirical analysis comprising both human and automatic evaluations, and demonstrate state-of-the-art performance on various multimodal tasks.
@article{arxiv.2309.16058,
title = {AnyMAL: An Efficient and Scalable Any-Modality Augmented Language Model},
author = {Seungwhan Moon and Andrea Madotto and Zhaojiang Lin and Tushar Nagarajan and Matt Smith and Shashank Jain and Chun-Fu Yeh and Prakash Murugesan and Peyman Heidari and Yue Liu and Kavya Srinet and Babak Damavandi and Anuj Kumar},
journal= {arXiv preprint arXiv:2309.16058},
year = {2023}
}