This paper studies the best practices for automatic machine learning (AutoML). While previous AutoML efforts have predominantly focused on unimodal data, the multimodal aspect remains under-explored. Our study delves into classification and regression problems involving flexible combinations of image, text, and tabular data. We curate a benchmark comprising 22 multimodal datasets from diverse real-world applications, encompassing all 4 combinations of the 3 modalities. Across this benchmark, we scrutinize design choices related to multimodal fusion strategies, multimodal data augmentation, converting tabular data into text, cross-modal alignment, and handling missing modalities. Through extensive experimentation and analysis, we distill a collection of effective strategies and consolidate them into a unified pipeline, achieving robust performance on diverse datasets.
@article{arxiv.2412.16243,
title = {Bag of Tricks for Multimodal AutoML with Image, Text, and Tabular Data},
author = {Zhiqiang Tang and Zihan Zhong and Tong He and Gerald Friedland},
journal= {arXiv preprint arXiv:2412.16243},
year = {2024}
}