Prompts have been proven to play a crucial role in large language models, and in recent years, vision models have also been using prompts to improve scalability for multiple downstream tasks. In this paper, we focus on adapting prompt design based on instruction tuning into a visual transformer model for image classification which we called Instruction-ViT. The key idea is to implement multi-modal prompts (text or image prompt) related to category information to guide the fine-tuning of the model. Based on the experiments of several image captionining tasks, the performance and domain adaptability were improved. Our work provided an innovative strategy to fuse multi-modal prompts with better performance and faster adaptability for visual classification models.
@article{arxiv.2305.00201,
title = {Instruction-ViT: Multi-Modal Prompts for Instruction Learning in ViT},
author = {Zhenxiang Xiao and Yuzhong Chen and Lu Zhang and Junjie Yao and Zihao Wu and Xiaowei Yu and Yi Pan and Lin Zhao and Chong Ma and Xinyu Liu and Wei Liu and Xiang Li and Yixuan Yuan and Dinggang Shen and Dajiang Zhu and Tianming Liu and Xi Jiang},
journal= {arXiv preprint arXiv:2305.00201},
year = {2023}
}