Building generalized models that can solve many computer vision tasks simultaneously is an intriguing direction. Recent works have shown image itself can be used as a natural interface for general-purpose visual perception and demonstrated inspiring results. In this paper, we explore diffusion-based vision generalists, where we unify different types of dense prediction tasks as conditional image generation and re-purpose pre-trained diffusion models for it. However, directly applying off-the-shelf latent diffusion models leads to a quantization issue. Thus, we propose to perform diffusion in pixel space and provide a recipe for finetuning pre-trained text-to-image diffusion models for dense vision tasks. In experiments, we evaluate our method on four different types of tasks and show competitive performance to the other vision generalists.
@article{arxiv.2407.00503,
title = {Toward a Diffusion-Based Generalist for Dense Vision Tasks},
author = {Yue Fan and Yongqin Xian and Xiaohua Zhai and Alexander Kolesnikov and Muhammad Ferjad Naeem and Bernt Schiele and Federico Tombari},
journal= {arXiv preprint arXiv:2407.00503},
year = {2024}
}