The use of denoising diffusion models is becoming increasingly popular in the field of image editing. However, current approaches often rely on either image-guided methods, which provide a visual reference but lack control over semantic consistency, or text-guided methods, which ensure alignment with the text guidance but compromise visual quality. To resolve this issue, we propose a framework that integrates a fusion of generated visual references and text guidance into the semantic latent space of a \textit{frozen} pre-trained diffusion model. Using only a tiny neural network, our framework provides control over diverse content and attributes, driven intuitively by the text prompt. Compared to state-of-the-art methods, the framework generates images of higher quality while providing realistic editing effects across various benchmark datasets.
@article{arxiv.2308.15854,
title = {Textual and Visual Prompt Fusion for Image Editing via Step-Wise Alignment},
author = {Zhanbo Feng and Zenan Ling and Xinyu Lu and Ci Gong and Feng Zhou and Wugedele Bao and Jie Li and Fan Yang and Robert C. Qiu},
journal= {arXiv preprint arXiv:2308.15854},
year = {2025}
}