Extracting geometry features from photographic images independently of surface texture and transferring them onto different materials remains a complex challenge. In this study, we introduce Harmonizing Attention, a novel training-free approach that leverages diffusion models for texture-aware geometry transfer. Our method employs a simple yet effective modification of self-attention layers, allowing the model to query information from multiple reference images within these layers. This mechanism is seamlessly integrated into the inversion process as Texture-aligning Attention and into the generation process as Geometry-aligning Attention. This dual-attention approach ensures the effective capture and transfer of material-independent geometry features while maintaining material-specific textural continuity, all without the need for model fine-tuning.
@article{arxiv.2408.10846,
title = {Harmonizing Attention: Training-free Texture-aware Geometry Transfer},
author = {Eito Ikuta and Yohan Lee and Akihiro Iohara and Yu Saito and Toshiyuki Tanaka},
journal= {arXiv preprint arXiv:2408.10846},
year = {2024}
}