Updating diffusion models in an incremental setting would be practical in real-world applications yet computationally challenging. We present a novel learning strategy of Concept Neuron Selection (CNS), a simple yet effective approach to perform personalization in a continual learning scheme. CNS uniquely identifies neurons in diffusion models that are closely related to the target concepts. In order to mitigate catastrophic forgetting problems while preserving zero-shot text-to-image generation ability, CNS finetunes concept neurons in an incremental manner and jointly preserves knowledge learned of previous concepts. Evaluation of real-world datasets demonstrates that CNS achieves state-of-the-art performance with minimal parameter adjustments, outperforming previous methods in both single and multi-concept personalization works. CNS also achieves fusion-free operation, reducing memory storage and processing time for continual personalization.
@article{arxiv.2510.02296,
title = {Continual Personalization for Diffusion Models},
author = {Yu-Chien Liao and Jr-Jen Chen and Chi-Pin Huang and Ci-Siang Lin and Meng-Lin Wu and Yu-Chiang Frank Wang},
journal= {arXiv preprint arXiv:2510.02296},
year = {2025}
}