Large language models often lose previously aligned safety behaviors when fine-tuned on benign data, a phenomenon known as catastrophic forgetting. Prior work shows that adding random safety examples can mitigate this effect, but it remains unclear which examples are most effective. We propose a behavior-aware sampling framework that selects safety examples based on two complementary factors: instruction-response behavior (e.g., refusal versus compliance) and semantic diversity across harm categories. Systematic evaluation shows that this approach substantially reduces harmful outputs while maintaining helpfulness, achieving up to a 41% reduction in harmfulness with only 0.5% additional training data. These results highlight how targeted data selection can improve the safety and efficiency of fine-tuning at scale.
@article{arxiv.2510.21885,
title = {Preventing Catastrophic Forgetting: Behavior-Aware Sampling for Safer Language Model Fine-Tuning},
author = {Anh Pham and Mihir Thalanki and Michael Sun and Aditya Chaloo and Ankita Gupta and Tian Xia and Aditya Mate and Ehimwenma Nosakhare and Soundararajan Srinivasan},
journal= {arXiv preprint arXiv:2510.21885},
year = {2025}
}