Dialogue safety problems severely limit the real-world deployment of neural conversational models and have attracted great research interests recently. However, dialogue safety problems remain under-defined and the corresponding dataset is scarce. We propose a taxonomy for dialogue safety specifically designed to capture unsafe behaviors in human-bot dialogue settings, with focuses on context-sensitive unsafety, which is under-explored in prior works. To spur research in this direction, we compile DiaSafety, a dataset with rich context-sensitive unsafe examples. Experiments show that existing safety guarding tools fail severely on our dataset. As a remedy, we train a dialogue safety classifier to provide a strong baseline for context-sensitive dialogue unsafety detection. With our classifier, we perform safety evaluations on popular conversational models and show that existing dialogue systems still exhibit concerning context-sensitive safety problems.
@article{arxiv.2110.08466,
title = {On the Safety of Conversational Models: Taxonomy, Dataset, and Benchmark},
author = {Hao Sun and Guangxuan Xu and Jiawen Deng and Jiale Cheng and Chujie Zheng and Hao Zhou and Nanyun Peng and Xiaoyan Zhu and Minlie Huang},
journal= {arXiv preprint arXiv:2110.08466},
year = {2022}
}