It is generally believed that robust training of extremely large networks is critical to their success in real-world applications. However, when taken to the extreme, methods that promote robustness can hurt the model's sensitivity to rare or underrepresented patterns. In this paper, we discuss this trade-off between sensitivity and robustness to natural (non-adversarial) perturbations by introducing two notions: contextual feature utility and contextual feature sensitivity. We propose Feature Contrastive Learning (FCL) that encourages a model to be more sensitive to the features that have higher contextual utility. Empirical results demonstrate that models trained with FCL achieve a better balance of robustness and sensitivity, leading to improved generalization in the presence of noise on both vision and NLP datasets.
@article{arxiv.2105.09394,
title = {Balancing Robustness and Sensitivity using Feature Contrastive Learning},
author = {Seungyeon Kim and Daniel Glasner and Srikumar Ramalingam and Cho-Jui Hsieh and Kishore Papineni and Sanjiv Kumar},
journal= {arXiv preprint arXiv:2105.09394},
year = {2021}
}