Adversarial training is the most promising method for learning robust models against adversarial examples. A recent study has shown that knowledge distillation between the same architectures is effective in improving the performance of adversarial training. Exploiting knowledge distillation is a new approach to improve adversarial training and has attracted much attention. However, its performance is still insufficient. Therefore, we propose Adversarial Robust Distillation with Internal Representation~(ARDIR) to utilize knowledge distillation even more effectively. In addition to the output of the teacher model, ARDIR uses the internal representation of the teacher model as a label for adversarial training. This enables the student model to be trained with richer, more informative labels. As a result, ARDIR can learn more robust student models. We show that ARDIR outperforms previous methods in our experiments.
@article{arxiv.2211.00239,
title = {ARDIR: Improving Robustness using Knowledge Distillation of Internal Representation},
author = {Tomokatsu Takahashi and Masanori Yamada and Yuuki Yamanaka and Tomoya Yamashita},
journal= {arXiv preprint arXiv:2211.00239},
year = {2022}
}