Recently, Pyramid Adversarial training (Herrmann et al., 2022) has been shown to be very effective for improving clean accuracy and distribution-shift robustness of vision transformers. However, due to the iterative nature of adversarial training, the technique is up to 7 times more expensive than standard training. To make the method more efficient, we propose Universal Pyramid Adversarial training, where we learn a single pyramid adversarial pattern shared across the whole dataset instead of the sample-wise patterns. With our proposed technique, we decrease the computational cost of Pyramid Adversarial training by up to 70% while retaining the majority of its benefit on clean performance and distribution-shift robustness. In addition, to the best of our knowledge, we are also the first to find that universal adversarial training can be leveraged to improve clean model performance.
@article{arxiv.2312.16339,
title = {Universal Pyramid Adversarial Training for Improved ViT Performance},
author = {Ping-yeh Chiang and Yipin Zhou and Omid Poursaeed and Satya Narayan Shukla and Ashish Shah and Tom Goldstein and Ser-Nam Lim},
journal= {arXiv preprint arXiv:2312.16339},
year = {2023}
}