Contrastive learning and supervised learning have both seen significant progress and success. However, thus far they have largely been treated as two separate objectives, brought together only by having a shared neural network. In this paper we show that through the perspective of hybrid discriminative-generative training of energy-based models we can make a direct connection between contrastive learning and supervised learning. Beyond presenting this unified view, we show our specific choice of approximation of the energy-based loss outperforms the existing practice in terms of classification accuracy of WideResNet on CIFAR-10 and CIFAR-100. It also leads to improved performance on robustness, out-of-distribution detection, and calibration.
@article{arxiv.2007.09070,
title = {Hybrid Discriminative-Generative Training via Contrastive Learning},
author = {Hao Liu and Pieter Abbeel},
journal= {arXiv preprint arXiv:2007.09070},
year = {2020}
}