A new loss function is proposed for neural networks on classification tasks which extends the hinge loss by assigning gradients to its critical points. We will show that for a linear classifier on linearly separable data with fixed step size, the margin of this modified hinge loss converges to the ℓ2 max-margin at the rate of O(1/t). This rate is fast when compared with the O(1/logt) rate of exponential losses such as the logistic loss. Furthermore, empirical results suggest that this increased convergence speed carries over to ReLU networks.
@article{arxiv.2006.14286,
title = {Implicitly Maximizing Margins with the Hinge Loss},
author = {Justin Lizama},
journal= {arXiv preprint arXiv:2006.14286},
year = {2020}
}