The rapid development of such natural language processing tasks as style transfer, paraphrase, and machine translation often calls for the use of semantic similarity metrics. In recent years a lot of methods to measure the semantic similarity of two short texts were developed. This paper provides a comprehensive analysis for more than a dozen of such methods. Using a new dataset of fourteen thousand sentence pairs human-labeled according to their semantic similarity, we demonstrate that none of the metrics widely used in the literature is close enough to human judgment in these tasks. A number of recently proposed metrics provide comparable results, yet Word Mover Distance is shown to be the most reasonable solution to measure semantic similarity in reformulated texts at the moment.
@article{arxiv.2004.05001,
title = {Style-transfer and Paraphrase: Looking for a Sensible Semantic Similarity Metric},
author = {Ivan P. Yamshchikov and Viacheslav Shibaev and Nikolay Khlebnikov and Alexey Tikhonov},
journal= {arXiv preprint arXiv:2004.05001},
year = {2022}
}