In the automatic evaluation of generative question answering (GenQA) systems, it is difficult to assess the correctness of generated answers due to the free-form of the answer. Especially, widely used n-gram similarity metrics often fail to discriminate the incorrect answers since they equally consider all of the tokens. To alleviate this problem, we propose KPQA-metric, a new metric for evaluating the correctness of GenQA. Specifically, our new metric assigns different weights to each token via keyphrase prediction, thereby judging whether a generated answer sentence captures the key meaning of the reference answer. To evaluate our metric, we create high-quality human judgments of correctness on two GenQA datasets. Using our human-evaluation datasets, we show that our proposed metric has a significantly higher correlation with human judgments than existing metrics. The code is available at https://github.com/hwanheelee1993/KPQA.
@article{arxiv.2005.00192,
title = {KPQA: A Metric for Generative Question Answering Using Keyphrase Weights},
author = {Hwanhee Lee and Seunghyun Yoon and Franck Dernoncourt and Doo Soon Kim and Trung Bui and Joongbo Shin and Kyomin Jung},
journal= {arXiv preprint arXiv:2005.00192},
year = {2021}
}