Interpretability and efficiency are two important considerations for the adoption of neural automatic metrics. In this work, we develop strong-performing automatic metrics for reference-based summarization evaluation, based on a two-stage evaluation pipeline that first extracts basic information units from one text sequence and then checks the extracted units in another sequence. The metrics we developed include two-stage metrics that can provide high interpretability at both the fine-grained unit level and summary level, and one-stage metrics that achieve a balance between efficiency and interpretability. We make the developed tools publicly available at https://github.com/Yale-LILY/AutoACU.
@article{arxiv.2303.03608,
title = {Towards Interpretable and Efficient Automatic Reference-Based Summarization Evaluation},
author = {Yixin Liu and Alexander R. Fabbri and Yilun Zhao and Pengfei Liu and Shafiq Joty and Chien-Sheng Wu and Caiming Xiong and Dragomir Radev},
journal= {arXiv preprint arXiv:2303.03608},
year = {2023}
}