Assessing the quality of summarizers poses significant challenges. In response, we propose a novel task-oriented evaluation approach that assesses summarizers based on their capacity to produce summaries that are useful for downstream tasks, while preserving task outcomes. We theoretically establish a direct relationship between the resulting error probability of these tasks and the mutual information between source texts and generated summaries. We introduce COSMIC as a practical implementation of this metric, demonstrating its strong correlation with human judgment-based metrics and its effectiveness in predicting downstream task performance. Comparative analyses against established metrics like BERTScore and ROUGE highlight the competitive performance of COSMIC.
@article{arxiv.2402.19457,
title = {$\texttt{COSMIC}$: Mutual Information for Task-Agnostic Summarization Evaluation},
author = {Maxime Darrin and Philippe Formont and Jackie Chi Kit Cheung and Pablo Piantanida},
journal= {arXiv preprint arXiv:2402.19457},
year = {2024}
}