To analyze the limitations and the future directions of the extractive summarization paradigm, this paper proposes an Integer Linear Programming (ILP) formulation to obtain extractive oracle summaries in terms of ROUGE-N. We also propose an algorithm that enumerates all of the oracle summaries for a set of reference summaries to exploit F-measures that evaluate which system summaries contain how many sentences that are extracted as an oracle summary. Our experimental results obtained from Document Understanding Conference (DUC) corpora demonstrated the following: (1) room still exists to improve the performance of extractive summarization; (2) the F-measures derived from the enumerated oracle summaries have significantly stronger correlations with human judgment than those derived from single oracle summaries.
@article{arxiv.1701.01614,
title = {Enumeration of Extractive Oracle Summaries},
author = {Tsutomu Hirao and Masaaki Nishino and Jun Suzuki and Masaaki Nagata},
journal= {arXiv preprint arXiv:1701.01614},
year = {2017}
}