Medical text generation aims to assist with administrative work and highlight salient information to support decision-making. To reflect the specific requirements of medical text, in this paper, we propose a set of metrics to evaluate the completeness, conciseness, and attribution of the generated text at a fine-grained level. The metrics can be computed by various types of evaluators including instruction-following (both proprietary and open-source) and supervised entailment models. We demonstrate the effectiveness of the resulting framework, DocLens, with three evaluators on three tasks: clinical note generation, radiology report summarization, and patient question summarization. A comprehensive human study shows that DocLens exhibits substantially higher agreement with the judgments of medical experts than existing metrics. The results also highlight the need to improve open-source evaluators and suggest potential directions.
@article{arxiv.2311.09581,
title = {DocLens: Multi-aspect Fine-grained Evaluation for Medical Text Generation},
author = {Yiqing Xie and Sheng Zhang and Hao Cheng and Pengfei Liu and Zelalem Gero and Cliff Wong and Tristan Naumann and Hoifung Poon and Carolyn Rose},
journal= {arXiv preprint arXiv:2311.09581},
year = {2024}
}