Our research is in the relatively unexplored area of question answering technologies for patient-specific questions over their electronic health records. A large dataset of human expert curated question and answer pairs is an important pre-requisite for developing, training and evaluating any question answering system that is powered by machine learning. In this paper, we describe a process for creating such a dataset of questions and answers. Our methodology is replicable, can be conducted by medical students as annotators, and results in high inter-annotator agreement (0.71 Cohen's kappa). Over the course of 11 months, 11 medical students followed our annotation methodology, resulting in a question answering dataset of 5696 questions over 71 patient records, of which 1747 questions have corresponding answers generated by the medical students.
@article{arxiv.1805.06816,
title = {Annotating Electronic Medical Records for Question Answering},
author = {Preethi Raghavan and Siddharth Patwardhan and Jennifer J. Liang and Murthy V. Devarakonda},
journal= {arXiv preprint arXiv:1805.06816},
year = {2018}
}