Large-language models have recently demonstrated impressive zero-shot capabilities in a variety of natural language tasks such as summarization, dialogue generation, and question-answering. Despite many promising applications in clinical medicine, adoption of these models in real-world settings has been largely limited by their tendency to generate incorrect and sometimes even toxic statements. In this study, we develop Almanac, a large language model framework augmented with retrieval capabilities for medical guideline and treatment recommendations. Performance on a novel dataset of clinical scenarios (n = 130) evaluated by a panel of 5 board-certified and resident physicians demonstrates significant increases in factuality (mean of 18% at p-value < 0.05) across all specialties, with improvements in completeness and safety. Our results demonstrate the potential for large language models to be effective tools in the clinical decision-making process, while also emphasizing the importance of careful testing and deployment to mitigate their shortcomings.
@article{arxiv.2303.01229,
title = {Almanac: Retrieval-Augmented Language Models for Clinical Medicine},
author = {Cyril Zakka and Akash Chaurasia and Rohan Shad and Alex R. Dalal and Jennifer L. Kim and Michael Moor and Kevin Alexander and Euan Ashley and Jack Boyd and Kathleen Boyd and Karen Hirsch and Curt Langlotz and Joanna Nelson and William Hiesinger},
journal= {arXiv preprint arXiv:2303.01229},
year = {2023}
}