The integration of Knowledge Graphs (KGs) into the Retrieval Augmented Generation (RAG) framework has attracted significant interest, with early studies showing promise in mitigating hallucinations and improving model accuracy. However, a systematic understanding and comparative analysis of the rapidly emerging KG-RAG methods are still lacking. This paper seeks to lay the foundation for systematically answering the question of when and how to use KG-RAG by analyzing their performance in various application scenarios associated with different technical configurations. After outlining the mind map using KG-RAG framework and summarizing its popular pipeline, we conduct a pilot empirical study of KG-RAG works to reimplement and evaluate 6 KG-RAG methods across 9 datasets in diverse domains and scenarios, analyzing the impact of 9 KG-RAG configurations in combination with 17 LLMs, and combining Metacognition with KG-RAG as a pilot attempt. Our results underscore the critical role of appropriate application conditions and optimal configurations of KG-RAG components.
@article{arxiv.2502.20854,
title = {A Pilot Empirical Study on When and How to Use Knowledge Graphs as Retrieval Augmented Generation},
author = {Xujie Yuan and Yongxu Liu and Shimin Di and Shiwen Wu and Libin Zheng and Rui Meng and Lei Chen and Xiaofang Zhou and Jian Yin},
journal= {arXiv preprint arXiv:2502.20854},
year = {2025}
}