Although LLMs have shown promising potential in vulnerability detection, this study reveals their limitations in distinguishing between vulnerable and similar-but-benign patched code (only 0.06 - 0.14 accuracy). It shows that LLMs struggle to capture the root causes of vulnerabilities during vulnerability detection. To address this challenge, we propose enhancing LLMs with multi-dimensional vulnerability knowledge distilled from historical vulnerabilities and fixes. We design a novel knowledge-level Retrieval-Augmented Generation framework Vul-RAG, which improves LLMs with an accuracy increase of 16% - 24% in identifying vulnerable and patched code. Additionally, vulnerability knowledge generated by Vul-RAG can further (1) serve as high-quality explanations to improve manual detection accuracy (from 60% to 77%), and (2) detect 10 previously-unknown bugs in the recent Linux kernel release with 6 assigned CVEs.
@article{arxiv.2406.11147,
title = {Vul-RAG: Enhancing LLM-based Vulnerability Detection via Knowledge-level RAG},
author = {Xueying Du and Geng Zheng and Kaixin Wang and Yi Zou and Yujia Wang and Wentai Deng and Jiayi Feng and Mingwei Liu and Bihuan Chen and Xin Peng and Tao Ma and Yiling Lou},
journal= {arXiv preprint arXiv:2406.11147},
year = {2025}
}