In this technical report, we evaluated the performance of the ChatGPT and GPT-3 models for the task of vulnerability detection in code. Our evaluation was conducted on our real-world dataset, using binary and multi-label classification tasks on CWE vulnerabilities. We decided to evaluate the model because it has shown good performance on other code-based tasks, such as solving programming challenges and understanding code at a high level. However, we found that the ChatGPT model performed no better than a dummy classifier for both binary and multi-label classification tasks for code vulnerability detection.
@article{arxiv.2304.07232,
title = {Evaluation of ChatGPT Model for Vulnerability Detection},
author = {Anton Cheshkov and Pavel Zadorozhny and Rodion Levichev},
journal= {arXiv preprint arXiv:2304.07232},
year = {2023}
}