LLM agents have become increasingly sophisticated, especially in the realm of cybersecurity. Researchers have shown that LLM agents can exploit real-world vulnerabilities when given a description of the vulnerability and toy capture-the-flag problems. However, these agents still perform poorly on real-world vulnerabilities that are unknown to the agent ahead of time (zero-day vulnerabilities). In this work, we show that teams of LLM agents can exploit real-world, zero-day vulnerabilities. Prior agents struggle with exploring many different vulnerabilities and long-range planning when used alone. To resolve this, we introduce HPTSA, a system of agents with a planning agent that can launch subagents. The planning agent explores the system and determines which subagents to call, resolving long-term planning issues when trying different vulnerabilities. We construct a benchmark of 14 real-world vulnerabilities and show that our team of agents improve over prior agent frameworks by up to 4.3X.
@article{arxiv.2406.01637,
title = {Teams of LLM Agents can Exploit Zero-Day Vulnerabilities},
author = {Yuxuan Zhu and Antony Kellermann and Akul Gupta and Philip Li and Richard Fang and Rohan Bindu and Daniel Kang},
journal= {arXiv preprint arXiv:2406.01637},
year = {2025}
}