Large language models (LLMs) have shown significant advancements in code generation, but still face challenges on tasks beyond their basic capabilities. Recently, the notion of self-debugging has been proposed to boost the performance of code generation by leveraging execution feedback from tests. Despite its promise, the availability of high-quality tests in real-world scenarios is limited. In this context, self-debugging with self-generated tests is a promising solution but lacks a full exploration of its limitations and practical potential. Therefore, we investigate its efficacy on diverse programming problems. To deepen our understanding, we propose two distinct paradigms for the process: post-execution and in-execution self-debugging. Within the scope of self-contained Python programming tasks, we find that post-execution self-debugging struggles on basic problems but shows potential for improvement on competitive ones, due to the bias introduced by self-generated tests. On the other hand, in-execution self-debugging enables LLMs to mitigate the bias by solely leveraging intermediate states during execution, thereby enhancing code generation.
@article{arxiv.2501.12793,
title = {Revisit Self-Debugging with Self-Generated Tests for Code Generation},
author = {Xiancai Chen and Zhengwei Tao and Kechi Zhang and Changzhi Zhou and Wanli Gu and Yuanpeng He and Mengdi Zhang and Xunliang Cai and Haiyan Zhao and Zhi Jin},
journal= {arXiv preprint arXiv:2501.12793},
year = {2025}
}