Despite the growing use of large language models (LLMs) for providing feedback, limited research has explored how to achieve high-quality feedback. This case study introduces an evaluation framework to assess different zero-shot prompt engineering methods. We varied the prompts systematically and analyzed the provided feedback on programming errors in R. The results suggest that prompts suggesting a stepwise procedure increase the precision, while omitting explicit specifications about which provided data to analyze improves error identification.
@article{arxiv.2412.15702,
title = {Cracking the Code: Evaluating Zero-Shot Prompting Methods for Providing Programming Feedback},
author = {Niklas Ippisch and Anna-Carolina Haensch and Jan Simson and Jacob Beck and Markus Herklotz and Malte Schierholz},
journal= {arXiv preprint arXiv:2412.15702},
year = {2024}
}