Large Language Models (LLMs) have gained prominence in the AI landscape due to their exceptional performance. Thus, it is essential to gain a better understanding of their capabilities and limitations, among others in terms of nonmonotonic reasoning. This paper proposes a benchmark that corresponds to various defeasible rule-based reasoning patterns. We modified an existing benchmark for defeasible logic reasoners by translating defeasible rules into text suitable for LLMs. We conducted preliminary experiments on nonmonotonic rule-based reasoning using ChatGPT and compared it with reasoning patterns defined by defeasible logic.
@article{arxiv.2410.12509,
title = {Benchmarking Defeasible Reasoning with Large Language Models -- Initial Experiments and Future Directions},
author = {Ilias Tachmazidis and Sotiris Batsakis and Grigoris Antoniou},
journal= {arXiv preprint arXiv:2410.12509},
year = {2024}
}