This work evaluates the potential of large language models (LLMs) to power digital assistants capable of complex action execution. These assistants rely on pre-trained programming knowledge to execute multi-step goals by composing objects and functions defined in assistant libraries into action execution programs. To achieve this, we develop ASPERA, a framework comprising an assistant library simulation and a human-assisted LLM data generation engine. Our engine allows developers to guide LLM generation of high-quality tasks consisting of complex user queries, simulation state and corresponding validation programs, tackling data availability and evaluation robustness challenges. Alongside the framework we release Asper-Bench, an evaluation dataset of 250 challenging tasks generated using ASPERA, which we use to show that program generation grounded in custom assistant libraries is a significant challenge to LLMs compared to dependency-free code generation.
@article{arxiv.2507.15501,
title = {ASPERA: A Simulated Environment to Evaluate Planning for Complex Action Execution},
author = {Alexandru Coca and Mark Gaynor and Zhenxing Zhang and Jianpeng Cheng and Bo-Hsiang Tseng and Pete Boothroyd and Héctor Martinez Alonso and Diarmuid Ó Séaghdha and Anders Johannsen},
journal= {arXiv preprint arXiv:2507.15501},
year = {2025}
}