We present an extensible user simulation toolkit to facilitate automatic evaluation of conversational recommender systems. It builds on an established agenda-based approach and extends it with several novel elements, including user satisfaction prediction, persona and context modeling, and conditional natural language generation. We showcase the toolkit with a pre-existing movie recommender system and demonstrate its ability to simulate dialogues that mimic real conversations, while requiring only a handful of manually annotated dialogues as training data.
@article{arxiv.2301.05544,
title = {UserSimCRS: A User Simulation Toolkit for Evaluating Conversational Recommender Systems},
author = {Jafar Afzali and Aleksander Mark Drzewiecki and Krisztian Balog and Shuo Zhang},
journal= {arXiv preprint arXiv:2301.05544},
year = {2023}
}
Comments
Proceedings of the Sixteenth ACM International Conference on Web Search and Data Mining