We describe the setting and results of the ConvAI2 NeurIPS competition that aims to further the state-of-the-art in open-domain chatbots. Some key takeaways from the competition are: (i) pretrained Transformer variants are currently the best performing models on this task, (ii) but to improve performance on multi-turn conversations with humans, future systems must go beyond single word metrics like perplexity to measure the performance across sequences of utterances (conversations) -- in terms of repetition, consistency and balance of dialogue acts (e.g. how many questions asked vs. answered).
@article{arxiv.1902.00098,
title = {The Second Conversational Intelligence Challenge (ConvAI2)},
author = {Emily Dinan and Varvara Logacheva and Valentin Malykh and Alexander Miller and Kurt Shuster and Jack Urbanek and Douwe Kiela and Arthur Szlam and Iulian Serban and Ryan Lowe and Shrimai Prabhumoye and Alan W Black and Alexander Rudnicky and Jason Williams and Joelle Pineau and Mikhail Burtsev and Jason Weston},
journal= {arXiv preprint arXiv:1902.00098},
year = {2019}
}