Knowledge Graphs (KGs) store information in the form of (head, predicate, tail)-triples. To augment KGs with new knowledge, researchers proposed models for KG Completion (KGC) tasks such as link prediction; i.e., answering (h; p; ?) or (?; p; t) queries. Such models are usually evaluated with averaged metrics on a held-out test set. While useful for tracking progress, averaged single-score metrics cannot reveal what exactly a model has learned -- or failed to learn. To address this issue, we propose KGxBoard: an interactive framework for performing fine-grained evaluation on meaningful subsets of the data, each of which tests individual and interpretable capabilities of a KGC model. In our experiments, we highlight the findings that we discovered with the use of KGxBoard, which would have been impossible to detect with standard averaged single-score metrics.
@article{arxiv.2208.11024,
title = {KGxBoard: Explainable and Interactive Leaderboard for Evaluation of Knowledge Graph Completion Models},
author = {Haris Widjaja and Kiril Gashteovski and Wiem Ben Rim and Pengfei Liu and Christopher Malon and Daniel Ruffinelli and Carolin Lawrence and Graham Neubig},
journal= {arXiv preprint arXiv:2208.11024},
year = {2022}
}