Measuring inter-annotator agreement is important for annotation tasks, but many metrics require a fully-annotated set of data, where all annotators annotate all samples. We define Sparse Probability of Agreement, SPA, which estimates the probability of agreement when not all annotator-item-pairs are available. We show that under certain conditions, SPA is an unbiased estimator, and we provide multiple weighing schemes for handling data with various degrees of annotation.
@article{arxiv.2208.06161,
title = {Sparse Probability of Agreement},
author = {Jeppe Nørregaard and Leon Derczynski},
journal= {arXiv preprint arXiv:2208.06161},
year = {2023}
}