We explore the use of a knowledge graphs, that capture general or commonsense knowledge, to augment the information extracted from images by the state-of-the-art methods for image captioning. The results of our experiments, on several benchmark data sets such as MS COCO, as measured by CIDEr-D, a performance metric for image captioning, show that the variants of the state-of-the-art methods for image captioning that make use of the information extracted from knowledge graphs can substantially outperform those that rely solely on the information extracted from images.
@article{arxiv.1901.08942,
title = {Improving Image Captioning by Leveraging Knowledge Graphs},
author = {Yimin Zhou and Yiwei Sun and Vasant Honavar},
journal= {arXiv preprint arXiv:1901.08942},
year = {2019}
}