We propose a learning system in which language is grounded in visual percepts without specific pre-defined categories of terms. We present a unified generative method to acquire a shared semantic/visual embedding that enables the learning of language about a wide range of real-world objects. We evaluate the efficacy of this learning by predicting the semantics of objects and comparing the performance with neural and non-neural inputs. We show that this generative approach exhibits promising results in language grounding without pre-specifying visual categories under low resource settings. Our experiments demonstrate that this approach is generalizable to multilingual, highly varied datasets.
@article{arxiv.2107.14593,
title = {Neural Variational Learning for Grounded Language Acquisition},
author = {Nisha Pillai and Cynthia Matuszek and Francis Ferraro},
journal= {arXiv preprint arXiv:2107.14593},
year = {2021}
}