We present Mean Box Pooling, a novel visual representation that pools over CNN representations of a large number, highly overlapping object proposals. We show that such representation together with nCCA, a successful multimodal embedding technique, achieves state-of-the-art performance on the Visual Madlibs task. Moreover, inspired by the nCCA's objective function, we extend classical CNN+LSTM approach to train the network by directly maximizing the similarity between the internal representation of the deep learning architecture and candidate answers. Again, such approach achieves a significant improvement over the prior work that also uses CNN+LSTM approach on Visual Madlibs.
@article{arxiv.1608.02717,
title = {Mean Box Pooling: A Rich Image Representation and Output Embedding for the Visual Madlibs Task},
author = {Ashkan Mokarian and Mateusz Malinowski and Mario Fritz},
journal= {arXiv preprint arXiv:1608.02717},
year = {2016}
}