The human hippocampus possesses "concept cells", neurons that fire when presented with stimuli belonging to a specific concept, regardless of the modality. Recently, similar concept cells were discovered in a multimodal network called CLIP (Radford et at., 2021). Here, we ask whether CLIP can explain the fMRI activity of the human hippocampus better than a purely visual (or linguistic) model. We extend our analysis to a range of publicly available uni- and multi-modal models. We demonstrate that "multimodality" stands out as a key component when assessing the ability of a network to explain the multivoxel activity in the hippocampus.
@article{arxiv.2201.11517,
title = {Multimodal neural networks better explain multivoxel patterns in the hippocampus},
author = {Bhavin Choksi and Milad Mozafari and Rufin VanRullen and Leila Reddy},
journal= {arXiv preprint arXiv:2201.11517},
year = {2022}
}