We introduce the 2-simplicial Transformer, an extension of the Transformer which includes a form of higher-dimensional attention generalising the dot-product attention, and uses this attention to update entity representations with tensor products of value vectors. We show that this architecture is a useful inductive bias for logical reasoning in the context of deep reinforcement learning.
@article{arxiv.1909.00668,
title = {Logic and the $2$-Simplicial Transformer},
author = {James Clift and Dmitry Doryn and Daniel Murfet and James Wallbridge},
journal= {arXiv preprint arXiv:1909.00668},
year = {2019}
}