We introduce a novel architecture, the Neuromodulation Gated Transformer (NGT), which is a simple implementation of neuromodulation in transformers via a multiplicative effect. We compare it to baselines and show that it results in the best average performance on the SuperGLUE benchmark validation sets.
Cite
@article{arxiv.2305.03232,
title = {Neuromodulation Gated Transformer},
author = {Kobe Knowles and Joshua Bensemann and Diana Benavides-Prado and Vithya Yogarajan and Michael Witbrock and Gillian Dobbie and Yang Chen},
journal= {arXiv preprint arXiv:2305.03232},
year = {2023}
}