We introduce HUBERT which combines the structured-representational power of Tensor-Product Representations (TPRs) and BERT, a pre-trained bidirectional Transformer language model. We show that there is shared structure between different NLP datasets that HUBERT, but not BERT, is able to learn and leverage. We validate the effectiveness of our model on the GLUE benchmark and HANS dataset. Our experiment results show that untangling data-specific semantics from general language structure is key for better transfer among NLP tasks.
@article{arxiv.1910.12647,
title = {HUBERT Untangles BERT to Improve Transfer across NLP Tasks},
author = {Mehrad Moradshahi and Hamid Palangi and Monica S. Lam and Paul Smolensky and Jianfeng Gao},
journal= {arXiv preprint arXiv:1910.12647},
year = {2021}
}