Graph Transformers have garnered significant attention for learning graph-structured data, thanks to their superb ability to capture long-range dependencies among nodes. However, the quadratic space and time complexity hinders the scalability of Graph Transformers, particularly for large-scale recommendation. Here we propose an efficient Masked Graph Transformer, named MGFormer, capable of capturing all-pair interactions among nodes with a linear complexity. To achieve this, we treat all user/item nodes as independent tokens, enhance them with positional embeddings, and feed them into a kernelized attention module. Additionally, we incorporate learnable relative degree information to appropriately reweigh the attentions. Experimental results show the superior performance of our MGFormer, even with a single attention layer.
@article{arxiv.2405.04028,
title = {Masked Graph Transformer for Large-Scale Recommendation},
author = {Huiyuan Chen and Zhe Xu and Chin-Chia Michael Yeh and Vivian Lai and Yan Zheng and Minghua Xu and Hanghang Tong},
journal= {arXiv preprint arXiv:2405.04028},
year = {2024}
}