We present a novel approach called Mixture of Mixture of Expert (MoMoE) that combines the strengths of Mixture-of-Experts (MoE) architectures with collaborative multi-agent frameworks. By modifying the LLaMA 3.1 8B architecture to incorporate MoE layers in each agent of a layered collaborative structure, we create an ensemble of specialized expert agents that iteratively refine their outputs. Each agent leverages an MoE layer in its final attention block, enabling efficient task decomposition while maintaining computational feasibility. This hybrid approach creates specialized pathways through both the model architecture and the agent collaboration layers. Experimental results demonstrate significant improvements across multiple language understanding and generation benchmarks, highlighting the synergistic benefits of combining expert routing at both the neural and agent levels.
@article{arxiv.2511.13983,
title = {MoMoE: A Mixture of Expert Agent Model for Financial Sentiment Analysis},
author = {Peng Shu and Junhao Chen and Zhengliang Liu and Hanqi Jiang and Yi Pan and Khanh Nhu Nguyen and Zihao Wu and Huaqin Zhao and Yiwei Li and Enze Shi and ShaoChen Xu},
journal= {arXiv preprint arXiv:2511.13983},
year = {2025}
}