Simultaneously transmitting and reflecting reconfigurable intelligent surfaces (STAR-RISs) enable full-space coverage but also expose wireless transmissions to security from multiple spatial directions. This paper investigates a STAR-RIS-assisted secure RSMA system where both internal and external eavesdroppers may coexist in the transmission and reflection regions. In such a scenario, the RSMA common stream simultaneously serves legitimate users, impairs external eavesdroppers, and avoids assisting internal eavesdroppers, leading to a challenging trade-off between spectral efficiency and confidentiality. To address this issue, we formulate a max-min fairness problem under secrecy constraints and develop an iterative algorithm to jointly optimize transmit beamforming and STAR-RIS phase shifts. Simulation results demonstrate that the proposed scheme improves spectral efficiency while maintaining confidentiality.
@article{arxiv.2603.26532,
title = {Security-Spectral Efficiency Tradeoff in STAR-RIS RSMA: A Max-Min Fairness Framework},
author = {Huiyun Xia and Yijie Mao and Sai Xu and Shuai Han and Hongbo Zhu},
journal= {arXiv preprint arXiv:2603.26532},
year = {2026}
}