Decentralised online learning enables runtime adaptation in cyber-physical multi-agent systems, but when operating conditions change, learned policies often require substantial trial-and-error interaction before recovering performance. To address this, we propose TwinLoop, a simulation-in-the-loop digital twin framework for online multi-agent reinforcement learning. When a context shift occurs, the digital twin is triggered to reconstruct the current system state, initialise from the latest agent policies, and perform accelerated policy improvement with simulation what-if analysis before synchronising updated parameters back to the agents in the physical system. We evaluate TwinLoop in a vehicular edge computing task-offloading scenario with changing workload and infrastructure conditions. The results suggest that digital twins can improve post-shift adaptation efficiency and reduce reliance on costly online trial-and-error.
@article{arxiv.2604.06610,
title = {TwinLoop: Simulation-in-the-Loop Digital Twins for Online Multi-Agent Reinforcement Learning},
author = {Nan Zhang and Zishuo Wang and Shuyu Huang and Georgios Diamantopoulos and Nikos Tziritas and Panagiotis Oikonomou and Georgios Theodoropoulos},
journal= {arXiv preprint arXiv:2604.06610},
year = {2026}
}