This paper rethinks steady-hand robotic manipulation by using a weakly supervised framework that fuses calibration-aware perception with admittance control. Unlike conventional automation that relies on labor-intensive 2D labeling, our framework leverages reusable warm-up trajectories to extract implicit spatial information, thereby achieving calibration-aware, depth-resolved perception without the need for external fiducials or manual depth annotation. By explicitly characterizing residuals from observation and calibration models, the system establishes a task-space error budget from recorded warm-ups. The uncertainty budget yields a lateral closed-loop accuracy of approx. 49 micrometers at 95% confidence (worst-case testing subset) and a depth accuracy of <= 291 micrometers at 95% confidence bound during large in-plane moves. In a within-subject user study (N=8), the learned agent reduces overall NASA-TLX workload by 77.1% relative to the simple steady-hand assistance baseline. These results demonstrate that the weakly supervised agent improves the reliability of microscope-guided biomedical micromanipulation without introducing complex setup requirements, offering a practical framework for microscope-guided intervention.
@article{arxiv.2601.20776,
title = {Learning From a Steady Hand: A Weakly Supervised Agent for Robot Assistance under Microscopy},
author = {Huanyu Tian and Martin Huber and Lingyun Zeng and Zhe Han and Wayne Bennett and Giuseppe Silvestri and Gerardo Mendizabal-Ruiz and Tom Vercauteren and Alejandro Chavez-Badiola and Christos Bergeles},
journal= {arXiv preprint arXiv:2601.20776},
year = {2026}
}