Executing contact-rich manipulation tasks necessitates the fusion of tactile and visual feedback. However, the distinct nature of these modalities poses significant challenges. In this paper, we introduce a system that leverages visual and tactile sensory inputs to enable dexterous in-hand manipulation. Specifically, we propose Robot Synesthesia, a novel point cloud-based tactile representation inspired by human tactile-visual synesthesia. This approach allows for the simultaneous and seamless integration of both sensory inputs, offering richer spatial information and facilitating better reasoning about robot actions. The method, trained in a simulated environment and then deployed to a real robot, is applicable to various in-hand object rotation tasks. Comprehensive ablations are performed on how the integration of vision and touch can improve reinforcement learning and Sim2Real performance. Our project page is available at https://yingyuan0414.github.io/visuotactile/ .
@article{arxiv.2312.01853,
title = {Robot Synesthesia: In-Hand Manipulation with Visuotactile Sensing},
author = {Ying Yuan and Haichuan Che and Yuzhe Qin and Binghao Huang and Zhao-Heng Yin and Kang-Won Lee and Yi Wu and Soo-Chul Lim and Xiaolong Wang},
journal= {arXiv preprint arXiv:2312.01853},
year = {2024}
}