We introduce RotateIt, a system that enables fingertip-based object rotation along multiple axes by leveraging multimodal sensory inputs. Our system is trained in simulation, where it has access to ground-truth object shapes and physical properties. Then we distill it to operate on realistic yet noisy simulated visuotactile and proprioceptive sensory inputs. These multimodal inputs are fused via a visuotactile transformer, enabling online inference of object shapes and physical properties during deployment. We show significant performance improvements over prior methods and the importance of visual and tactile sensing.
@article{arxiv.2309.09979,
title = {General In-Hand Object Rotation with Vision and Touch},
author = {Haozhi Qi and Brent Yi and Sudharshan Suresh and Mike Lambeta and Yi Ma and Roberto Calandra and Jitendra Malik},
journal= {arXiv preprint arXiv:2309.09979},
year = {2023}
}