Autonomous robotic manipulation in clutter is challenging. A large variety of objects must be perceived in complex scenes, where they are partially occluded and embedded among many distractors, often in restricted spaces. To tackle these challenges, we developed a deep-learning approach that combines object detection and semantic segmentation. The manipulation scenes are captured with RGB-D cameras, for which we developed a depth fusion method. Employing pretrained features makes learning from small annotated robotic data sets possible. We evaluate our approach on two challenging data sets: one captured for the Amazon Picking Challenge 2016, where our team NimbRo came in second in the Stowing and third in the Picking task, and one captured in disaster-response scenarios. The experiments show that object detection and semantic segmentation complement each other and can be combined to yield reliable object perception.
@article{arxiv.1810.00818,
title = {RGB-D Object Detection and Semantic Segmentation for Autonomous Manipulation in Clutter},
author = {Max Schwarz and Anton Milan and Arul Selvam Periyasamy and Sven Behnke},
journal= {arXiv preprint arXiv:1810.00818},
year = {2018}
}