Current NLP techniques have been greatly applied in different domains. In this paper, we propose a human-in-the-loop framework for robotic grasping in cluttered scenes, investigating a language interface to the grasping process, which allows the user to intervene by natural language commands. This framework is constructed on a state-of-the-art rasping baseline, where we substitute a scene-graph representation with a text representation of the scene using BERT. Experiments on both simulation and physical robot show that the proposed method outperforms conventional object-agnostic and scene-graph based methods in the literature. In addition, we find that with human intervention, performance can be significantly improved.
@article{arxiv.2209.14026,
title = {Human-in-the-loop Robotic Grasping using BERT Scene Representation},
author = {Yaoxian Song and Penglei Sun and Pengfei Fang and Linyi Yang and Yanghua Xiao and Yue Zhang},
journal= {arXiv preprint arXiv:2209.14026},
year = {2022}
}