Images are a convenient way to specify which particular object instance an embodied agent should navigate to. Solving this task requires semantic visual reasoning and exploration of unknown environments. We present a system that can perform this task in both simulation and the real world. Our modular method solves sub-tasks of exploration, goal instance re-identification, goal localization, and local navigation. We re-identify the goal instance in egocentric vision using feature-matching and localize the goal instance by projecting matched features to a map. Each sub-task is solved using off-the-shelf components requiring zero fine-tuning. On the HM3D InstanceImageNav benchmark, this system outperforms a baseline end-to-end RL policy 7x and a state-of-the-art ImageNav model 2.3x (56% vs 25% success). We deploy this system to a mobile robot platform and demonstrate effective real-world performance, achieving an 88% success rate across a home and an office environment.
@article{arxiv.2304.01192,
title = {Navigating to Objects Specified by Images},
author = {Jacob Krantz and Theophile Gervet and Karmesh Yadav and Austin Wang and Chris Paxton and Roozbeh Mottaghi and Dhruv Batra and Jitendra Malik and Stefan Lee and Devendra Singh Chaplot},
journal= {arXiv preprint arXiv:2304.01192},
year = {2023}
}