We introduce an approach for selecting objects in neural volumetric 3D representations, such as multi-plane images (MPI) and neural radiance fields (NeRF). Our approach takes a set of foreground and background 2D user scribbles in one view and automatically estimates a 3D segmentation of the desired object, which can be rendered into novel views. To achieve this result, we propose a novel voxel feature embedding that incorporates the neural volumetric 3D representation and multi-view image features from all input views. To evaluate our approach, we introduce a new dataset of human-provided segmentation masks for depicted objects in real-world multi-view scene captures. We show that our approach out-performs strong baselines, including 2D segmentation and 3D segmentation approaches adapted to our task.
@article{arxiv.2205.14929,
title = {Neural Volumetric Object Selection},
author = {Zhongzheng Ren and Aseem Agarwala and Bryan Russell and Alexander G. Schwing and Oliver Wang},
journal= {arXiv preprint arXiv:2205.14929},
year = {2022}
}