While spatial reasoning has made progress in object localization relationships, it often overlooks object orientation-a key factor in 6-DoF fine-grained manipulation. Traditional pose representations rely on pre-defined frames or templates, limiting generalization and semantic grounding. In this paper, we introduce the concept of semantic orientation, which defines object orientations using natural language in a reference-frame-free manner (e.g., the "plug-in" direction of a USB or the "handle" direction of a cup). To support this, we construct OrienText300K, a large-scale dataset of 3D objects annotated with semantic orientations, and develop PointSO, a general model for zero-shot semantic orientation prediction. By integrating semantic orientation into VLM agents, our SoFar framework enables 6-DoF spatial reasoning and generates robotic actions. Extensive experiments demonstrated the effectiveness and generalization of our SoFar, e.g., zero-shot 48.7% successful rate on Open6DOR and zero-shot 74.9% successful rate on SIMPLER-Env.
@article{arxiv.2502.13143,
title = {SoFar: Language-Grounded Orientation Bridges Spatial Reasoning and Object Manipulation},
author = {Zekun Qi and Wenyao Zhang and Yufei Ding and Runpei Dong and Xinqiang Yu and Jingwen Li and Lingyun Xu and Baoyu Li and Xialin He and Guofan Fan and Jiazhao Zhang and Jiawei He and Jiayuan Gu and Xin Jin and Kaisheng Ma and Zhizheng Zhang and He Wang and Li Yi},
journal= {arXiv preprint arXiv:2502.13143},
year = {2025}
}