Spatial reasoning is a fundamental aspect of human cognition, enabling intuitive understanding and manipulation of objects in three-dimensional space. While foundation models demonstrate remarkable performance on some benchmarks, they still struggle with 3D reasoning tasks like arranging objects in space according to open-ended language instructions, particularly in dense and physically constrained environments. We introduce LayoutVLM, a framework and scene layout representation that exploits the semantic knowledge of Vision-Language Models (VLMs) and supports differentiable optimization to ensure physical plausibility. LayoutVLM employs VLMs to generate two mutually reinforcing representations from visually marked images, and a self-consistent decoding process to improve VLMs spatial planning. Our experiments show that LayoutVLM addresses the limitations of existing LLM and constraint-based approaches, producing physically plausible 3D layouts better aligned with the semantic intent of input language instructions. We also demonstrate that fine-tuning VLMs with the proposed scene layout representation extracted from existing scene datasets can improve their reasoning performance.
@article{arxiv.2412.02193,
title = {LayoutVLM: Differentiable Optimization of 3D Layout via Vision-Language Models},
author = {Fan-Yun Sun and Weiyu Liu and Siyi Gu and Dylan Lim and Goutam Bhat and Federico Tombari and Manling Li and Nick Haber and Jiajun Wu},
journal= {arXiv preprint arXiv:2412.02193},
year = {2025}
}