Large Language Models (LLMs) hold promise in automating data analysis tasks, yet open-source models face significant limitations in these kinds of reasoning-intensive scenarios. In this work, we investigate strategies to enhance the data analysis capabilities of open-source LLMs. By curating a seed dataset of diverse, realistic scenarios, we evaluate model behavior across three core dimensions: data understanding, code generation, and strategic planning. Our analysis reveals three key findings: (1) Strategic planning quality serves as the primary determinant of model performance; (2) Interaction design and task complexity significantly influence reasoning capabilities; (3) Data quality demonstrates a greater impact than diversity in achieving optimal performance. We leverage these insights to develop a data synthesis methodology, demonstrating significant improvements in open-source LLMs' analytical reasoning capabilities. Code is available at https://github.com/zjunlp/DataMind.
@article{arxiv.2506.19794,
title = {Why Do Open-Source LLMs Struggle with Data Analysis? A Systematic Empirical Study},
author = {Yuqi Zhu and Yi Zhong and Jintian Zhang and Ziheng Zhang and Shuofei Qiao and Yujie Luo and Lun Du and Da Zheng and Ningyu Zhang and Huajun Chen},
journal= {arXiv preprint arXiv:2506.19794},
year = {2025}
}