While AI shows promise for enhancing the efficiency of qualitative analysis, the unique human-AI interaction resulting from varied coding strategies makes it challenging to develop a trustworthy AI-assisted qualitative coding system (AIQCs) that supports coding tasks effectively. We bridge this gap by exploring the impact of varying coding strategies on user trust and reliance on AI. We conducted a mixed-methods split-plot 3x3 study, involving 30 participants, and a follow-up study with 6 participants, exploring varying text selection and code length in the use of our AIQCs system for qualitative analysis. Our results indicate that qualitative open coding should be conceptualized as a series of distinct subtasks, each with differing levels of complexity, and therefore, should be given tailored design considerations. We further observed a discrepancy between perceived and behavioral measures, and emphasized the potential challenges of under- and over-reliance on AIQCs systems. Additional design implications were also proposed for consideration.
@article{arxiv.2309.13858,
title = {Impact of Human-AI Interaction on User Trust and Reliance in AI-Assisted Qualitative Coding},
author = {Jie Gao and Junming Cao and ShunYi Yeo and Kenny Tsu Wei Choo and Zheng Zhang and Toby Jia-Jun Li and Shengdong Zhao and Simon Tangi Perrault},
journal= {arXiv preprint arXiv:2309.13858},
year = {2023}
}