Cloud infrastructure supports the efficient operation of data pipelines regarding requirements like cost, speed, and resource utilization. We present an integrated view of optimization opportunities for cloud-based data pipelines by conducting a systematic review of existing literature on optimization approaches to cloud infrastructure performance for data pipelines. Our study contributes a theory of optimization goals like minimizing cost, reducing execution time, and cost-makespan trade-offs, consisting of dimensions such as single vs. multi-cloud, batch vs. stream processing, etc. We highlight gaps in primary research, including the underexploration of multi-tenant environments and lack of industry evaluation, and suggest directions for future research.
@article{arxiv.2604.01954,
title = {Optimization Opportunities for Cloud-Based Data Pipeline Infrastructures},
author = {Johannes Jablonski and Georg-Daniel Schwarz and Philip Heltweg and Dirk Riehle},
journal= {arXiv preprint arXiv:2604.01954},
year = {2026}
}