Most research on data discovery has so far focused on improving individual discovery operators such as join, correlation, or union discovery. However, in practice, a combination of these techniques and their corresponding indexes may be necessary to support arbitrary discovery tasks. We propose BLEND, a comprehensive data discovery system that supports existing operators and enables their flexible pipelining. BLEND is based on a set of lower-level operators that serve as fundamental building blocks for more complex and sophisticated user tasks. To reduce the execution runtime of discovery pipelines, we propose a unified index structure and a rule-based optimizer that rewrites SQL statements into low-level operators when possible. We show the superior flexibility and efficiency of our system compared to ad-hoc discovery pipelines and stand-alone solutions.
@article{arxiv.2310.02656,
title = {Blend: A Unified Data Discovery System},
author = {Mahdi Esmailoghli and Christoph Schnell and Renée J. Miller and Ziawasch Abedjan},
journal= {arXiv preprint arXiv:2310.02656},
year = {2024}
}