We explore the trade-offs of performing linear algebra using Apache Spark, compared to traditional C and MPI implementations on HPC platforms. Spark is designed for data analytics on cluster computing platforms with access to local disks and is optimized for data-parallel tasks. We examine three widely-used and important matrix factorizations: NMF (for physical plausability), PCA (for its ubiquity) and CX (for data interpretability). We apply these methods to TB-sized problems in particle physics, climate modeling and bioimaging. The data matrices are tall-and-skinny which enable the algorithms to map conveniently into Spark's data-parallel model. We perform scaling experiments on up to 1600 Cray XC40 nodes, describe the sources of slowdowns, and provide tuning guidance to obtain high performance.
@article{arxiv.1607.01335,
title = {Matrix Factorization at Scale: a Comparison of Scientific Data Analytics in Spark and C+MPI Using Three Case Studies},
author = {Alex Gittens and Aditya Devarakonda and Evan Racah and Michael Ringenburg and Lisa Gerhardt and Jey Kottalam and Jialin Liu and Kristyn Maschhoff and Shane Canon and Jatin Chhugani and Pramod Sharma and Jiyan Yang and James Demmel and Jim Harrell and Venkat Krishnamurthy and Michael W. Mahoney and Prabhat},
journal= {arXiv preprint arXiv:1607.01335},
year = {2016}
}