Analyzing large scale networks requires high performance streaming updates of graph representations of these data. Associative arrays are mathematical objects combining properties of spreadsheets, databases, matrices, and graphs, and are well-suited for representing and analyzing streaming network data. The Dynamic Distributed Dimensional Data Model (D4M) library implements associative arrays in a variety of languages (Python, Julia, and Matlab/Octave) and provides a lightweight in-memory database. Associative arrays are designed for block updates. Streaming updates to a large associative array requires a hierarchical implementation to optimize the performance of the memory hierarchy. Running 34,000 instances of a hierarchical D4M associative arrays on 1,100 server nodes on the MIT SuperCloud achieved a sustained update rate of 1,900,000,000 updates per second. This capability allows the MIT SuperCloud to analyze extremely large streaming network data sets.
@article{arxiv.1902.00846,
title = {A Billion Updates per Second Using 30,000 Hierarchical In-Memory D4M Databases},
author = {Jeremy Kepner and Vijay Gadepally and Lauren Milechin and Siddharth Samsi and William Arcand and David Bestor and William Bergeron and Chansup Byun and Matthew Hubbell and Micheal Houle and Micheal Jones and Anne Klein and Peter Michaleas and Julie Mullen and Andrew Prout and Antonio Rosa and Charles Yee and Albert Reuther},
journal= {arXiv preprint arXiv:1902.00846},
year = {2019}
}