Social media data is now widely used by many academic researchers. However, long-term social media data collection projects, which most typically involve collecting data from public-use APIs, often encounter issues when relying on local-area network servers (LANs) to collect high-volume streaming social media data over long periods of time. In this technical report, we present a cloud-based data collection, pre-processing, and archiving infrastructure, and argue that this system mitigates or resolves the problems most typically encountered when running social media data collection projects on LANs at minimal cloud-computing costs. We show how this approach works in different cloud computing architectures, and how to adapt the method to collect streaming data from other social media platforms.
@article{arxiv.2005.02442,
title = {Reliable and Efficient Long-Term Social Media Monitoring},
author = {Jian Cao and Nicholas Adams-Cohen and R. Michael Alvarez},
journal= {arXiv preprint arXiv:2005.02442},
year = {2020}
}