We present a hierarchical neural network model called SemText to detect HTML boilerplate based on a novel semantic representation of HTML tags, class names, and text blocks. We train SemText on three published datasets of news webpages and fine-tune it using a small number of development data in CleanEval and GoogleTrends-2017. We show that SemText achieves the state-of-the-art accuracy on these datasets. We then demonstrate the robustness of SemText by showing that it also detects boilerplate effectively on out-of-domain community-based question-answer webpages.
@article{arxiv.2203.04467,
title = {Boilerplate Detection via Semantic Classification of TextBlocks},
author = {Hao Zhang and Jie Wang},
journal= {arXiv preprint arXiv:2203.04467},
year = {2022}
}