Automatic phenotyping is a task of identifying cohorts of patients that match a predefined set of criteria. Phenotyping typically involves classifying long clinical documents that contain thousands of tokens. At the same time, recent state-of-art transformer-based pre-trained language models limit the input to a few hundred tokens (e.g. 512 tokens for BERT). We evaluate several strategies for incorporating pre-trained sentence encoders into document-level representations of clinical text, and find that hierarchical transformers without pre-training are competitive with task pre-trained models.
@article{arxiv.2105.06752,
title = {Classifying Long Clinical Documents with Pre-trained Transformers},
author = {Xin Su and Timothy Miller and Xiyu Ding and Majid Afshar and Dmitriy Dligach},
journal= {arXiv preprint arXiv:2105.06752},
year = {2021}
}