We propose a novel framework ConceptX, to analyze how latent concepts are encoded in representations learned within pre-trained language models. It uses clustering to discover the encoded concepts and explains them by aligning with a large set of human-defined concepts. Our analysis on seven transformer language models reveal interesting insights: i) the latent space within the learned representations overlap with different linguistic concepts to a varying degree, ii) the lower layers in the model are dominated by lexical concepts (e.g., affixation), whereas the core-linguistic concepts (e.g., morphological or syntactic relations) are better represented in the middle and higher layers, iii) some encoded concepts are multi-faceted and cannot be adequately explained using the existing human-defined concepts.
@article{arxiv.2206.13289,
title = {Analyzing Encoded Concepts in Transformer Language Models},
author = {Hassan Sajjad and Nadir Durrani and Fahim Dalvi and Firoj Alam and Abdul Rafae Khan and Jia Xu},
journal= {arXiv preprint arXiv:2206.13289},
year = {2022}
}