The emergence of pre-trained models has significantly impacted Natural Language Processing (NLP) and Computer Vision to relational datasets. Traditionally, these models are assessed through fine-tuned downstream tasks. However, this raises the question of how to evaluate these models more efficiently and more effectively. In this study, we explore a novel approach where we leverage the meta-features associated with each entity as a source of worldly knowledge and employ entity representations from the models. We propose using the consistency between these representations and the meta-features as a metric for evaluating pre-trained models. Our method's effectiveness is demonstrated across various domains, including models with relational datasets, large language models and image models.
@article{arxiv.2401.02987,
title = {Has Your Pretrained Model Improved? A Multi-head Posterior Based Approach},
author = {Prince Aboagye and Yan Zheng and Junpeng Wang and Uday Singh Saini and Xin Dai and Michael Yeh and Yujie Fan and Zhongfang Zhuang and Shubham Jain and Liang Wang and Wei Zhang},
journal= {arXiv preprint arXiv:2401.02987},
year = {2024}
}