In current benchmarks for evaluating large language models (LLMs), there are issues such as evaluation content restriction, untimely updates, and lack of optimization guidance. In this paper, we propose a new paradigm for the measurement of LLMs: Benchmarking-Evaluation-Assessment. Our paradigm shifts the "location" of LLM evaluation from the "examination room" to the "hospital". Through conducting a "physical examination" on LLMs, it utilizes specific task-solving as the evaluation content, performs deep attribution of existing problems within LLMs, and provides recommendation for optimization.
@article{arxiv.2407.07531,
title = {Beyond Benchmarking: A New Paradigm for Evaluation and Assessment of Large Language Models},
author = {Jin Liu and Qingquan Li and Wenlong Du},
journal= {arXiv preprint arXiv:2407.07531},
year = {2024}
}