Pre-training on larger datasets with ever increasing model size is now a proven recipe for increased performance across almost all NLP tasks. A notable exception is information retrieval, where additional pre-training has so far failed to produce convincing results. We show that, with the right pre-training setup, this barrier can be overcome. We demonstrate this by pre-training large bi-encoder models on 1) a recently released set of 65 million synthetically generated questions, and 2) 200 million post-comment pairs from a preexisting dataset of Reddit conversations made available by pushshift.io. We evaluate on a set of information retrieval and dialogue retrieval benchmarks, showing substantial improvements over supervised baselines.
@article{arxiv.2107.13602,
title = {Domain-matched Pre-training Tasks for Dense Retrieval},
author = {Barlas Oğuz and Kushal Lakhotia and Anchit Gupta and Patrick Lewis and Vladimir Karpukhin and Aleksandra Piktus and Xilun Chen and Sebastian Riedel and Wen-tau Yih and Sonal Gupta and Yashar Mehdad},
journal= {arXiv preprint arXiv:2107.13602},
year = {2021}
}