We describe an approach to Grammatical Error Correction (GEC) that is effective at making use of models trained on large amounts of weakly supervised bitext. We train the Transformer sequence-to-sequence model on 4B tokens of Wikipedia revisions and employ an iterative decoding strategy that is tailored to the loosely-supervised nature of the Wikipedia training corpus. Finetuning on the Lang-8 corpus and ensembling yields an F0.5 of 58.3 on the CoNLL'14 benchmark and a GLEU of 62.4 on JFLEG. The combination of weakly supervised training and iterative decoding obtains an F0.5 of 48.2 on CoNLL'14 even without using any labeled GEC data.
@article{arxiv.1811.01710,
title = {Weakly Supervised Grammatical Error Correction using Iterative Decoding},
author = {Jared Lichtarge and Christopher Alberti and Shankar Kumar and Noam Shazeer and Niki Parmar},
journal= {arXiv preprint arXiv:1811.01710},
year = {2018}
}