@inproceedings{5a31c492902b498fad08a3d9419a782d,
title = "Cloze Distillation: Improving Neural Language Models with Human Next-Word Predictions",
abstract = "Contemporary autoregressive language models (LMs) trained purely on corpus data have been shown to capture numerous features of human incremental processing. However, past work has also suggested dissociations between corpus probabilities and human next-word predictions. Here we evaluate several state-of-the-art language models for their match to human next-word predictions and to reading time behavior from eye movements. We then propose a novel method for distilling the linguistic information implicit in human linguistic predictions into pre-trained LMs: Cloze Distillation. We apply this method to a baseline neural LM and show potential improvement in reading time prediction and generalization to held-out human cloze data.",
author = "Eisape, {Tiwalayo N.} and Noga Zaslavsky and Levy, {Roger P.}",
note = "Publisher Copyright: {\textcopyright} 2020 Association for Computational Linguistics.; 24th Conference on Computational Natural Language Learning, CoNLL 2020 ; Conference date: 19-11-2020 Through 20-11-2020",
year = "2020",
language = "English (US)",
series = "CoNLL 2020 - 24th Conference on Computational Natural Language Learning, Proceedings of the Conference",
publisher = "Association for Computational Linguistics (ACL)",
pages = "609--619",
editor = "Raquel Fernandez and Tal Linzen",
booktitle = "CoNLL 2020 - 24th Conference on Computational Natural Language Learning, Proceedings of the Conference",
}