@inproceedings{656fba141dae4c8fafb392578bc622fe,
title = "Crowdsourcing Beyond Annotation: Case Studies in Benchmark Data Collection",
abstract = "Crowdsourcing from non-experts is one of the most common approaches to collecting data and annotations in NLP. Even though it is such a fundamental tool in NLP, crowdsourcing use is largely guided by common practices and the personal experience of researchers. Developing a theory of crowdsourcing use for practical language problems remains an open challenge. However, there are various principles and practices that have proven effective in generating high quality and diverse data. This tutorial exposes NLP researchers to such data collection crowdsourcing methods and principles through a detailed discussion of a diverse set of case studies.",
author = "Alane Suhr and Clara Vania and Nikita Nangia and Maarten Sap and Mark Yatskar and Bowman, {Samuel R.} and Yoav Artzi",
note = "Publisher Copyright: {\textcopyright} 2021 Association for Computational Linguistics.; 2021 Conference on Empirical Methods in Natural Language Processing, EMNLP 2021 ; Conference date: 07-11-2021 Through 11-11-2021",
year = "2021",
language = "English (US)",
series = "EMNLP 2021 - 2021 Conference on Empirical Methods in Natural Language Processing: Tutorial Abstracts",
publisher = "Association for Computational Linguistics (ACL)",
pages = "1--6",
booktitle = "EMNLP 2021 - 2021 Conference on Empirical Methods in Natural Language Processing",
}