@inproceedings{87836e063b744972a3e9d3ad8f06fce2,
title = "Online hate ratings vary by extremes: A statistical analysis",
abstract = "Analyzing 5,665 crowd ratings on 1,133 social media comments, we find that individuals tend to agree on the extremes of a hate rating scale more than in the middle when evaluating the hatefulness of online comments. The agreement is higher for less hateful comments and lowest on moderately hateful comments. The results have implications for researchers developing machine learning models for online hate processing, as the extreme classes are likely to require fewer annotations for reaching statistical stability. Our findings suggest that the models developed in this domain should consider the distributions of hate ratings rather than average hate scores.",
keywords = "Crowdsourcing, Interpretation, Online hate, Ratings, Toxicity",
author = "Joni Salminen and Hind Almerekhi and Kamel, {Ahmed Mohamed} and Jung, {Soon Gyo} and Jansen, {Bernard J.}",
note = "Publisher Copyright: {\textcopyright} 2019 Copyright is held by the owner/author(s). Publication rights licensed to ACM.; 4th ACM SIGIR Conference on Information Interaction and Retrieval, CHIIR 2019 ; Conference date: 10-03-2019 Through 14-03-2019",
year = "2019",
month = mar,
day = "8",
doi = "10.1145/3295750.3298954",
language = "English",
series = "CHIIR 2019 - Proceedings of the 2019 Conference on Human Information Interaction and Retrieval",
publisher = "Association for Computing Machinery, Inc",
pages = "213--217",
booktitle = "CHIIR 2019 - Proceedings of the 2019 Conference on Human Information Interaction and Retrieval",
}