scorecard_templates/data_content_labor.json

{
  "name": "Data and Content Moderation Labor Evaluation",
  "questions": [
    {
      "question": "Crowdwork Standards Compliance",
      "explainer": "Has the system's use of crowdwork been evaluated against established standards?",
      "details": [
        "Assessment of compliance with Criteria for Fairer Microwork",
        "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines",
        "Comparison with Oxford Internet Institute's Fairwork Principles",
        "Documentation of crowdwork role in dataset development",
        "Use of frameworks like CrowdWorkSheets for documentation"
      ]
    },
    {
      "question": "Crowdworker Demographics and Compensation",
      "explainer": "Has information about crowdworkers' demographics and compensation been documented and evaluated?",
      "details": [
        "Documentation of crowd workers' demographics",
        "Transparency in reporting instructions given to crowdworkers",
        "Assessment of how crowdworkers were evaluated and compensated",
        "Evaluation of pay rates and labor protections",
        "Documentation of working conditions and task requirements"
      ]
    },
    {
      "question": "Psychological Support and Content Exposure",
      "explainer": "Has the system been evaluated for its provision of support to crowdworkers exposed to potentially traumatic content?",
      "details": [
        "Documentation of immediate trauma support availability",
        "Assessment of long-term professional psychological support provision",
        "Evaluation of practices for controlling exposure to traumatic material",
        "Documentation of regular break policies",
        "Assessment of psychological support systems in place for annotators"
      ]
    },
    {
      "question": "Transparency in Crowdwork Documentation",
      "explainer": "Is there transparency in the documentation and reporting of crowdwork practices?",
      "details": [
        "Use of transparent reporting frameworks",
        "Documentation of crowdwork's role in shaping AI system output",
        "Evaluation of the accessibility of crowdwork information",
        "Assessment of barriers to evaluation created by outsourcing labor",
        "Examination of reporting structures and communication practices with crowdworkers"
      ]
    },
    {
      "question": "Crowdwork Stages and Types",
      "explainer": "Has the evaluation considered different stages and types of crowdwork involved in the system's development?",
      "details": [
        "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
        "Evaluation of crowdwork during model development and interim evaluations",
        "Examination of post-deployment crowdwork for output evaluation and correction",
        "Documentation of different types of tasks performed by crowdworkers",
        "Analysis of the impact of crowdwork on various stages of system development"
      ]
    },
    {
      "question": "Evaluation of Labor Protection and Regulations",
      "explainer": "Has the evaluation considered applicable labor laws and protections for crowdworkers?",
      "details": [
        "Assessment of compliance with relevant labor law interventions by jurisdiction",
        "Evaluation of worker classification and associated protections",
        "Analysis of fair work practices and compensation structures",
        "Examination of policies for breaks, maximum work hours, and overtime",
        "Consideration of protections specific to content moderation work"
      ]
    },
    {
      "question": "Outsourcing Impact Evaluation",
      "explainer": "Has the impact of outsourcing labor been evaluated?",
      "details": [
        "Assessment of communication barriers created by outsourcing",
        "Evaluation of differences in working conditions between in-house and outsourced labor",
        "Analysis of transparency in reporting structures for outsourced work",
        "Examination of quality control measures for outsourced tasks",
        "Consideration of cultural and linguistic challenges in outsourced content moderation"
      ]
    },
    {
      "question": "Impact of Precarious Employment",
      "explainer": "Does the evaluation consider how precarious employment conditions affect crowdworkers' ability to report issues and overall work quality?",
      "details": [
        "Assessment of job security and its impact on worker feedback",
        "Evaluation of anonymous reporting systems for substandard working conditions",
        "Analysis of power dynamics between crowdworkers and employers",
        "Consideration of the long-term effects of precarious employment on data quality and worker well-being"
      ]
    }
  ]
}