scarecrow.yml

template_name: scarecrow
template_label: SCARECROW 🎃
template_description: "Scarecrow: A framework for scrutinizing machine text"

# ========================================================================================
#
# See https://yao-dou.github.io for more details
#
# ========================================================================================

interface_text:
  typology:
    source_label: "Prompt (human-written)"
    target_label: "Continuation"

tutorial_link: https://yao-dou.github.io/qual/

font_size:
  target: 18

edits:
  - name: grammar_usage
    label: "Grammar and Usage"
    color: "#9a9a9a"
    icon: fa-spell-check
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: grammar_usage_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: bad_math
    label: "Bad Math"
    color: "#111"
    icon: fa-calculator
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: bad_math_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: commonsense
    label: "Commonsense"
    color: "#4ecc9e"
    icon: fa-brain
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: commonsense_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: encyclopedic
    label: "Encyclopedic"
    color: "#137752"
    icon: fa-book
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: encyclopedic_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: incoherent
    label: "Incoherent"
    color: "#d93898"
    icon: fa-comment-slash
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: incoherent_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: needs_google
    label: "Needs Google"
    color: "#5e2ca5"
    icon: fa-magnifying-glass
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: needs_google_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: off_prompt
    label: "Off-prompt"
    color: "gold"
    icon: fa-x
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: off_prompt_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: redundant
    label: "Redundant"
    color: "#e7040f"
    icon: fa-arrows-spin
    enable_output: true
    type: multi_span
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: redundant_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: self_contradiction
    label: "Self-Contradiction"
    color: "#00449e"
    icon: fa-rotate-left
    enable_output: true
    type: multi_span
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: self_contradiction_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
  - name: technical_jargon
    label: "Technical Jargon"
    color: "#ff6300"
    icon: fa-flask
    enable_output: true
    annotation:
      - name: explanation
        label: "Explanation"
        question: "Explain your selection."
        options: textarea
      - name: technical_jargon_type
        label: "Severity"
        question: "Rate the severity."
        options: likert-3
paper_link: https://aclanthology.org/2022.acl-long.501/
citation: |
  @inproceedings{dou-etal-2022-gpt,
      title = "Is {GPT}-3 Text Indistinguishable from Human Text? Scarecrow: A Framework for Scrutinizing Machine Text",
      author = "Dou, Yao  and
        Forbes, Maxwell  and
        Koncel-Kedziorski, Rik  and
        Smith, Noah A.  and
        Choi, Yejin",
      booktitle = "Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
      month = may,
      year = "2022",
      address = "Dublin, Ireland",
      publisher = "Association for Computational Linguistics",
      url = "https://aclanthology.org/2022.acl-long.501",
      doi = "10.18653/v1/2022.acl-long.501",
      pages = "7250--7274",
  }