HumanVsAIDetection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "56617d15aca7469ea79ffcedeb05f9f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6c9ea7faf3d04f448e83d2f8238fa155",
              "IPY_MODEL_1f39f3c2a1a04acea017d9264485a254",
              "IPY_MODEL_d0f082d399e34b64954a1fd252f3e438"
            ],
            "layout": "IPY_MODEL_c2b213cb87a449099a8e548d537555ea"
          }
        },
        "6c9ea7faf3d04f448e83d2f8238fa155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be0916b6b99f40749fac45a136578b34",
            "placeholder": "​",
            "style": "IPY_MODEL_f324b812700244d997252bb1562112e5",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "1f39f3c2a1a04acea017d9264485a254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c661cbafac5840a1b3b3cb856ee46fb6",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_61c2e4d651634f44a632d9e800a9dc33",
            "value": 231508
          }
        },
        "d0f082d399e34b64954a1fd252f3e438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5c953dc6b6b453db3f863730000a7e9",
            "placeholder": "​",
            "style": "IPY_MODEL_6c2fa603658f4026a93d90025665c733",
            "value": " 232k/232k [00:00&lt;00:00, 336kB/s]"
          }
        },
        "c2b213cb87a449099a8e548d537555ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be0916b6b99f40749fac45a136578b34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f324b812700244d997252bb1562112e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c661cbafac5840a1b3b3cb856ee46fb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61c2e4d651634f44a632d9e800a9dc33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a5c953dc6b6b453db3f863730000a7e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c2fa603658f4026a93d90025665c733": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f254439486a4a9d8a1c2a6181d9743c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_59e460f8e26946d1979daf91e316b33d",
              "IPY_MODEL_05a03b418572439db592e9e1f45a1b2d",
              "IPY_MODEL_7aec85f7b73e449d877639abc9bcd259"
            ],
            "layout": "IPY_MODEL_1c483bdcc4ed4f3abbe0127d9c3d26a3"
          }
        },
        "59e460f8e26946d1979daf91e316b33d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6c73102593e54eec91fc6903a81e904d",
            "placeholder": "​",
            "style": "IPY_MODEL_9eed9a12a00e4a99a97fd6b87a9c6344",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "05a03b418572439db592e9e1f45a1b2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a0db0cb73da14fd9bad0d9c71c3d5680",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_83244c80a1944beba1e3252284a04990",
            "value": 28
          }
        },
        "7aec85f7b73e449d877639abc9bcd259": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e97decd89f704897979f39953f6ec00b",
            "placeholder": "​",
            "style": "IPY_MODEL_7e90e0c6467146a2aeee5461166d042a",
            "value": " 28.0/28.0 [00:00&lt;00:00, 729B/s]"
          }
        },
        "1c483bdcc4ed4f3abbe0127d9c3d26a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c73102593e54eec91fc6903a81e904d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9eed9a12a00e4a99a97fd6b87a9c6344": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a0db0cb73da14fd9bad0d9c71c3d5680": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "83244c80a1944beba1e3252284a04990": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e97decd89f704897979f39953f6ec00b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e90e0c6467146a2aeee5461166d042a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d38ff3fc63e04eb4a14f846ae59db0ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_97d0cfefec364924ad86f763d28fb6ae",
              "IPY_MODEL_13ea37de85cd4863be7ca3f559cbfac6",
              "IPY_MODEL_1e0283482e124cd692aacd2618539642"
            ],
            "layout": "IPY_MODEL_40384a0b8eff4973a7315f6421e42e5b"
          }
        },
        "97d0cfefec364924ad86f763d28fb6ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6792ed20d40e49098ae2d0e9c769002b",
            "placeholder": "​",
            "style": "IPY_MODEL_9b1601e2f40a45e5a8aa00cf52949611",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "13ea37de85cd4863be7ca3f559cbfac6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a94a3e058bfd4dad880695cb334d0fb6",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cbb8cda271c846bba3fa5054aeffdede",
            "value": 570
          }
        },
        "1e0283482e124cd692aacd2618539642": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d13cc133a89243f38e731afa319092b4",
            "placeholder": "​",
            "style": "IPY_MODEL_3fbc7e50e0544a66828dccff0ba5c7f3",
            "value": " 570/570 [00:00&lt;00:00, 7.92kB/s]"
          }
        },
        "40384a0b8eff4973a7315f6421e42e5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6792ed20d40e49098ae2d0e9c769002b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b1601e2f40a45e5a8aa00cf52949611": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a94a3e058bfd4dad880695cb334d0fb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbb8cda271c846bba3fa5054aeffdede": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d13cc133a89243f38e731afa319092b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fbc7e50e0544a66828dccff0ba5c7f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4779b463a9bf4c25ab04954f6a9876b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_035165f9cd73413ea54cc3dc2df16972",
              "IPY_MODEL_ea8f1b3fc63e43ac9e9cc189a2b7cade",
              "IPY_MODEL_c53fde7439a44cb19cbce6eea6aacc02"
            ],
            "layout": "IPY_MODEL_e37718912ff942cdb22ea1ba19559aae"
          }
        },
        "035165f9cd73413ea54cc3dc2df16972": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0871655c55f5444085e331e93180918e",
            "placeholder": "​",
            "style": "IPY_MODEL_bf9b955e14bc43f78f5d7d3839c973fb",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "ea8f1b3fc63e43ac9e9cc189a2b7cade": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64da70f9e7ae4594a06fc569701656d2",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_277b1f68991941569edff378ee0b63e7",
            "value": 440473133
          }
        },
        "c53fde7439a44cb19cbce6eea6aacc02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e80b11ba888741af88893e19ab8dbe89",
            "placeholder": "​",
            "style": "IPY_MODEL_ddc839e3a41b477d998621f8375d162f",
            "value": " 440M/440M [00:02&lt;00:00, 122MB/s]"
          }
        },
        "e37718912ff942cdb22ea1ba19559aae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0871655c55f5444085e331e93180918e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf9b955e14bc43f78f5d7d3839c973fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64da70f9e7ae4594a06fc569701656d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "277b1f68991941569edff378ee0b63e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e80b11ba888741af88893e19ab8dbe89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ddc839e3a41b477d998621f8375d162f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "UD7yvkwgy82S",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16ecc85f-b4e8-4f7d-ab4a-7e3a10f06dce"
      },
      "source": [
        "import random\n",
        "random.seed(42)\n",
        "\n",
        "from sklearn.metrics import cohen_kappa_score\n",
        "\n",
        "import pandas as pd \n",
        "df1 = pd.read_excel('/content/annotator1.xlsx')\n",
        "df2 = pd.read_excel('/content/annotator2.xlsx')\n",
        "\n",
        "#Creating a contingency table\n",
        "cont_table = pd.crosstab(df1['label'], df2['label'])\n",
        "\n",
        "# calculate the percentage agreement\n",
        "total_instances = cont_table.values.sum() #Getting the total number of entries in the dataset\n",
        "agreement_instances = sum(cont_table[i][i] for i in cont_table.index) #Taking the diagnol entries which corresponds to agreed labels\n",
        "raw_agreement = (agreement_instances / total_instances) * 100 #Calcultaing the percentage agreement \n",
        "\n",
        "print(f\"Raw Percent Agreement: {raw_agreement:.2f}%\")\n",
        "\n",
        "print(f\"Cohens Kappa Score: {cohen_kappa_score(df1['label'].values, df2['label'].values)}\") #Using the sklearn implementation of Cohen's Kappa"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Raw Percent Agreement: 73.33%\n",
            "Cohens Kappa Score: 0.19463087248322153\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N25dvF4jvYoy"
      },
      "source": [
        "Now we'll move onto fine-tuning  pretrained language models specifically on your dataset. Since we're dealing with large models, the first step is to change to a GPU runtime.\n",
        "\n",
        "## Adding a hardware accelerator\n",
        "\n",
        "Please go to the menu and add a GPU as follows:\n",
        "\n",
        "`Edit > Notebook Settings > Hardware accelerator > (GPU)`\n",
        "\n",
        "Run the following cell to confirm that the GPU is detected."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edOh9ooiIW1B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cfb35bc4-8d2a-4fef-ce59-bce9d08c3e71"
      },
      "source": [
        "import torch\n",
        "\n",
        "# Confirm that the GPU is detected\n",
        "\n",
        "assert torch.cuda.is_available()\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = torch.cuda.get_device_name()\n",
        "n_gpu = torch.cuda.device_count()\n",
        "print(f\"Found device: {device_name}, n_gpu: {n_gpu}\")\n",
        "device = torch.device(\"cuda\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found device: Tesla T4, n_gpu: 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xrvH7xx9LnMC"
      },
      "source": [
        "## Installing Hugging Face's Transformers library\n",
        "We will use Hugging Face's Transformers (https://github.com/huggingface/transformers), an open-source library that provides general-purpose architectures for natural language understanding and generation with a collection of various pretrained models made by the NLP community. This library will allow us to easily use pretrained models like `BERT` and perform experiments on top of them. We can use these models to solve downstream target tasks, such as text classification, question answering, and sequence labeling.\n",
        "\n",
        "Note that you will be asked to link with your Google Drive account to download some of these files. If you're concerned about security risks (there have not been any issues in previous semesters), feel free to make a new Google account and use it!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gtqS2e5fxpqa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9c6ea5ab-baa8-4347-b00f-162964505820"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install -U -q PyDrive\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "# Authenticate and create the PyDrive client.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)\n",
        "print('success!')\n",
        "\n",
        "import os\n",
        "import zipfile\n",
        "\n",
        "# Download helper functions file\n",
        "helper_file = drive.CreateFile({'id': '16HW-z9Y1tM3gZ_vFpJAuwUDohz91Aac-'})\n",
        "helper_file.GetContentFile('helpers.py')\n",
        "print('helper file downloaded! (helpers.py)')\n",
        "\n",
        "# Download sample file of tweets\n",
        "data_file = drive.CreateFile({'id': '1QcoAmjOYRtsMX7njjQTYooIbJHPc6Ese'})\n",
        "data_file.GetContentFile('tweets.csv')\n",
        "print('sample tweets downloaded! (tweets.csv)')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.27.4-py3-none-any.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m90.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m25.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m105.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.7)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.13.3 tokenizers-0.13.2 transformers-4.27.4\n",
            "success!\n",
            "helper file downloaded! (helpers.py)\n",
            "sample tweets downloaded! (tweets.csv)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Taseb33Sovg0"
      },
      "source": [
        "from helpers import tokenize_and_format, flat_accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGhkeLQlNNr8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 252,
          "referenced_widgets": [
            "56617d15aca7469ea79ffcedeb05f9f5",
            "6c9ea7faf3d04f448e83d2f8238fa155",
            "1f39f3c2a1a04acea017d9264485a254",
            "d0f082d399e34b64954a1fd252f3e438",
            "c2b213cb87a449099a8e548d537555ea",
            "be0916b6b99f40749fac45a136578b34",
            "f324b812700244d997252bb1562112e5",
            "c661cbafac5840a1b3b3cb856ee46fb6",
            "61c2e4d651634f44a632d9e800a9dc33",
            "a5c953dc6b6b453db3f863730000a7e9",
            "6c2fa603658f4026a93d90025665c733",
            "0f254439486a4a9d8a1c2a6181d9743c",
            "59e460f8e26946d1979daf91e316b33d",
            "05a03b418572439db592e9e1f45a1b2d",
            "7aec85f7b73e449d877639abc9bcd259",
            "1c483bdcc4ed4f3abbe0127d9c3d26a3",
            "6c73102593e54eec91fc6903a81e904d",
            "9eed9a12a00e4a99a97fd6b87a9c6344",
            "a0db0cb73da14fd9bad0d9c71c3d5680",
            "83244c80a1944beba1e3252284a04990",
            "e97decd89f704897979f39953f6ec00b",
            "7e90e0c6467146a2aeee5461166d042a",
            "d38ff3fc63e04eb4a14f846ae59db0ad",
            "97d0cfefec364924ad86f763d28fb6ae",
            "13ea37de85cd4863be7ca3f559cbfac6",
            "1e0283482e124cd692aacd2618539642",
            "40384a0b8eff4973a7315f6421e42e5b",
            "6792ed20d40e49098ae2d0e9c769002b",
            "9b1601e2f40a45e5a8aa00cf52949611",
            "a94a3e058bfd4dad880695cb334d0fb6",
            "cbb8cda271c846bba3fa5054aeffdede",
            "d13cc133a89243f38e731afa319092b4",
            "3fbc7e50e0544a66828dccff0ba5c7f3"
          ]
        },
        "outputId": "b8a1e00e-0a39-49ae-dc13-7823f3d10763"
      },
      "source": [
        "from helpers import tokenize_and_format, flat_accuracy\n",
        "import pandas as pd\n",
        "\n",
        "df = pd.read_excel('final_data.xlsx')\n",
        "\n",
        "df = df.sample(frac=1).reset_index(drop=True)\n",
        "\n",
        "texts = df.text.values\n",
        "labels = df.label.values\n",
        "\n",
        "### tokenize_and_format() is a helper function provided in helpers.py ###\n",
        "input_ids, attention_masks = tokenize_and_format(texts)\n",
        "\n",
        "# Convert the lists into tensors.\n",
        "input_ids = torch.cat(input_ids, dim=0)\n",
        "attention_masks = torch.cat(attention_masks, dim=0)\n",
        "labels = torch.tensor(labels)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', texts[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "56617d15aca7469ea79ffcedeb05f9f5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f254439486a4a9d8a1c2a6181d9743c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d38ff3fc63e04eb4a14f846ae59db0ad"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Original:  if 2020 was a nose it’d be mine 🐦\n",
            "Token IDs: tensor([  101,  2065, 12609,  2001,  1037,  4451,  2009,  1521,  1040,  2022,\n",
            "         3067,   100,   102,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
            "            0,     0,     0,     0])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H3D-CzQEUXYz"
      },
      "source": [
        "## Create train/test/validation splits\n",
        "\n",
        "Here we split your dataset into 3 parts: a training set, a validation set, and a testing set. Each item in your dataset will be a 3-tuple containing an input_id tensor, an attention_mask tensor, and a label tensor.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGgeZ3M0UWs0"
      },
      "source": [
        "\n",
        "total = len(df)\n",
        "\n",
        "num_train = int(total * .8)\n",
        "num_val = int(total * .1)\n",
        "num_test = total - num_train - num_val\n",
        "\n",
        "# make lists of 3-tuples (already shuffled the dataframe in cell above)\n",
        "\n",
        "train_set = [(input_ids[i], attention_masks[i], labels[i]) for i in range(num_train)]\n",
        "val_set = [(input_ids[i], attention_masks[i], labels[i]) for i in range(num_train, num_val+num_train)]\n",
        "test_set = [(input_ids[i], attention_masks[i], labels[i]) for i in range(num_val + num_train, total)]\n",
        "\n",
        "train_text = [texts[i] for i in range(num_train)]\n",
        "val_text = [texts[i] for i in range(num_train, num_val+num_train)]\n",
        "test_text = [texts[i] for i in range(num_val + num_train, total)]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lPo640_ZlEPK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "4779b463a9bf4c25ab04954f6a9876b3",
            "035165f9cd73413ea54cc3dc2df16972",
            "ea8f1b3fc63e43ac9e9cc189a2b7cade",
            "c53fde7439a44cb19cbce6eea6aacc02",
            "e37718912ff942cdb22ea1ba19559aae",
            "0871655c55f5444085e331e93180918e",
            "bf9b955e14bc43f78f5d7d3839c973fb",
            "64da70f9e7ae4594a06fc569701656d2",
            "277b1f68991941569edff378ee0b63e7",
            "e80b11ba888741af88893e19ab8dbe89",
            "ddc839e3a41b477d998621f8375d162f"
          ]
        },
        "outputId": "20de6272-473a-4cc7-aced-484fa17bb4e0"
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4779b463a9bf4c25ab04954f6a9876b3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dd2JdC6IletV"
      },
      "source": [
        "batch_size = [8,16,32]\n",
        "learning_rate = [2e-5, 5e-5, 2e-4, 5e-4, 5e-3]\n",
        "weight_decay = [2e-3,5e-3,1e-2,0.1]\n",
        "epochs = [5,10,15,20]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_Mzr-kd5RaY"
      },
      "source": [
        "import numpy as np\n",
        "# function to get validation accuracy\n",
        "def get_validation_performance(val_set, b_size):\n",
        "    # Put the model in evaluation mode\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "\n",
        "    num_batches = int(len(val_set)/b_size) + 1\n",
        "\n",
        "    total_correct = 0\n",
        "\n",
        "    for i in range(num_batches):\n",
        "\n",
        "      end_index = min(b_size * (i+1), len(val_set))\n",
        "\n",
        "      batch = val_set[i*b_size:end_index]\n",
        "      \n",
        "      if len(batch) == 0: continue\n",
        "\n",
        "      input_id_tensors = torch.stack([data[0] for data in batch])\n",
        "      input_mask_tensors = torch.stack([data[1] for data in batch])\n",
        "      label_tensors = torch.stack([data[2] for data in batch])\n",
        "      \n",
        "      # Move tensors to the GPU\n",
        "      b_input_ids = input_id_tensors.to(device)\n",
        "      b_input_mask = input_mask_tensors.to(device)\n",
        "      b_labels = label_tensors.to(device)\n",
        "        \n",
        "      # Tell pytorch not to bother with constructing the compute graph during\n",
        "      # the forward pass, since this is only needed for backprop (training).\n",
        "      with torch.no_grad():        \n",
        "\n",
        "        # Forward pass, calculate logit predictions.\n",
        "        outputs = model(b_input_ids, \n",
        "                                token_type_ids=None, \n",
        "                                attention_mask=b_input_mask,\n",
        "                                labels=b_labels)\n",
        "        loss = outputs.loss\n",
        "        logits = outputs.logits\n",
        "            \n",
        "        # Accumulate the validation loss.\n",
        "        total_eval_loss += loss.item()\n",
        "        \n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        # Calculate the number of correctly labeled examples in batch\n",
        "        pred_flat = np.argmax(logits, axis=1).flatten()\n",
        "        labels_flat = label_ids.flatten()\n",
        "        num_correct = np.sum(pred_flat == labels_flat)\n",
        "        total_correct += num_correct\n",
        "        \n",
        "    # Report the final accuracy for this validation run.\n",
        "    avg_val_accuracy = total_correct / len(val_set)\n",
        "    return avg_val_accuracy\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HTf_ipbjWNoV"
      },
      "source": [
        "import random\n",
        "\n",
        "performance_track = []\n",
        "for epoc in epochs:\n",
        "  for learning_r in learning_rate:\n",
        "    for b_size in batch_size:\n",
        "      for weight_d in weight_decay:\n",
        "        \n",
        "        model = BertForSequenceClassification.from_pretrained(\n",
        "          \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "          num_labels = 2, # The number of output labels.   \n",
        "          output_attentions = False, # Whether the model returns attentions weights.\n",
        "          output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        "        )\n",
        "        model.cuda()\n",
        "        optimizer = AdamW(model.parameters(),\n",
        "                    lr = learning_r,\n",
        "                    weight_decay = weight_d\n",
        "                  )\n",
        "        # training loop\n",
        "        # For each epoch...\n",
        "        for epoch_i in range(0, epoc):\n",
        "            # Perform one full pass over the training set.\n",
        "\n",
        "            print(\"\")\n",
        "            print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epoc))\n",
        "            print('Training...')\n",
        "\n",
        "            # Reset the total loss for this epoch.\n",
        "            total_train_loss = 0\n",
        "\n",
        "            # Put the model into training mode.\n",
        "            model.train()\n",
        "\n",
        "            # For each batch of training data...\n",
        "            num_batches = int(len(train_set)/b_size) + 1\n",
        "\n",
        "            for i in range(num_batches):\n",
        "              end_index = min(b_size * (i+1), len(train_set))\n",
        "\n",
        "              batch = train_set[i*b_size:end_index]\n",
        "\n",
        "              if len(batch) == 0: continue\n",
        "\n",
        "              input_id_tensors = torch.stack([data[0] for data in batch])\n",
        "              input_mask_tensors = torch.stack([data[1] for data in batch])\n",
        "              label_tensors = torch.stack([data[2] for data in batch])\n",
        "\n",
        "              # Move tensors to the GPU\n",
        "              b_input_ids = input_id_tensors.to(device)\n",
        "              b_input_mask = input_mask_tensors.to(device)\n",
        "              b_labels = label_tensors.to(device)\n",
        "\n",
        "              # Clear the previously calculated gradient\n",
        "              model.zero_grad()        \n",
        "\n",
        "              # Perform a forward pass (evaluate the model on this training batch).\n",
        "              outputs = model(b_input_ids, \n",
        "                                    token_type_ids=None, \n",
        "                                    attention_mask=b_input_mask, \n",
        "                                    labels=b_labels)\n",
        "              loss = outputs.loss\n",
        "              logits = outputs.logits\n",
        "\n",
        "              total_train_loss += loss.item()\n",
        "\n",
        "              # Perform a backward pass to calculate the gradients.\n",
        "              loss.backward()\n",
        "\n",
        "              # Update parameters and take a step using the computed gradient.\n",
        "              optimizer.step()\n",
        "                \n",
        "            # ========================================\n",
        "            #               Validation\n",
        "            # ========================================\n",
        "            # After the completion of each training epoch, measure our performance on\n",
        "            # our validation set. Implement this function in the cell above.\n",
        "            print(f\"Parameters:\")\n",
        "            print(f\"Epochs: {epoc}, Learning rate: {learning_r}, Batch size: {b_size}, Weight Decay: {weight_d}\")\n",
        "            print(f\"Total loss: {total_train_loss}\")\n",
        "            val_acc = get_validation_performance(val_set, b_size)\n",
        "            performance_track.append([epoc, learning_r, b_size, weight_d, total_train_loss, val_acc])\n",
        "            print(f\"Validation accuracy: {val_acc}\")\n",
        "    \n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Training the model with optimal parameter values\n",
        "epochs= 10\n",
        "b_size = 32\n",
        "weight_decay = 0.1\n",
        "learning_rate = 5e-5\n",
        "\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "          \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "          num_labels = 2, # The number of output labels.   \n",
        "          output_attentions = False, # Whether the model returns attentions weights.\n",
        "          output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        "        )\n",
        "model.cuda()\n",
        "optimizer = AdamW(model.parameters(),\n",
        "            lr = learning_rate,\n",
        "            weight_decay = weight_decay\n",
        "          )\n",
        "# training loop\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_train_loss = 0\n",
        "\n",
        "    # Put the model into training mode.\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    num_batches = int(len(train_set)/b_size) + 1\n",
        "\n",
        "    for i in range(num_batches):\n",
        "      end_index = min(b_size * (i+1), len(train_set))\n",
        "\n",
        "      batch = train_set[i*b_size:end_index]\n",
        "\n",
        "      if len(batch) == 0: continue\n",
        "\n",
        "      input_id_tensors = torch.stack([data[0] for data in batch])\n",
        "      input_mask_tensors = torch.stack([data[1] for data in batch])\n",
        "      label_tensors = torch.stack([data[2] for data in batch])\n",
        "\n",
        "      # Move tensors to the GPU\n",
        "      b_input_ids = input_id_tensors.to(device)\n",
        "      b_input_mask = input_mask_tensors.to(device)\n",
        "      b_labels = label_tensors.to(device)\n",
        "\n",
        "      # Clear the previously calculated gradient\n",
        "      model.zero_grad()        \n",
        "\n",
        "      # Perform a forward pass (evaluate the model on this training batch).\n",
        "      outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask, \n",
        "                            labels=b_labels)\n",
        "      loss = outputs.loss\n",
        "      logits = outputs.logits\n",
        "\n",
        "      total_train_loss += loss.item()\n",
        "\n",
        "      # Perform a backward pass to calculate the gradients.\n",
        "      loss.backward()\n",
        "\n",
        "      # Update parameters and take a step using the computed gradient.\n",
        "      optimizer.step()\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set. Implement this function in the cell above.\n",
        "    print(f\"Parameters:\")\n",
        "    print(f\"Total loss: {total_train_loss}\")\n",
        "    val_acc = get_validation_performance(val_set, b_size)\n",
        "    print(f\"Validation accuracy: {val_acc}\")\n",
        "    \n",
        "print(\"\")\n",
        "print(\"Training complete!\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "otdn70jr9oSS",
        "outputId": "0b91863b-5608-4591-ac42-ad8d3be9ca0a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "======== Epoch 1 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 2.269478678703308\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 2 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 1.7473209500312805\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 3 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 1.5795238614082336\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 4 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 1.4203079044818878\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 5 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 1.2445179224014282\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 6 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 0.9792076945304871\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 7 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 0.7272263169288635\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 8 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 0.4920831471681595\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "======== Epoch 9 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 0.3618459478020668\n",
            "Validation accuracy: 0.9166666666666666\n",
            "\n",
            "======== Epoch 10 / 10 ========\n",
            "Training...\n",
            "Parameters:\n",
            "Total loss: 0.26559726893901825\n",
            "Validation accuracy: 0.8333333333333334\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J9DpRJE5mHkO"
      },
      "source": [
        "# Evaluation on the test set\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msvZ78ii3cZZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bc9f04b9-5f07-4a06-a3cf-afb43a6a01ce"
      },
      "source": [
        "get_validation_performance(test_set, b_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9166666666666666"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X72mumhI9WdR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c8a2d943-a841-4660-c248-79e946dd2298"
      },
      "source": [
        "##ERROR ANALYSIS CODE\n",
        "def get_sentence_idx(val_set, b_size):\n",
        "    model.eval()\n",
        "    total_eval_accuracy = 0\n",
        "    total_eval_loss = 0\n",
        "\n",
        "    num_batches = int(len(val_set)/b_size) + 1\n",
        "\n",
        "    total_correct = 0\n",
        "\n",
        "    for i in range(num_batches):\n",
        "\n",
        "      end_index = min(b_size * (i+1), len(val_set))\n",
        "\n",
        "      batch = val_set[i*b_size:end_index]\n",
        "      \n",
        "      if len(batch) == 0: continue\n",
        "\n",
        "      input_id_tensors = torch.stack([data[0] for data in batch])\n",
        "      input_mask_tensors = torch.stack([data[1] for data in batch])\n",
        "      label_tensors = torch.stack([data[2] for data in batch])\n",
        "      \n",
        "      b_input_ids = input_id_tensors.to(device)\n",
        "      b_input_mask = input_mask_tensors.to(device)\n",
        "      b_labels = label_tensors.to(device)\n",
        "        \n",
        "      with torch.no_grad():        \n",
        "\n",
        "        outputs = model(b_input_ids, \n",
        "                                token_type_ids=None, \n",
        "                                attention_mask=b_input_mask,\n",
        "                                labels=b_labels)\n",
        "        logits = outputs.logits\n",
        "            \n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "\n",
        "        pred_flat = np.argmax(logits, axis=1).flatten()\n",
        "        labels_flat = label_ids.flatten()\n",
        "\n",
        "        #Getting the indices of sentences that were predicted incorrectly by the model \n",
        "        sentence_idx = (pred_flat!=labels_flat)\n",
        "    return sentence_idx\n",
        "\n",
        "incorrect_text = []\n",
        "ea_labels = []\n",
        "\n",
        "idx = np.array(get_sentence_idx(test_set, b_size))\n",
        "idx = [int(i) for i in idx]\n",
        "filtered_values = list(filter(lambda x: x[1], zip(test_set, idx)))\n",
        "for i in range(len(input_ids)):\n",
        "  for j in range(len(filtered_values)):\n",
        "    if torch.equal(input_ids[i],filtered_values[j][0][0]):\n",
        "      incorrect_text.append(texts[i])\n",
        "      ea_labels.append(labels[i])\n",
        "\n",
        "incorrect_text.append(\"Wow, thanks for the feedback. I had no idea that my presentation skills were so terrible.#notreally\")\n",
        "ea_labels.append(0)\n",
        "\n",
        "incorrect_text.append(\"Oh great, another meeting. I can't wait to sit through hours of mind-numbing presentations.\")\n",
        "ea_labels.append(0)\n",
        "\n",
        "incorrect_text.append(\"I love sitting in traffic for hours. It gives me so much time to contemplate life's mysteries.\")\n",
        "ea_labels.append(0)\n",
        "\n",
        "incorrect_text.append(\"Oh, I just love filling out paperwork. It's my favorite thing to do on a Friday afternoon.\")\n",
        "ea_labels.append(0)\n",
        "\n",
        "ea_input_ids, ea_attention_masks = tokenize_and_format(incorrect_text)\n",
        "ea_input_ids = torch.cat(ea_input_ids, dim=0)\n",
        "ea_attention_masks = torch.cat(ea_attention_masks, dim=0)\n",
        "ea_labels = torch.tensor(ea_labels)\n",
        "\n",
        "analysis_set = [(ea_input_ids[i], ea_attention_masks[i], ea_labels[i]) for i in range(len(ea_input_ids))]\n",
        "print(f\"Error Analysis performace: {get_validation_performance(analysis_set,len(analysis_set))}\")\n",
        "\n",
        "## print out up to 5 test set examples (or adversarial examples) that your model gets wrong\n",
        "for i in range(len(incorrect_text)):\n",
        "  print(f\"Incorrect text: {incorrect_text[i]}, Actual label: {ea_labels[i]}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error Analysis performace: 0.0\n",
            "Incorrect text: Trying to know all this history tonight is gonna kill me, Actual label: 0\n",
            "Incorrect text: Wow, thanks for the feedback. I had no idea that my presentation skills were so terrible.#notreally, Actual label: 0\n",
            "Incorrect text: Oh great, another meeting. I can't wait to sit through hours of mind-numbing presentations., Actual label: 0\n",
            "Incorrect text: I love sitting in traffic for hours. It gives me so much time to contemplate life's mysteries., Actual label: 0\n",
            "Incorrect text: Oh, I just love filling out paperwork. It's my favorite thing to do on a Friday afternoon., Actual label: 0\n"
          ]
        }
      ]
    }
  ]
}