Subtask_A.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Subtask-A.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPMb3LEd2XqCxQyQ7bEnv5p",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "09af900110594687b0ba40f73a1367c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b03aa3bbf67148cdb7f96efe2d2d1fe7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b9d179b9c2ab4cf490eec863e443d6da",
              "IPY_MODEL_b6000403f1ad409bbe6a84cc90b09dad"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "b03aa3bbf67148cdb7f96efe2d2d1fe7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "b9d179b9c2ab4cf490eec863e443d6da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5e47cb2e20774c66acab3447914756b9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_53c0afab173e4f80bc5ac6bc8f653ae3"
          },
          "model_module_version": "1.5.0"
        },
        "b6000403f1ad409bbe6a84cc90b09dad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_09eb55d8fcea4b62ba723f6a26ea743b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 326kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9cced980fa99438f9b6ef845dc5f3e65"
          },
          "model_module_version": "1.5.0"
        },
        "5e47cb2e20774c66acab3447914756b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "53c0afab173e4f80bc5ac6bc8f653ae3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "09eb55d8fcea4b62ba723f6a26ea743b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "9cced980fa99438f9b6ef845dc5f3e65": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0541700167e649c795fc0d60a5a3fe2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_241fa49ed65f49ba8e55859c9ea3d6f4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_18aa7d0ff61a44799e021f5cdf191b33",
              "IPY_MODEL_358bdb278c744716a83c23f67eb0855d"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "241fa49ed65f49ba8e55859c9ea3d6f4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "18aa7d0ff61a44799e021f5cdf191b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e4beed34689d4ca893a04b5d0341dab0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9f21e1919d2e4d4a8242d99a8b640aa2"
          },
          "model_module_version": "1.5.0"
        },
        "358bdb278c744716a83c23f67eb0855d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6934129b69de436484cecc848592489f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 167B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_be969f54c8964f8383a4f3313e26c451"
          },
          "model_module_version": "1.5.0"
        },
        "e4beed34689d4ca893a04b5d0341dab0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "9f21e1919d2e4d4a8242d99a8b640aa2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "6934129b69de436484cecc848592489f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "be969f54c8964f8383a4f3313e26c451": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "f9078c55e2264056961e9cadb4ff7944": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1b66b85ceb00428f8a545d7a1f0536c9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5b59068360ab4ad6ade33b4baab0a772",
              "IPY_MODEL_4aa56b3314884a93b07ef3f2294ad30c"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "1b66b85ceb00428f8a545d7a1f0536c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "5b59068360ab4ad6ade33b4baab0a772": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_90cc521647e04fdebe0a9191b269a564",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8141337994944d19d9970f0eb23f93a"
          },
          "model_module_version": "1.5.0"
        },
        "4aa56b3314884a93b07ef3f2294ad30c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2813a49de6c840269024bf7bb617a08a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 3.22MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2492c2b931424ce99c081a5e4cfe5171"
          },
          "model_module_version": "1.5.0"
        },
        "90cc521647e04fdebe0a9191b269a564": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "a8141337994944d19d9970f0eb23f93a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "2813a49de6c840269024bf7bb617a08a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "2492c2b931424ce99c081a5e4cfe5171": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "3a8d41b2efcb4b32a1a26b75816a9267": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5cf4382411af4cf38f5700f238a0139c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_99f277d3ff064c35baf85d0df6a279f7",
              "IPY_MODEL_1d1e9b9b9ff644a1a0cdbcf568523939"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "5cf4382411af4cf38f5700f238a0139c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "99f277d3ff064c35baf85d0df6a279f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8a5f0ca9a73d4439837d22ef2c4ae07f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ba502fb30a64fada2ebdbbb3d0cb6ef"
          },
          "model_module_version": "1.5.0"
        },
        "1d1e9b9b9ff644a1a0cdbcf568523939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f48d5d8790724229894ea13dcbcc77ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 1.42kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02d68df963f04f72b7d9827254ad46f7"
          },
          "model_module_version": "1.5.0"
        },
        "8a5f0ca9a73d4439837d22ef2c4ae07f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "9ba502fb30a64fada2ebdbbb3d0cb6ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "f48d5d8790724229894ea13dcbcc77ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "02d68df963f04f72b7d9827254ad46f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "0f830824bd67448787e67478e00645e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9c68116e95134cbbafbdcc0c79f7c88f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_40161767a79c4fdeb2bb2d9648986b98",
              "IPY_MODEL_de2753c509974303a2e54f407a16dfce"
            ]
          },
          "model_module_version": "1.5.0"
        },
        "9c68116e95134cbbafbdcc0c79f7c88f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "40161767a79c4fdeb2bb2d9648986b98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_47c7e9e711db48339a239b3b89e339a9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c7e67f438d6499b8e55a7279f79d2e6"
          },
          "model_module_version": "1.5.0"
        },
        "de2753c509974303a2e54f407a16dfce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1d58825ca63140f2aaadd03ea0cf532f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:11&lt;00:00, 37.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_455820a6e8ff44d39bda766dcb7cb826"
          },
          "model_module_version": "1.5.0"
        },
        "47c7e9e711db48339a239b3b89e339a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "7c7e67f438d6499b8e55a7279f79d2e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        },
        "1d58825ca63140f2aaadd03ea0cf532f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          },
          "model_module_version": "1.5.0"
        },
        "455820a6e8ff44d39bda766dcb7cb826": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          },
          "model_module_version": "1.2.0"
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rupakdas18/SemEval-2017-Task-4-A-B-C-using-BERT/blob/main/Subtask_A.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V50A_eby71xQ",
        "outputId": "073befc9-3814-4c37-969d-b0a6e235ebab"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)\n",
            "\u001b[K     |████████████████████████████████| 2.1MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 26.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 34.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.5.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fS1BYrdt9gbZ"
      },
      "source": [
        "import random\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180,
          "referenced_widgets": [
            "09af900110594687b0ba40f73a1367c1",
            "b03aa3bbf67148cdb7f96efe2d2d1fe7",
            "b9d179b9c2ab4cf490eec863e443d6da",
            "b6000403f1ad409bbe6a84cc90b09dad",
            "5e47cb2e20774c66acab3447914756b9",
            "53c0afab173e4f80bc5ac6bc8f653ae3",
            "09eb55d8fcea4b62ba723f6a26ea743b",
            "9cced980fa99438f9b6ef845dc5f3e65",
            "0541700167e649c795fc0d60a5a3fe2e",
            "241fa49ed65f49ba8e55859c9ea3d6f4",
            "18aa7d0ff61a44799e021f5cdf191b33",
            "358bdb278c744716a83c23f67eb0855d",
            "e4beed34689d4ca893a04b5d0341dab0",
            "9f21e1919d2e4d4a8242d99a8b640aa2",
            "6934129b69de436484cecc848592489f",
            "be969f54c8964f8383a4f3313e26c451",
            "f9078c55e2264056961e9cadb4ff7944",
            "1b66b85ceb00428f8a545d7a1f0536c9",
            "5b59068360ab4ad6ade33b4baab0a772",
            "4aa56b3314884a93b07ef3f2294ad30c",
            "90cc521647e04fdebe0a9191b269a564",
            "a8141337994944d19d9970f0eb23f93a",
            "2813a49de6c840269024bf7bb617a08a",
            "2492c2b931424ce99c081a5e4cfe5171"
          ]
        },
        "id": "-h1XyGuT78FP",
        "outputId": "597b8a1f-118f-4ade-98e6-a1d192316da2"
      },
      "source": [
        "#Device Selection\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "09af900110594687b0ba40f73a1367c1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0541700167e649c795fc0d60a5a3fe2e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f9078c55e2264056961e9cadb4ff7944",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-6j92ep-tNk"
      },
      "source": [
        "train_data_df = pd.read_csv(\"data/twitter-2016train-A.txt\", delimiter='\\t', header=None, names=['id', 'label', 'tweet'])\n",
        "val_data_df = pd.read_csv(\"data/twitter-2016devtest-A.txt\", delimiter='\\t', header=None, names=['id', 'label', 'tweet'])\n",
        "\n",
        "\n",
        "train_tweet = train_data_df.tweet.values\n",
        "y_train = train_data_df.label.values\n",
        "\n",
        "val_tweet = val_data_df.tweet.values\n",
        "y_val = val_data_df.label.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_4sZYvwHVojK"
      },
      "source": [
        "train_labels=[]\n",
        "val_labels=[]\n",
        "label_dict = {'negative':0, 'neutral':1, 'positive':2}\n",
        "\n",
        "for label in y_train:\n",
        "  train_labels.append(label_dict[label])\n",
        "\n",
        "for label in y_val:\n",
        "  val_labels.append(label_dict[label])\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0ZTtLkLUiUcI",
        "outputId": "6050580b-1d17-4572-e76f-0dce51d8212c"
      },
      "source": [
        "print(len(train_labels))\n",
        "print(len(val_labels))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5868\n",
            "2000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YPuRCatL-7tn"
      },
      "source": [
        "def processdata(tweets,labels):\n",
        "  input_ids = []\n",
        "  attention_masks = []\n",
        "  for tweet in tweets:\n",
        "    encoded_dict = tokenizer.encode_plus(\n",
        "                        tweet,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    input_ids.append(encoded_dict['input_ids'])\n",
        "    attention_masks.append(encoded_dict['attention_mask'])\n",
        "\n",
        "  input_ids = torch.cat(input_ids, dim=0)\n",
        "  attention_masks = torch.cat(attention_masks, dim=0)\n",
        "  labels = torch.tensor(labels)\n",
        "  return input_ids,attention_masks,labels\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4qeqEt_P_QHv",
        "outputId": "d34ade3b-f388-49d3-d8ca-83f3d49e569e"
      },
      "source": [
        "train_input_ids,train_attention_masks,train_labels = processdata(train_tweet,train_labels)\n",
        "val_input_ids,val_attention_masks,val_labels = processdata(val_tweet,val_labels)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zOlq6zd9heG-",
        "outputId": "6575f2aa-907c-4b37-ba81-b522ac3df5ab"
      },
      "source": [
        "print(len(val_input_ids))\n",
        "print(len(val_attention_masks))\n",
        "print(len(val_labels))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2000\n",
            "2000\n",
            "2000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lg7xTMFJB2SF"
      },
      "source": [
        "train_dataset = TensorDataset(train_input_ids, train_attention_masks, train_labels)\n",
        "val_dataset = TensorDataset(val_input_ids, val_attention_masks, val_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zUUifx4wCYDs"
      },
      "source": [
        "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
        "batch_size = 32\n",
        "\n",
        "# Create the DataLoaders for our training and validation sets.\n",
        "train_dataloader = DataLoader(\n",
        "            train_dataset,  # The training samples.\n",
        "            sampler = RandomSampler(train_dataset), # Select batches randomly\n",
        "            batch_size = batch_size # Trains with this batch size.\n",
        "        )\n",
        "\n",
        "# For validation the order doesn't matter, so we'll just read them sequentially.\n",
        "validation_dataloader = DataLoader(\n",
        "            val_dataset, # The validation samples.\n",
        "            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially.\n",
        "            batch_size = batch_size # Evaluate with this batch size.\n",
        "        )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3KKLdEjKKLV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "3a8d41b2efcb4b32a1a26b75816a9267",
            "5cf4382411af4cf38f5700f238a0139c",
            "99f277d3ff064c35baf85d0df6a279f7",
            "1d1e9b9b9ff644a1a0cdbcf568523939",
            "8a5f0ca9a73d4439837d22ef2c4ae07f",
            "9ba502fb30a64fada2ebdbbb3d0cb6ef",
            "f48d5d8790724229894ea13dcbcc77ee",
            "02d68df963f04f72b7d9827254ad46f7",
            "0f830824bd67448787e67478e00645e2",
            "9c68116e95134cbbafbdcc0c79f7c88f",
            "40161767a79c4fdeb2bb2d9648986b98",
            "de2753c509974303a2e54f407a16dfce",
            "47c7e9e711db48339a239b3b89e339a9",
            "7c7e67f438d6499b8e55a7279f79d2e6",
            "1d58825ca63140f2aaadd03ea0cf532f",
            "455820a6e8ff44d39bda766dcb7cb826"
          ]
        },
        "outputId": "3f6d5bdc-767d-4d46-a63e-2891488677ff"
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", \n",
        "    num_labels = 3, # The number of output labels = 3\n",
        "                    \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3a8d41b2efcb4b32a1a26b75816a9267",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0f830824bd67448787e67478e00645e2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6kj7-l14KL6p"
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqEpYxr6KPsD"
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs. The BERT authors recommend between 2 and 4. \n",
        "EPOCHS = 4\n",
        "\n",
        "# Total number of training steps is [number of batches] x [number of epochs]. \n",
        "# (Note that this is not the same as the number of training samples).\n",
        "total_steps = len(train_dataloader) * EPOCHS\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yQ1rRTzUXSe1"
      },
      "source": [
        "def accuracy(y_pred, y_test):\n",
        "  acc = (torch.log_softmax(y_pred, dim=1).argmax(dim=1) == y_test).sum().float() / float(y_test.size(0))\n",
        "  return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UkP_nc3LLOVc"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JaU-c68eLQSu"
      },
      "source": [
        "training_stats=[]\n",
        "def train(model, train_loader, val_loader, optimizer,scheduler):  \n",
        "  total_step = len(train_loader)\n",
        "\n",
        "  for epoch in range(EPOCHS):\n",
        "    # Measure how long the training epoch takes.\n",
        "    train_start = time.time()\n",
        "    model.train()\n",
        "\n",
        "    # Reset the total loss and accuracy for this epoch.\n",
        "    total_train_loss = 0\n",
        "    total_train_acc  = 0\n",
        "    for batch_idx, (pair_token_ids, mask_ids, y) in enumerate(train_loader):\n",
        "\n",
        "      # Unpack this training batch from our dataloader. \n",
        "      pair_token_ids = pair_token_ids.to(device)\n",
        "      mask_ids = mask_ids.to(device)\n",
        "      labels = y.to(device)\n",
        "\n",
        "      #clear any previously calculated gradients before performing a backward pass\n",
        "      optimizer.zero_grad()\n",
        "\n",
        "      #Get the loss and prediction\n",
        "      loss, prediction = model(pair_token_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=mask_ids, \n",
        "                             labels=labels).values()\n",
        "\n",
        "      acc = accuracy(prediction, labels)\n",
        "      \n",
        "      # Accumulate the training loss and accuracy over all of the batches so that we can\n",
        "      # calculate the average loss at the end\n",
        "      total_train_loss += loss.item()\n",
        "      total_train_acc  += acc.item()\n",
        "\n",
        "      # Perform a backward pass to calculate the gradients.\n",
        "      loss.backward()\n",
        "\n",
        "      # Clip the norm of the gradients to 1.0.\n",
        "      # This is to help prevent the \"exploding gradients\" problem.\n",
        "      torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "      # Update parameters and take a step using the computed gradient.\n",
        "      optimizer.step()\n",
        "\n",
        "      # Update the learning rate.\n",
        "      scheduler.step()\n",
        "\n",
        "    # Calculate the average accuracy and loss over all of the batches.\n",
        "    train_acc  = total_train_acc/len(train_loader)\n",
        "    train_loss = total_train_loss/len(train_loader)\n",
        "    train_end = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode\n",
        "    model.eval()\n",
        "\n",
        "    total_val_acc  = 0\n",
        "    total_val_loss = 0\n",
        "    val_start = time.time()\n",
        "    with torch.no_grad():\n",
        "      for batch_idx, (pair_token_ids, mask_ids, y) in enumerate(val_loader):\n",
        "\n",
        "        #clear any previously calculated gradients before performing a backward pass\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        # Unpack this validation batch from our dataloader. \n",
        "        pair_token_ids = pair_token_ids.to(device)\n",
        "        mask_ids = mask_ids.to(device)\n",
        "        labels = y.to(device)\n",
        "        \n",
        "        #Get the loss and prediction\n",
        "        loss, prediction = model(pair_token_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=mask_ids, \n",
        "                             labels=labels).values()\n",
        "\n",
        "        # Calculate the accuracy for this batch\n",
        "        acc = accuracy(prediction, labels)\n",
        "\n",
        "        # Accumulate the validation loss and Accuracy\n",
        "        total_val_loss += loss.item()\n",
        "        total_val_acc  += acc.item()\n",
        "\n",
        "    # Calculate the average accuracy and loss over all of the batches.\n",
        "    val_acc  = total_val_acc/len(val_loader)\n",
        "    val_loss = total_val_loss/len(val_loader)\n",
        "\n",
        "    #end = time.time()\n",
        "    val_end = time.time()\n",
        "    hours, rem = divmod(val_end-train_start, 3600)\n",
        "    minutes, seconds = divmod(rem, 60)\n",
        "\n",
        "    print(f'Epoch {epoch+1}: train_loss: {train_loss:.4f} train_acc: {train_acc:.4f} | val_loss: {val_loss:.4f} val_acc: {val_acc:.4f}')\n",
        "    print(\"{:0>2}:{:0>2}:{:05.2f}\".format(int(hours),int(minutes),seconds))\n",
        "\n",
        "    training_stats.append(\n",
        "        {\n",
        "            'epoch': epoch + 1,\n",
        "            'Training Loss': train_loss,\n",
        "            'Valid. Loss': val_loss,\n",
        "            'Valid. Accur.': val_acc,\n",
        "            'Training Time': train_end-train_start,\n",
        "            'Validation Time': val_end-val_start\n",
        "        }\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XavSM92OLY55",
        "outputId": "b0ce944f-a7a0-48d9-d145-9e358d8360a4"
      },
      "source": [
        "train(model, train_dataloader, validation_dataloader, optimizer,scheduler)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1: train_loss: 0.8247 train_acc: 0.6165 | val_loss: 0.7410 val_acc: 0.6448\n",
            "00:00:43.23\n",
            "Epoch 2: train_loss: 0.6135 train_acc: 0.7301 | val_loss: 0.7844 val_acc: 0.6548\n",
            "00:00:43.02\n",
            "Epoch 3: train_loss: 0.4655 train_acc: 0.8119 | val_loss: 0.8410 val_acc: 0.6523\n",
            "00:00:43.07\n",
            "Epoch 4: train_loss: 0.3579 train_acc: 0.8712 | val_loss: 0.9030 val_acc: 0.6458\n",
            "00:00:43.07\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "vIOqCuhB5UFQ",
        "outputId": "89d8e2af-8ebe-4a67-f692-c468945935d6"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Display floats with two decimal places.\n",
        "pd.set_option('precision', 2)\n",
        "\n",
        "# Create a DataFrame from our training statistics.\n",
        "df_stats = pd.DataFrame(data=training_stats)\n",
        "\n",
        "# Use the 'epoch' as the row index.\n",
        "df_stats = df_stats.set_index('epoch')\n",
        "\n",
        "# A hack to force the column headers to wrap.\n",
        "#df = df.style.set_table_styles([dict(selector=\"th\",props=[('max-width', '70px')])])\n",
        "\n",
        "# Display the table.\n",
        "df_stats"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Valid. Loss</th>\n",
              "      <th>Valid. Accur.</th>\n",
              "      <th>Training Time</th>\n",
              "      <th>Validation Time</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>epoch</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.82</td>\n",
              "      <td>0.74</td>\n",
              "      <td>0.64</td>\n",
              "      <td>39.26</td>\n",
              "      <td>3.97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.61</td>\n",
              "      <td>0.78</td>\n",
              "      <td>0.65</td>\n",
              "      <td>39.05</td>\n",
              "      <td>3.97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.47</td>\n",
              "      <td>0.84</td>\n",
              "      <td>0.65</td>\n",
              "      <td>39.10</td>\n",
              "      <td>3.97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.36</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.65</td>\n",
              "      <td>39.10</td>\n",
              "      <td>3.97</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Training Loss  Valid. Loss  ...  Training Time  Validation Time\n",
              "epoch                              ...                                \n",
              "1               0.82         0.74  ...          39.26             3.97\n",
              "2               0.61         0.78  ...          39.05             3.97\n",
              "3               0.47         0.84  ...          39.10             3.97\n",
              "4               0.36         0.90  ...          39.10             3.97\n",
              "\n",
              "[4 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        },
        "id": "QaVW6f6QMGvd",
        "outputId": "33620aa7-3b2d-40a2-8a6a-a86c398eb145"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "#% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(df_stats['Training Loss'], 'b-o', label=\"Training\")\n",
        "plt.plot(df_stats['Valid. Loss'], 'g-o', label=\"Validation\")\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training & Validation Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend()\n",
        "plt.xticks([1, 2, 3, 4])\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuUAAAGaCAYAAACopj13AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeVxU5f4H8M/MMDMw7MsMqyyCgCIgmLjgvuKaqaS3rmZ799av7dZNb9utbnXzWtlqWbZo5oa4m2m4lGmS+4amuCKyyM6wDMOc3x/AyDigAwFngM/79fJVnDnnzHemjvPhme/zHIkgCAKIiIiIiEg0UrELICIiIiLq7BjKiYiIiIhExlBORERERCQyhnIiIiIiIpExlBMRERERiYyhnIiIiIhIZAzlRNRhZWRkICwsDB999FGzzzFnzhyEhYW1YFUdV2Pvd1hYGObMmWPROT766COEhYUhIyOjxetLTk5GWFgY9u/f3+LnJiL6s2zELoCIOo+mhNuUlBT4+fm1YjXtT1lZGT777DNs2bIFOTk5cHNzQ+/evfH3v/8dwcHBFp3jySefxI8//oh169ahe/fuDe4jCAJGjBiB4uJi7NmzB7a2ti35MlrV/v37kZqaivvuuw9OTk5il2MmIyMDI0aMwL333otXXnlF7HKIyIowlBNRm5k3b57JzwcPHsTKlSsxffp09O7d2+QxNze3P/18vr6+OHbsGGQyWbPP8cYbb+C1117707W0hJdeegmbN2/GhAkTEBcXh9zcXOzYsQNHjx61OJRPmzYNP/74I9asWYOXXnqpwX1+++03XL16FdOnT2+RQH7s2DFIpW3zxWxqaio+/vhj3HXXXWah/M4778T48eMhl8vbpBYioqZgKCeiNnPnnXea/FxdXY2VK1eiV69eZo/drLS0FA4ODk16PolEAqVS2eQ667OWAFdeXo6tW7di4MCBePfdd43bn3jiCeh0OovPM3DgQHh7e2Pjxo345z//CYVCYbZPcnIygJoA3xL+7H+DliKTyf7UL2hERK2JPeVEZHWGDx+OmTNn4tSpU3jwwQfRu3dvTJo0CUBNOH///feRmJiIvn37omfPnhg1ahTmz5+P8vJyk/M01ONcf9vOnTsxdepUREZGYuDAgXjnnXeg1+tNztFQT3ndtpKSErz66qvo378/IiMjMWPGDBw9etTs9RQUFGDu3Lno27cvYmJiMGvWLJw6dQozZ87E8OHDLXpPJBIJJBJJg78kNBSsGyOVSnHXXXehsLAQO3bsMHu8tLQU27ZtQ2hoKKKiopr0fjemoZ5yg8GAzz//HMOHD0dkZCQmTJiADRs2NHh8eno6/v3vf2P8+PGIiYlBdHQ0pkyZgtWrV5vsN2fOHHz88ccAgBEjRiAsLMzkv39jPeX5+fl47bXXMGTIEPTs2RNDhgzBa6+9hoKCApP96o7ft28fFi9ejJEjR6Jnz54YM2YM1q5da9F70RSnT5/G448/jr59+yIyMhLjxo3DF198gerqapP9rl27hrlz52LYsGHo2bMn+vfvjxkzZpjUZDAY8M0332DixImIiYlBbGwsxowZg3/961+oqqpq8dqJqOk4Uk5EVikzMxP33XcfEhISMHr0aJSVlQEAsrOzkZSUhNGjR2PChAmwsbFBamoqvvzyS6SlpWHx4sUWnX/37t34/vvvMWPGDEydOhUpKSn46quv4OzsjMcee8yiczz44INwc3PD448/jsLCQnz99dd45JFHkJKSYhzV1+l0uP/++5GWloYpU6YgMjISZ86cwf333w9nZ2eL3w9bW1tMnjwZa9aswaZNmzBhwgSLj73ZlClTsHDhQiQnJyMhIcHksc2bN6OiogJTp04F0HLv983efvttLFmyBH369MHs2bORl5eH119/HV26dDHbNzU1FQcOHMDQoUPh5+dn/NbgpZdeQn5+Ph599FEAwPTp01FaWort27dj7ty5cHV1BXDruQwlJSX4y1/+gkuXLmHq1Kno0aMH0tLSsHz5cvz2229YvXq12Tc077//PioqKjB9+nQoFAosX74cc+bMgb+/v1kbVnMdP34cM2fOhI2NDe699154eHhg586dmD9/Pk6fPm38tkSv1+P+++9HdnY27rnnHgQGBqK0tBRnzpzBgQMHcNdddwEAFi5ciA8//BDDhg3DjBkzIJPJkJGRgR07dkCn01nNN0JEnZpARCSSNWvWCKGhocKaNWtMtg8bNkwIDQ0VVq1aZXZMZWWloNPpzLa///77QmhoqHD06FHjtitXrgihoaHChx9+aLYtOjpauHLlinG7wWAQxo8fL8THx5uc94UXXhBCQ0Mb3Pbqq6+abN+yZYsQGhoqLF++3Ljtu+++E0JDQ4VPP/3UZN+67cOGDTN7LQ0pKSkRHn74YaFnz55Cjx49hM2bN1t0XGNmzZoldO/eXcjOzjbZfvfddwsRERFCXl6eIAh//v0WBEEIDQ0VXnjhBePP6enpQlhYmDBr1ixBr9cbt584cUIICwsTQkNDTf7baLVas+evrq4W/vrXvwqxsbEm9X344Ydmx9ep+//tt99+M2577733hNDQUOG7774z2bfuv8/7779vdvydd94pVFZWGrdnZWUJERERwjPPPGP2nDere49ee+21W+43ffp0oXv37kJaWppxm8FgEJ588kkhNDRU2Lt3ryAIgpCWliaEhoYKixYtuuX5Jk+eLIwdO/a29RGReNi+QkRWycXFBVOmTDHbrlAojKN6er0eRUVFyM/Px4ABAwCgwfaRhowYMcJkdReJRIK+ffsiNzcXWq3WonPMnj3b5Od+/foBAC5dumTctnPnTshkMsyaNctk38TERDg6Olr0PAaDAU899RROnz6NH374AYMHD8Zzzz2HjRs3muz38ssvIyIiwqIe82nTpqG6uhrr1q0zbktPT8eRI0cwfPhw40Tblnq/60tJSYEgCLj//vtNerwjIiIQHx9vtr9KpTL+e2VlJQoKClBYWIj4+HiUlpbi/PnzTa6hzvbt2+Hm5obp06ebbJ8+fTrc3Nzw008/mR1zzz33mLQMeXp6IigoCBcvXmx2HfXl5eXh8OHDGD58OMLDw43bJRIJ/va3vxnrBmD8f2j//v3Iy8tr9JwODg7Izs7GgQMHWqRGImp5bF8hIqvUpUuXRiflLVu2DCtWrMC5c+dgMBhMHisqKrL4/DdzcXEBABQWFsLe3r7J56hrlygsLDRuy8jIgEajMTufQqGAn58fiouLb/s8KSkp2LNnD/73v//Bz88PH3zwAZ544gn885//hF6vN7YonDlzBpGRkRb1mI8ePRpOTk5ITk7GI488AgBYs2YNABhbV+q0xPtd35UrVwAAXbt2NXssODgYe/bsMdmm1Wrx8ccf44cffsC1a9fMjrHkPWxMRkYGevbsCRsb049DGxsbBAYG4tSpU2bHNPb/ztWrV5tdx801AUBISIjZY127doVUKjW+h76+vnjsscewaNEiDBw4EN27d0e/fv2QkJCAqKgo43HPPvssHn/8cdx7773QaDSIi4vD0KFDMWbMmCbNSSCi1sNQTkRWyc7OrsHtX3/9Nf773/9i4MCBmDVrFjQaDeRyObKzszFnzhwIgmDR+W+1CsefPYelx1uqbmJinz59ANQE+o8//hh/+9vfMHfuXOj1eoSHh+Po0aN48803LTqnUqnEhAkT8P333+PQoUOIjo7Ghg0b4OXlhUGDBhn3a6n3+8/4xz/+gV27duHuu+9Gnz594OLiAplMht27d+Obb74x+0WhtbXV8o6WeuaZZzBt2jTs2rULBw4cQFJSEhYvXoyHHnoIzz//PAAgJiYG27dvx549e7B//37s378fmzZtwsKFC/H9998bfyElIvEwlBNRu7J+/Xr4+vriiy++MAlHP//8s4hVNc7X1xf79u2DVqs1GS2vqqpCRkaGRTe4qXudV69ehbe3N4CaYP7pp5/isccew8svvwxfX1+EhoZi8uTJFtc2bdo0fP/990hOTkZRURFyc3Px2GOPmbyvrfF+1400nz9/Hv7+/iaPpaenm/xcXFyMXbt24c4778Trr79u8tjevXvNzi2RSJpcy4ULF6DX601Gy/V6PS5evNjgqHhrq2urOnfunNlj58+fh8FgMKurS5cumDlzJmbOnInKyko8+OCD+PLLL/HAAw/A3d0dAGBvb48xY8ZgzJgxAGq+AXn99deRlJSEhx56qJVfFRHdjnX9uk9EdBtSqRQSicRkhFav1+OLL74QsarGDR8+HNXV1ViyZInJ9lWrVqGkpMSicwwZMgRAzaof9fvFlUol3nvvPTg5OSEjIwNjxowxa8O4lYiICHTv3h1btmzBsmXLIJFIzNYmb433e/jw4ZBIJPj6669Nlvc7efKkWdCu+0Xg5hH5nJwcsyURgRv955a21YwcORL5+flm51q1ahXy8/MxcuRIi87Tktzd3RETE4OdO3fijz/+MG4XBAGLFi0CAIwaNQpAzeoxNy9pqFQqja1Bde9Dfn6+2fNERESY7ENE4uJIORG1KwkJCXj33Xfx8MMPY9SoUSgtLcWmTZuaFEbbUmJiIlasWIEFCxbg8uXLxiURt27dioCAALN10RsSHx+PadOmISkpCePHj8edd94JLy8vXLlyBevXrwdQE7A++eQTBAcHY+zYsRbXN23aNLzxxhv45ZdfEBcXZzYC2xrvd3BwMO6991589913uO+++zB69Gjk5eVh2bJlCA8PN+njdnBwQHx8PDZs2ABbW1tERkbi6tWrWLlyJfz8/Ez69wEgOjoaADB//nxMnDgRSqUS3bp1Q2hoaIO1PPTQQ9i6dStef/11nDp1Ct27d0daWhqSkpIQFBTUaiPIJ06cwKeffmq23cbGBo888ghefPFFzJw5E/feey/uueceqNVq7Ny5E3v27MGECRPQv39/ADWtTS+//DJGjx6NoKAg2Nvb48SJE0hKSkJ0dLQxnI8bNw69evVCVFQUNBoNcnNzsWrVKsjlcowfP75VXiMRNY11fooRETXiwQcfhCAISEpKwptvvgm1Wo2xY8di6tSpGDdunNjlmVEoFPj2228xb948pKSk4IcffkBUVBS++eYbvPjii6ioqLDoPG+++Sbi4uKwYsUKLF68GFVVVfD19UVCQgIeeOABKBQKTJ8+Hc8//zwcHR0xcOBAi847ceJEzJs3D5WVlWYTPIHWe79ffPFFeHh4YNWqVZg3bx4CAwPxyiuv4NKlS2aTK//3v//h3XffxY4dO7B27VoEBgbimWeegY2NDebOnWuyb+/evfHcc89hxYoVePnll6HX6/HEE080GsodHR2xfPlyfPjhh9ixYweSk5Ph7u6OGTNm4P/+7/+afBdZSx09erTBlWsUCgUeeeQRREZGYsWKFfjwww+xfPlylJWVoUuXLnjuuefwwAMPGPcPCwvDqFGjkJqaio0bN8JgMMDb2xuPPvqoyX4PPPAAdu/ejaVLl6KkpATu7u6Ijo7Go48+arLCCxGJRyK0xSwdIiIyUV1djX79+iEqKqrZN+AhIqKOgz3lREStrKHR8BUrVqC4uLjBdbmJiKjzYfsKEVEre+mll6DT6RATEwOFQoHDhw9j06ZNCAgIwN133y12eUREZAXYvkJE1MrWrVuHZcuW4eLFiygrK4O7uzuGDBmCp556Ch4eHmKXR0REVoChnIiIiIhIZOwpJyIiIiISGUM5EREREZHIONGzVkGBFgZD23byuLs7IC+vtE2fk6g94rVCZBleK0SWEetakUolcHW1b/AxhvJaBoPQ5qG87nmJ6PZ4rRBZhtcKkWWs7Vph+woRERERkcgYyomIiIiIRMZQTkREREQkMoZyIiIiIiKRMZQTEREREYmMoZyIiIiISGQM5UREREREImMoJyIiIiISmaihXKfT4X//+x8GDhyIqKgo3H333di3b59Fx65btw4TJ05EZGQkBg4ciP/85z/QarWtXDERERERUcsT9Y6ec+bMwbZt2zBr1iwEBARg7dq1ePjhh7F06VLExMQ0ety3336Lt956C/Hx8ZgxYways7OxZMkSnD17Ft988w0kEkkbvgoiIiIiag9Ssw5hQ/pWFFYWwkXpgknBCYjzihW7LACARBAEUe4xeuzYMSQmJmLu3LmYPXs2AKCyshITJkyARqPBsmXLGjxOp9NhwIABiIiIMAngO3fuxGOPPYZPPvkEI0eObHI9eXmlbX67VbXaEbm5JW36nETtEa8VIsvwWiFqXGrWIXx/eg2qDFXGbXKpHPeET22zYC6VSuDu7tDwY21SQQO2bt0KuVyOxMRE4zalUolp06bh4MGDyMnJafC4s2fPoqSkBOPGjTMZER82bBhUKhW2bNnS6rUTERERUfugq9bhckkGkv7YYBLIAaDKUIUN6VtFqsyUaO0raWlpCAoKgr29vcn2qKgoCIKAtLQ0aDQas+N0Oh2AmgB/M1tbW5w8ebJ1CiYiIiIiq6Wr1iFLm4Nr2mzjnyxtNvIqCiCg8W6IgsrCNqyycaKF8tzcXHh6epptV6vVANDoSHlAQAAkEgkOHTqEyZMnG7efP38e+fn5qKioaJ2CiYiIiEh0ldU6ZGmzzQJ4fr3wLZPI4KlSw9/JD329e8PL3hNJf6xHkc68vctV6dLWL6FBooXyiooKyOVys+11I+CVlZUNHufm5oaxY8dizZo16Nq1K0aMGIHs7Gy88cYbkMvljR53O43197Q2tdpRlOclam94rRBZhtcKdRQVVRW4WpKNK0WZyCjOQkZRJq4UX0OuNs+4j0wqg4+jJ8LUQfBzHgA/J2/4OXvDy0EDG6nM5HwqBzk+/30ZdNU64zaFTIG/xtxlFdeNaKHc1tYWVVVVZtvrQnVD7Sl1Xn/9dVRUVODtt9/G22+/DQCYNGkS/P39LV5S8Wac6ElkvXitEFmG1wq1RxX6SmSX5SCztt2kfttJHRuJDBqVGv72fuir6Q1ve0942XtCbecO2U3hG5VAQWWZ2fOEq7rjL2FTzFZfCVd1b7Pr5lYTPUUL5Wq1usEWldzcXABosJ+8jqOjIxYuXIjMzExcvXoVPj4+8PX1xYwZMxAQENBqNRMRERFR81ToK5FVlo1r2hxc02YZ20/ybwrfnvYaBDr5o793HLztNfC294RHQ+G7GeK8YhHnFWuVv8CKFsrDw8OxdOlSaLVak8meR48eNT5+Oz4+PvDx8QEAFBcX48SJE8blFYmIiIio7VXoK5BVloNrpdm4Vlbb812abTKhsi58d3UOQLxPHLzsPWvCt61bi4Tv9ki0UJ6QkICvvvoKq1evNgZpnU6H5ORkxMbGGieBZmZmory8HMHBwbc837vvvgupVIrp06e3dulEREREnV65vsI42p1Vb8KlSfiW2sBTpUawSyC8a4O3VycP340RLZRHR0cjISEB8+fPR25uLvz9/bF27VpkZmYa+8QB4IUXXkBqairOnDlj3LZw4UKkp6cjOjoaMpkMKSkp2LNnD15//XV06dJFjJdDRERE1CHVhG/ztpObw7eXSlMbvr1M2k6kEtFui9OuiBbKAWDevHlYsGAB1q9fj6KiIoSFhWHRokXo3bv3LY8LCwtDSkoKUlJSAAARERH44osvMHjw4LYom4iIiKjDKdeXmy0zeE2bjcLKIuM+cqkNPFUahLgEGUe9a8K3G8P3nyQRBKFtlxyxUlx9hch68VohsgyvFbJEub7cbNS7ofDtpdLAq96ot1cHCt9iXStWufoKEREREbWesqry2tVO6pYZzGkgfMvhZa9BN5dg+Nh7wtvBE14qT7jbuXaI8N2eMJQTERERtWNlVeVmky2vabNRpCs27iOXyuFtr0Goa7BxwqW3vSfcbBm+rQVDOREREVE7UFZV1kDbSZbJreMVtSPf4W7d4FXbdsLw3T4wlBMRERFZkbKqMmTWu7Nl3T/Nw7cnwt1CTZYadLN1YfhupxjKiYiIiESgrSozaTepC+DF9cO3TAFvlWn49rb3hCvDd4fDUE5ERETUikqrtLXtJlm17Sc1bSclulLjPnXhu4dbWO1kSw287b3gauvM8N1JMJQTERERtYDSKm3NreW12TWrntTeZr5++FbKFPCy90SEe/iNthOVJ8M3MZQTERERNUWpTmsy6l3XdlJSdSN828qUZuHb294TrkoXSCQSEasna8VQTkRERNSAEl1pvWUG69pPslFapTXuYytTwtveEz09uhsnW/rYe8JF6czwTU3CUE5ERESdWomu1GyypXn4toW3vQZRHj2Mt5b3ZvimFsRQTkRERB2eIAg1Pd8NtJ2Yh29PRHn0qA3eXvCy1zB8U6tjKCciIqIOQxAElFTVtJ3cvNa3tqrMuJ+dTV34joC3gye8VTW3mHdWODF8kygYyomIiKjdEQQBxfV7vmtXO8kqazh891L3NGk7Yfgma8NQTkRERFarLnzfuLV8TftJljYbWn398G1XG74j693hUsPwTe0GQzkRERGJriZ8lzR4h8syfblxP1Vd+NZEmiw16KRwZPimdo2hnIiIiNqMIAgo0hXXjnqbBvCGwnesJso42dLb3gtOCgeGb+qQGMqJiIioxdWF75rAfWON72vaHJTXC9/2Nip42Xsi1jO6ZrJl7VrfDN/U2TCUExERUbMZw3ep6WRLs/AtV8Hb3hO9PaNrWk5qVztxlDN8EwEM5URERGQBQRBQWFlkMtnymrYmgJfrK4z7Ocjt4W3viTs8e9X2e9e0nTjI7Rm+iW6BoZyIiIiM6sL3zXe3vKbNQUW1efju4xljstSgo8JBxOqJ2i+GciIiok6oLnxn1gvfWbcI33FeMfWWGmT4JmppDOVERETtXGrWIWxI34rCykK4KF0wKTgBcV6xAGrCd0FlYW27SZZx1ZMsbTYqqiuN53CUO9SG71hj2wnDN1HbYSgnIiJqx1KzDuH702tQZagCABRUFuK7tNXYc3U/9IIeWdpsVFbrjPs7KhzgrfJEX+/e8FLdaDtxUNiL9RKICAzlRERE7UqFvgLZZbnILstFTlkufrr8szGQ16kWqnG+6CK6uQajn3cf42RLL3sNHOQM30TWiKGciIjIylQbqpFXUYCceuG77p9FuhLjfhJIIEBo8BwCBDwV80hblUxEfxJDORERkUhKdVqTUe+68J1bnodqodq4n72NChqVGt3dwuCpUkNjr4anSg0PO3e8tm8eCioLzc7tqnRpy5dCRH8SQzkREVErqjLokVt2vTZ0XzcJ4Vp9mXE/mUQGtZ07PFVqRHr0gKdKDU97NTQq9S1bTiYFJ5j0lAOAXCrHpOCEVn1dRNSyGMqJiIj+pLq7WtaNdBuDtzYXeRUFJi0mzgpHaFRqxGgia0a9VWp4qjRws3WBTCpr8nPXrbLS2OorRNQ+MJQTERFZqEJfidxy09Huun/WX+FEIZVDo1IjwKkL+njF1oZvD2hUatjZ2LZ4XXFesYjzioVa7Yjc3JLbH0BEVoehnIiIqB6DYEB+RaFJ6K7798LKIuN+EkjgZusCjUqNYO/AeqPeajgrnSCVSEV8FUTU3jCUi2DfySwk705HfnEl3JyUmDIkGP0jvMQui4ioUymrKjNtNak3yVJv0Bv3s7Oxg6dKjTDXEGhqR7w9VWqo7TygkMlFfAVE1JEwlLexfSez8O0Pp6HTGwAAecWV+PaH0wDAYE5E1ML0Bj2ul+c3OOpdWqU17ieVSKG2c4dGpUaEe3ht8NbAs3aSpUQiEfFVEFFnwFDexpJ3pxsDeR2d3oDk3ekM5UREzSAIAop1pcgpy7lpecHruF6RD4Nw4+9cR4UDPFVqRHlEwLN2WUGNSg0PW7dmTbIkImopDOVtLK+4stHtlbpqKBX8UCAiaoiuWmdcUvDGBMuanyuqK4z7yaU20KjU8HX0QawmqqbP214NjZ0aKrmdiK+AiKhxDOVtzN1J2Wgwn7NoH6YODsaASC9I+VUpEXVCBsGAgoqiG6G7PBfZ2pp/v/kGOa5KF3iq1IirXd2kbtTb1daZkyyJqN1hKG9jU4YEm/SUA4DCRoqEvv44fj4fX21JQ8rBDMwYEYIwf1cRKyUiaj3l+vKaVhNtLnLKr5u0nNS/CY6tTAmNSo0QlyCT1U00Kg8oZAoRXwERUctiKG9jdX3jDa2+MmlgEPafykbSrnS88/1h9A5VI3FYMDSuKpGrJiJqumpDNa5X5Jus5V3X812iKzXuJ5VI4W7ralzh5MaotwZOCgdOsiSiTkEiCIJw+906vry8UhgMbftWNHaTh8qqavyYehlbfrsEg0HAyN5dMGFAIFS2/B2KOifeEMV6CYKA0iptg6ub5JbnmUyydJDbG0e66496e9i5wUbKv99aAq8VIsuIda1IpRK4uzs0+Bj/FrRCSrkMk+KDMCjKB8k/p+PH1MvYc/wa7hoUhMG9fCCTsleSiNpWVXUVcsvzzNb0zi7LRbm+3LifjUQGtcoD3vaeiFb3NAng9nJ+60dE1BiOlNeyppHym13KKsHylLP440ohfD3sMX14CHp2dW+DComsA0f/2oYgCCisLGpw1Du/ohACbvwd6aJ0bnDU283WhZMsRcRrhcgy1jhSLmoo1+l0+OCDD7B+/XoUFxcjPDwczzzzDPr373/bY/fu3YuFCxfijz/+gMFgQNeuXXHfffdh3LhxzarFmkM5UPNheeiPXKzaeQ65hRWI7OqO6cND4ONh38pVEomPQaNlVegrjEsJmi4vmAtdvUmWSpmi3sTKGxMsNXZq2NooRXwF1BheK0SWYSi/ybPPPott27Zh1qxZCAgIwNq1a3HixAksXboUMTExjR63c+dO/O1vf0NMTAzGjx8PANi8eTMOHTqE//znP0hMTGxyLdYeyutU6Q1IOZiBjXsvoFJnwNAYH9w5MAiOKq5CQB0Xg0bTGQQD8soLkF2Wc9Oo93UU6YqN+0kggbutKzT29Ua97WrW9XZWOHGSZTvDa4XIMgzl9Rw7dgyJiYmYO3cuZs+eDQCorKzEhAkToNFosGzZskaPfeihh3DmzBmkpKRAoagJozqdDiNGjEBAQAC+++67JtfTXkJ5neIyHdb/cgG7jlyFrcIGk+IDMaK3H2xk/NqYOh4GjcaVVmlrQrfWdNT7enke9KHWWV4AACAASURBVEK1cT97G9VN7SYe0KjUUKs8IOckyw6D1wqRZawxlIv2N/HWrVshl8tNRrWVSiWmTZuG999/Hzk5OdBoNA0eW1paCmdnZ2MgBwCFQgFnZ2colZ3jK1UnlQIzx4RheKwvVu44h5U7zmHn4au4e1gIYrp5cHSLqAOpMuhxvXaSZY623k11ynKhrSoz7ieTyKC2c4enSo1Ijx4mIdxBwVY3IiJrJlooT0tLQ1BQEOztTT8ooqKiIAgC0tLSGg3lcXFx+Pzzz7FgwQJMmTIFAJCcnIyLFy9i7ty5rV67NfFVO+DZ6b1w/HweVu44h4+TjyPc3wXTh3dDgJej2OURkYUEQUCRrtis1SS7LBd55fkmkyydFY7QqNTopY40mWjpbusKmVQm4qsgIqLmEi2U5+bmwtPT02y7Wq0GAOTk5DR67GOPPYbLly/js88+w8KFCwEAKpUKn376KeLj41unYCsX2dUdPQJdsftIJtb9cgGvf/M74iO9MWVIV7g4dI5vD4jag8pqndnKJnV3sqyorjTup5DKoVGp4e/oiz6evUwmXNrZ2Ir4CoiIqDWIFsorKiogl8vNtte1n1RWVpo9VkehUCAwMBAJCQkYNWoUqqursWrVKjz99NP45ptvEBUV1eR6GuvvaW1qdcuOZk8f44zxg0OwcvsZbNpzHgfO5GDa8G6YPDQESjlH0Kj9aulrpTUZDAZcL8tHZkm28c+1kmxkFucgr7zAuJ8EEnjYu8HHUYMeXt3g4+hZ88fJE252XFqQmqc9XStEYrK2a0W0UG5ra4uqqiqz7XVh/Fa94W+88QaOHz+OpKQkSGtvpDN27FhMmDABb731FlasWNHketrbRM/bmdQ/AH3D1Vi9Mx3fbT2NLXsvYNqQYPTt4cl+c2p3rHXyWllVmUmbiXHku/w69Aa9cT87G1toVGoEOwdhgHeccdRbbecBhcx8cELQAnlabVu+FOogrPVaIbI2nOhZj1qtbrBFJTc3FwAa7SfX6XRISkrCo48+agzkACCXyzFo0CAsX74cer0eNjZcTcDTVYUnpkTi9KUCrNhxFos2nsJPBzMwY0Q3hPg6i10eUbtQbahGbnmeyVredQG8tOpGcJZKpPCwc4OnSo3u7qG1vd6amkmWcnv+MkxERLckWnINDw/H0qVLodVqTSZ7Hj161Ph4QwoLC6HX61FdXW32mF6vh16vB29Saio8wBWv3NcHv564huTd5/HW0oOI667BtKHB8HC2E7s8ItEJgoCSqlJka83vZHm9Ih8GwWDc11HuAI1KjSiPCHja3+jz9rB14yRLIiJqNtFCeUJCAr766iusXr3auE65TqdDcnIyYmNjjZNAMzMzUV5ejuDgYACAu7s7nJycsH37djzxxBPGvnStVoudO3ciNDS0wV71zk4qlWBQlA/6hGvww2+XsTX1Mg6fvY7RfbpgXL8A2Cn5zQJ1fLrqKuSW17aaaOu3m+SiXF9h3E8utYHazgO+Dt6I1UTVtJvY19xURyXnL7JERNTyREti0dHRSEhIwPz585Gbmwt/f3+sXbsWmZmZePvtt437vfDCC0hNTcWZM2cAADKZDA888AAWLFiA6dOnY9KkSTAYDEhKSkJWVhZeeOEFsV5Su2CrsMFdg7tiSC8fJO1Ox+Z9l/DLsWuYMrgrBkZ6QyrlV+xkPVKzDmFD+lYUVhbCRemCScEJiPOKveUxBsGAwsoi01vI1wbwgspCk31dlS7wVKnRxzPW5KY6rracZElERG1LtDt6AjWTOhcsWICNGzeiqKgIYWFhePbZZzFgwADjPjNnzjQJ5XU2btyIJUuW4OLFi9DpdAgLC8PDDz+MUaNGNauWjjbR01LpmUVYkXIW6VeL0UXjgBnDQ9A90E3UmoiAmkD+/ek1qDLcmBAul8pxT/hUxHnFolxfYdJqkl1vacH6x9jKlDfdybLujweUMkVDT03UblnD5wpRe2CNEz1FDeXWpLOGcqCmn/b30zlYvTMdecUV6BXigenDQ+DpphK7NOrEXvr1LbORbQCwkdhAJbdDse7GtSOVSOFu62oM3fX/6aRw5CRL6jSs5XOFyNpZYyhnIzFBIpEgrrsneoV4YPuBK9i07xJe+nI/hsf6YdLAQNjbskefWl61oRqFlcUoqCxEfkUBCiuKkF9ZiIKKAuRXFDYYyAFAL+gR4R5ubDXxVKnhYecOGyn/OiMiovaLn2JkpJDLML5/IAZGemPtL+fx04Er2HviGu4cGIShMb6wkbHHliwjCAK0+jIUVBQaA3ZBRaHJz0WVxSa3jgcAe7kKbkoXuNu5Ia+iAJXV5jcRc1W64K/dE9vqpRAREbUJhnIy4+ygxOyx3TE81g8rd5zD9z+dxc7DV3H3sBBEBbuzFYBQVV1VO8JdG7YrC80CuM5genMwG6kN3JQucLV1QbhrN7jausDNtuZn19rt9Xu8G+spnxSc0Gavk4iIqK2wp7xWZ+4pvxVBEHDk3HWs2nEO2QXliAh0xfQR3eCnbrgfito/g2BAia7UfIS7trWkoKIIJVWlZsc5KxzhausKV6VzbeB2rflnbeBuzg10mrP6ClFn1h4+V4isgTX2lDOU12IovzV9tQE7Dl3Fhj0XUK7TY0i0DyYP6gone65e0d5U6CsaCdw1I92FlUWoFkxvzqWUKYwh21VZO8KtvDHS7ax0hrwVe7rb07VCJCZeK0SWscZQzvYVsoiNTIrRfbpgQE8vrN9zATsPXcX+tGxM6B+IkXd0gdyG/ebW4PaTJ4tQri83OUYqkcJF6QxXpQuCnP1rwrfStLXEzsaWbUtEREStiKGcmsTBTo57R4VieKwvVu44h9W70o395r3D1AxuraglJk92c+1qFridlU68UQ4REZHIGMqpWbzd7fF0YjROXsjHih1n8em6Ewj1c8b0Ed0Q5O0kdnntksnkycqi2v5tyyZPulg4eZKIiIisE3vKa7GnvPmqDQb8cuwa1v18HsVlVRjQ0wtTBneFm5Ot2KVZjZaePOmqdIabrWuzJk+2Rx3lWiFqbbxWiCzDnnLqkGRSKYb28kXf7p7YtO8itv9+BQdO5yChrz/G9g2AUiETu8RWd6vJkwW1I9+NTp5UuqCLo59x8mTdaHdrT54kIiIi68FPfGoxdkobJA4NwdBevli9Kx0bfr2IX45dw5TBXdG/pxek7XREt/7kyZtHuG8/edIZgc7+iOXkSSIiIroFhnJqcWoXO/x9ck/8caUQK1LOYvHmNKQczMCMEd0Q2sVF7PJM/JnJk66cPElEREQthD3ltdhT3joMgoDfTmZhze7zKCipxB1haiQOC4Haxa5Nnv/Wkydrfr7V5Em3eu0kda0lnDzZ9jrDtULUEnitEFmGPeXU6UglEgzo6Y3eoRpsTb2MH/ZfwpFz1zHqji4Y3z8QKtvm/y9468mTNX8amzzpYusCH3tPRLiHddrJk0RERGQ9GMqpTSgVMtw5MAiDo32QvDsdP+y/jD3Hr2HyoK4YHO0NmdS81ePG5Ml6I9xNmjzpy8mTRERE1C6wfaUW21faVvq1AizfdRwX83Lg5iGgZ5gKCpWuJnxXFiG/ovCWkyeNywMqXeBq62z8d06e7Jg687VC1BS8Vogsw/YV6hTqT568eXTbZPKkpwClJ6AFsL8AkOYpoVa5QuPghhCXIJMRbk6eJCIioo6MoZyarLmTJ2tGuF1N7zypdIGj3AmHT5Vi676ruKwzIDjGF+MGBcHBTi7SKyQiIiJqWwzlZKJFJ0/WazO53eRJv37AkEh/rNtzATsOZ2DfySxMig/E8N5+sJFxdJyIiIg6NobyTqa5kyddbV3h1sqTJ53sFZg1JgzDY32xcsc5rNhxDjsPX8Xdw0LQq5sHe8WJiIiow+JEz1odYaLnre48eavJk84KJ+ONb6xl8qQgCDh+Pg8rd5zDtbwyhPu7YMaIbvD3dGzTOsg6cPIakWV4rRBZxhonejKU12rLUJ6adQgb0reisLIQLkoXTApOQJxX7C2PudXkybr+7gbvPGmjMt7w5uYR7vYweVJfbcDuI5lY98t5lFXoMTDKG1MGd4Wzg1Ls0qgNMWgQWYbXCpFlGMqtWFuF8tSsQ/j+9BpU1ZsIKZfKMT10MoJdAlFQUXRjdLsJkyfdjHebdIab0rXD3XlSW1GFjb9eRMrBDNjYSDG+XwBG9+kChVwmdmnUBhg0iCzDa4XIMgzlVqytQvlLv76FgspCi/Z1UjjWjGrXu737jdu+d847T2bnl2HVznM4fPY63J2UmDY0BHHdNZ3ufehsGDSILMNrhcgy1hjKOdGzjd0qkM/qPp13nrwNTzcV/m9qFNIuFWBFyll8vuEkfjp4BTNGdEOwj7PY5RERERE1i/U2E3dQrkqXRrf39e6NUNdgeNi5M5DfRvcAV7w6uw/uHxuO64UVeHPJQSzacBJ5RRVil0ZERETUZAzlbWxScALkUtOb4silckwKThCpovZLKpVgULQP3nqkHyYMCMDBP3Lxry9+Q/LP6ajQ6cUuj4iIiMhiHI5tY3WrrDR19RVqnJ3SBlMGB2NItC+Sdqdj095L+OXoNUwZ3BXxkd6QStlvTkRERNaNEz1rdYR1yqlG+tUirEg5i/TMYvhrHDBjRDeEB7iKXRb9CbxWiCzDa4XIMtY40ZPtK9ThBPs6418ze+ORST1QWlGFecsP46M1x5BdUCZ2aUREREQNYvsKdUgSiQT9enghtpsaP/5+BVv2XcJLX+zHiN5+mBQfCJWt/PYnISIiImojDOXUoSnkMkwcEIhBUd5Y+/N5bP/9CvaeyMKdA4MwNMYHMim/LCIiIiLxMZFQp+DioMT947rj1fv7wE9tj2Xb/8Ari1NxLD1P7NKIiIiIGMqpc/H3dMTzf4nB/02JRLVBwILVR/HeyiO4mlsqdmlERETUibF9hTodiUSCmFA1IoPdseNgBjb8ehGvfJWKIb18MXlQEJxUCrFLJCIiok6GoZw6LRuZFKPj/NG/pxc27LmInYevYv+pLEwYEIiRvbtAbsMvkoiIiKhtMHVQp+eoUuDe0aF4/cE4dPNzweqd6Xjpy99w4HQOuIw/ERERtQWGcqJaPh72eDoxGs9Oj4bCRoZP153AO98fxsWsYrFLIyIiog6OoZzoJj2D3PHvB/pg5pgwXMvT4o1vDmDxplMoKKkUuzQiIiLqoNhTTtQAmVSKYTG+6NvdE5v2XcRPB67g9zM5GNc3AGP6+kMpl4ldIhEREXUgooZynU6HDz74AOvXr0dxcTHCw8PxzDPPoH///rc8bvjw4bh69WqDjwUEBGDbtm2tUS51QipbG9w9LARDY3yxeuc5rNtzAbuPZmLakGD0jfCEVCIRu0QiIiLqAEQN5XPmzMG2bdswa9YsBAQEYO3atXj44YexdOlSxMTENHrcv/71L2i1WpNtmZmZWLBgAeLj41u7bOqENC52ePyuSJy5XIAVO87hi02n8NPBK5gxohu6+bmIXR4RERG1cxJBpOUljh07hsTERMydOxezZ88GAFRWVmLChAnQaDRYtmxZk8736aef4oMPPsDy5csRGxvb5Hry8kphMLTtW6FWOyI3t6RNn5P+PIMgYN+JLKzZnY7CUh36hGswbWgw1C52YpfWYfFaIbIMrxUiy4h1rUilEri7OzT8WBvXYrR161bI5XIkJiYatymVSkybNg0HDx5ETk5Ok863adMm+Pn5NSuQEzWFVCJBfKQ33n6kPybFB+Louet48Yv9WL3rHMor9WKXR0RERO2QaKE8LS0NQUFBsLe3N9keFRUFQRCQlpZm8blOnTqF9PR0TJgwoaXLJGqUUiHD5EFd8dYj/dAnXIMffruMuZ/vw64jV9v8WxciIiJq30QL5bm5udBoNGbb1Wo1ADRppHzjxo0AgEmTJrVMcURN4OZki4cn9sDL990BjZsKS7aewb+/TsXJi/lil0ZERETthGgTPSsqKiCXy822K5VKADX95ZYwGAzYvHkzevTogeDg4GbX01h/T2tTqx1FeV5qeWq1I/pE+uDXY5n4etMpvLviCPr08MQDEyPgp+F/5z+L1wqRZXitEFnG2q4V0UK5ra0tqqqqzLbXhfG6cH47qampyM7ONk4WbS5O9KSWEubjhDce6IPtBzKwae9FPPG/nRgW44tJA4PgYGf+iyjdHq8VIsvwWiGyjDVO9BQtlKvV6gZbVHJzcwGgwdaWhmzcuBFSqRTjx49v0fqI/gy5jQzj+gUgPtIb6345j5RDGdh3MguT4oMwLNYXNjLeTJeIiIhuEC0ZhIeH48KFC2brjR89etT4+O3odDps27YNcXFx8PT0bJU6if4MZ3sF7ksIx2v3xyHAyxHLU87i5cWpOHL2OkRajZSIiIiskGihPCEhAVVVVVi9erVxm06nQ3JyMmJjY40hOzMzE+np6Q2eY/fu3SguLsbEiRPbpGai5vLTOOAf03vhqWlRkAD4cM0xzF9xBFdySsUujYiIiKyAaO0r0dHRSEhIwPz585Gbmwt/f3+sXbsWmZmZePvtt437vfDCC0hNTcWZM2fMzrFx40YoFAqMGTOmLUsnahaJRILoEA9EBLlh1+GrWL/nAv79dSoGRfngrsFd4WyvELtEIiIiEolooRwA5s2bhwULFmD9+vUoKipCWFgYFi1ahN69e9/22NLSUuzatQtDhw6Fo6N1zZ4luhUbmRQj7+iCfhFe2PjrRew4lIHUtGyM7x+A0X26QG4jE7tEIiIiamMSgY2tALj6CoknK78Mq3acw5Fz1+HuZIvEYcHoE66BRCIRuzSrwWuFyDK8VogsY42rr3AJCCKRebmp8OS0KDw3oxfslDb4bP1JvP3dIZzPLBa7NCIiImojDOVEVqJHoBv+fX8fzB4bjpzCcvxnyQEs2ngS+cUVYpdGRERErUzUnnIiMiWVSjA42gd9wjXY8tsl/Jh6BYfO5GJMnD/G9vOHrYKXLBERUUfET3giK2SntMHUIcEYEu2DpN3p2Lj3In4+lompg4MxINILUvabExERdShsXyGyYh4udnjszp741197w83RFl9tScMb3xzAmcsFYpdGRERELYihnKgdCPFzxouzeuPhiT1QXKbDO98fxifJx5FTUCZ2aURERNQC2L5C1E5IJRL0j/BCbKga21IvY8tvl3Hk3HWMvMMPEwcEQmUrF7tEIiIiaiaGcqJ2RimXYWJ8EAZG+WDtz+exLfUKfj2ehcmDgjCklw9kUn4BRkRE1N7w05uonXJ1VOKB8d3xyuw+8PWwx3fb/sCrX/2O4+fzxC6NiIiImoihnKidC/ByxD/vicHjd0VCrzfg/VVH8d6qI7h6XSt2aURERGQhtq8QdQASiQS9w9SICnZHysEMbNx7Ea8uTsWQGB9MHhgER5VC7BKJiIjoFhjKiToQuY0UCX39MSDSC+v3XMDuw5n47WQ2Jg4IxIjefpDb8MsxIiIia8RPaKIOyEmlwMzRYXjtwTiE+Dpj1c5zePnL/Th4JheCIIhdHhEREd2EoZyoA/P1sMczd0fj2bujYWMjxSdrj2Pe94dxKatE7NKIiIionhYJ5Xq9Hj/++CNWrVqF3NzcljglEbWgnl3d8doDfTBzdCiuXtfi9W9+x1eb01BYWil2aURERIRm9JTPmzcP+/fvx5o1awAAgiDg/vvvx4EDByAIAlxcXLBq1Sr4+/u3eLFE1HwyqRTDYv3Qt4cnNu29hO0HruD30zkY188fY+L8oZDLxC6RiIio02rySPkvv/yCO+64w/jzjh078Pvvv+PBBx/Eu+++CwBYtGhRy1VIRC1KZSvH3cND8ObDfdEzyA1rf7mAf33xG347mcV+cyIiIpE0eaQ8KysLAQEBxp937twJPz8/PPfccwCAs2fPYuPGjS1XIRG1Co2rCo9PicSZywVYnnIWizaewk8HMzBjeDeE+DmLXR4REVGn0uSR8qqqKtjY3Mjy+/fvx4ABA4w/d+nShX3lRO1ImL8rXpndBw+M64684gq89d1BfLb+BK4XlYtdGhERUafR5FDu5eWFw4cPA6gZFb9y5Qr69OljfDwvLw8qlarlKiSiVieVSDAwyhtvP9IPEwcE4sjZ6/jXov1Yszsd5ZV6scsjIiLq8JrcvjJ+/Hh8+umnyM/Px9mzZ+Hg4IAhQ4YYH09LS+MkT6J2ylZhg7sGd8WQXj5I2p2Ozfsu4Zdj1zBlcFcMjPSGVCoRu0QiIqIOqckj5Y8++ijuuusuHDlyBBKJBO+88w6cnJwAACUlJdixYwf69+/f4oUSUdtxc7LFIxMj8NKsO6BxscM3P5zGa9/8jrSL+WKXRkRE1CFJhBZcbsFgMECr1cLW1hZyubylTtsm8vJKYTC07coTarUjcnN5ExeyboIg4PfTOVi9Mx15xRXoFeKBu4eHwMut7drUeK0QWYbXCpFlxLpWpFIJ3N0dGnysye0rt6LX6+Ho6NiSpyQikUkkEsR190RMNw9s+/0KNu27hJe/3I9hsb6YFB8EB7v29Qs4ERGRNWpy+8ru3bvx0UcfmWxbtmwZYmNj0atXL/zjH/9AVVVVixVIRNZBbiPD+P6B+O+j/REf6Y2UgxmY+/k+bD9wBfpqg9jlERERtWtNDuWLFy/G+fPnjT+np6fjrbfegkajwYABA7BlyxYsW7asRYskIuvhbK/A7LHh+Pf9cfD3dMTyn87ilcWpOHLuOm8+RERE1ExNDuXnz59Hz549jT9v2bIFSqUSSUlJ+PLLLzFu3DisW7euRYskIuvTReOA52b0wpNToyAA+DDpGN5deQQZOaVil0ZERNTuNDmUFxUVwdXV1fjz3r170a9fPzg41DStx8XFISMjo+UqJCKrJZFI0KubB954MA5/GdENl7JK8OrXqfh262kUa3Vil0dERNRuNDmUu7q6IjMzEwBQWlqK48eP44477jA+rtfrUV1d3XIVEpHVs5FJMapPF7z9aH+MiPXDnmPXMOfzfdjy2yVU6fn3ARER0e00efWVXr16YcWKFQgJCcHPP/+M6upqDB482Pj4pUuXoNFoWrRIImofHOzkuGdUKIbF+mLVjnNI2pWOXYevInFYCO4IU0Mi4c2HiIiIGtLkkfInn3wSBoMBTz/9NJKTkzF58mSEhIQAqFnP+KeffkJsbGyLF0pE7Ye3uz2eSozGP2b0gq1ChoXrTuC/yw7hwrVisUsjIiKySs26eVBhYSEOHToER0dH9OnTx7i9qKgI69atQ9++fREeHt6ihbY23jyIqHUYDAJ+OZaJtT+fR3FZFfpHeGHqkK5wc7K1+By8Vogsw2uFyDLWePOgFr2jZ3vGUE7Uusor9di87xK2/X4FUgmQ0NcfY/sGQKmQ3fZYXitEluG1QmQZawzlzb6j5+XLl5GSkoIrV64AALp06YIRI0bA39+/uackog7MTmmDaUODMaSXD5J2pWPDrxfx89FMTB0SjP49vSBlvzkREXVizRopX7BgAb744guzVVakUikeffRRPPXUUy1WYFvhSDlR2zqbUYgVKWdx4VoJArwc8ZcR3RDaxaXBfXmtEFmG1wqRZTrESHlSUhI+++wzxMTE4KGHHkK3bt0AAGfPnsXixYvx2WefoUuXLpgyZcqfq5qIOrRufi54cdYd2H8yG0m70/HfZYfQO0yNxGEh0LjYiV0eERFRm2rySPmUKVMgl8uxbNky2NiYZnq9Xo97770XVVVVSE5ObtFCWxtHyonEU1lVjR/3X8aW/ZdgMAgYeUcXTOgfiKPp15G8Ox35xZVwc1JiypBg9I/wErtcIqvFzxUiy3SIkfL09HQ8++yzZoEcAGxsbDBu3Di89957Ta+SiDotpVyGSQODMCjaB8m707F1/2XsPJQBfbWA6tpflvOKK/HtD6cBgMGciIg6nCavUy6Xy1FWVtbo41qtFnK5/E8VRUSdk6ujEg9O6IFXZt+BasONQF5HpzcgeXe6SNURERG1niaH8sjISKxcuRLXr183eywvLw+rVq1CdHR0ixRHRJ1ToJcT9NUNt5PlFVe2cTVEREStr8ntK3//+98xe/ZsjBs3DlOnTjXezfPcuXNITk6GVqvF/PnzW7xQIupc3J2UjQbwT9cex5Bevuge6MqlFImIqENo1pKIO3bswBtvvIFr166ZbPfx8cErr7yCoUOHWnQenU6HDz74AOvXr0dxcTHCw8PxzDPPoH///hYdv3HjRnz77bc4d+4cFAoFQkND8c9//hNRUVFNfUmc6ElkZfadzMK3P5yGTm8wbpPLpAjzd8HFrBKUllfBw9kWg6N9MDDKGy4OShGrJbIO/Fwhsow1TvRs9h09DQYDTpw4gYyMDAA1Nw+KiIjAqlWrsGTJEmzZsuW253j22Wexbds2zJo1CwEBAVi7di1OnDiBpUuXIiYm5pbHvv/++/jyyy8xadIkxMbGoqysDKdPn8bIkSMxYsSIJr8ehnIi67PvZFaDq69U6atx8I9c/HwkE6cvF0IqkSA6xB1DevmiZ5AbpFKOnlPnxM8VIst0qFDemIULF+LDDz9EWlraLfc7duwYEhMTMXfuXMyePRsAUFlZiQkTJkCj0WDZsmWNHnvo0CHcc889+OijjzBq1KgWqZuhnMh63epayc4vw89HM7Hn+DWUlFXB3UmJQVE1o+duTrZtXCmRuPi5QmQZawzlTZ7o2VK2bt0KuVyOxMRE4zalUolp06bh4MGDyMnJafTYJUuWIDIyEqNGjYLBYIBWq22LkonICnm6qZA4LATvPh6Pv0/uCS83FdbtuYDnF+7FB6uP4sjZ66g2GG5/IiIiIhE1eaJnS0lLS0NQUBDs7e1NtkdFRUEQBKSlpUGj0TR47L59+zB+/Hi89957WLp0KcrKyuDr64unn34akyZNaovyicjK2MikuCNcgzvCNcgpLMcvRzOx59g1HF1zDC4OCgyK8sGgaG94OPNuoUREZH1EC+W5ubnw9PQ0265WzqCymAAAIABJREFUqwGg0ZHyoqIiFBYWYvPmzZDJZHjuuefg4uKCZcuW4fnnn4ednV2LtbQQUfukcbHD1CHBuHNgEI6l52H3kUxs2nsRm/ZeRERXNwyJ9kV0iDtsZKJ9WUhERGRCtFBeUVHR4E2GlMqaFRQqKxteCq3uxkWFhYUma6KPGjUKo0aNwieffNKsUN5Yf09rU6sdRXleovamudeKt5czxsR3RU5+GbanXsb21Ev4ZO1xuDoqMTLOH6P7BsDL3f72JyJqJ/i5QmQZa7tWLArlX3/9tcUnPHTokEX72draoqqqymx7XRivC+c3q9vu5+dncpMihUKBMWPGYMmSJdBqtWZtMbfDiZ5E1qslrhUJgNG9fTEixhvHz+fj5yOZSNpxFqtTzqJHoCsGR/sgNlTN0XNq1/i5QmQZa5zoaVEof+edd5r0hBILbuahVqsbbFHJzc0FgEb7yV1cXKBQKODh4WH2mIeHBwRBQGlpaZNDORF1DjKpFL1CPNArxAP5xRXYc/wafjmaic/Wn4SjSo74SG8MjvaBl5tK7FKJiKgTsSiUL1mypMWfODw8HEuXLjUb1T569Kjx8YZIpVJ0794d2dnZZo9lZWVBJpPB2dm5xesloo7HzckWk+KDMKF/IE5ezMfuI5nYlnoFW/dfRri/CwZH+6B3mBpyG5nYpRIRUQdnUSiPi4tr8SdOSEjAV199hdWrVxvXKdfpdEhOTkZsbKxxEmhmZibKy8sRHBxscuw777yDX3/9FfHx8QCA0tJS/PDDD4iJiYGtLdcmJiLLSaUSRHZ1R2RXdxSWVuLX49fw89FMLNp4CvbbbTCgpzcG9/KBrwe/gSMiotbR4jcPaoqnnnoKKSkpuO++++Dv72+8o+e3336L3r17AwBmzpyJ1NRUnDlzxnhceXk5pkyZguzsbMyePRtOTk5Ys2YNLly4YHJsU7CnnMh6iXGtGAQBaZcK8PORTBz6IxfVBgEhfs4YEu2DPuEaKOQcPSfrw88VIsu0257y1jJv3jwsWLAA69evR1FREcLCwrBo0aLbhmo7OzssWbIE8+bNw3fffYeKigpERETg66+/blYgJyK6mVQiQUSgGyIC3VCs1WHviSzsPnIVizenYflPZ9E/wgtDevnATyPOyk1ERNSxiDpSbk04Uk5kvazlWhEEAWf+v737jq+6Pvs//jore+9JSAIkEDIYopERkSEqFNtiqQOt9aZV0d7qr63a3sP2rq0PpY6iUovtXeW21aIgiMoUAQFFQZIwApKwwskOmUDm+f2RcDSGESThe5K8n3+R7/mO6/DIJ7nOlc/n+hytYlO2nc/3l9Lc4iAhyo+s9CjGDA3H3U3VczGWq4wVEVfnipVyJeXtlJSLuC5XHCt1p5qc1fOiipN4uFm4KiWCrPQo4iJcq/et9B+uOFZEXJErJuWGTl8REemtfDxtTL0ilimjYzh4vJqNu+xsyS3ioy+OExfhS1ZGFFcODcfTXT9mRUTkwlQpb6dKuYjr6i1jpf50E5/sKWHjruMUltXjbrNw5bAwsjKiGRjh26U9HEQuRW8ZKyJGU6VcRKQP8/awMWlUDNeOjKagqIaNu+x8sreETdlFxIb5kJURxVXDIvDy0I9eERHpSJXydqqUi7iu3jxWTjU088netur50ZI63KxmrhgaRlZ6NInRfqqeS7fqzWNF5HJSpVxEpJ/xdLcycUQ0E0dEc7j4q+r5ltxiokO8mZAeRebwCHw8bUaHKiIiBlKlvJ0q5SKuq6+NldONzWzfV8rGXXYOFdVgtZi5IjmUCelRDIkNUPVcvrW+NlZEeooq5SIigoeblQnpUUxIj+JoSS0bs+18sqeYbXtKiAjyYkJ6FGNTI/D1cjM6VBERuUxUKW+nSrmI6+oPY6WhsYXP8krZlG3n4PFqrBYTI4eEkpUeRVJcIGZVz6UL+sNYEekOqpSLiMhZubtZGJcWybi0SI6X1bEx28623cVs31dKWIAnEzKiGJsaib+3quciIn2RKuXtVCkXcV39daw0NrWw40AZG3fZOXCsCovZRMbgELIyohg2MEjVc+mkv44VkYulSrmIiHSZm81CZkoEmSkRFFXUsynbzpbcYnbsLyPE34Px6VGMS40k0Nfd6FBFROQSqVLeTpVyEdelsfKVpuZWvviyrXq+78gJzCYT6YOCycqIYnh8MGazquf9mcaKSNeoUi4iIpfEZjUzZmg4Y4aGU3LiZFv1PKeIL74sJ8jPnfFpUYxPiyTIz8PoUEVE5CKoUt5OlXIR16Wxcn7NLa3s+rKcjdl29h6qBBOkJrRVz9MSg7GYzUaHKJeJxopI16hSLiIi3c5qMTM6OYzRyWGUVZ1ic46dzTlFLHg7lwAfN8alRTEhLZKQAE+jQxURkXNQpbydKuUirktj5eK1tLaSc7CCjdl2cgsqwAEp8UFkZUSRPigEq0XV875IY0Wka1QpFxGRy8JiNjNiSCgjhoRSWXOazTlFbMq28+Ky3fh5uzE2NYIJ6VGEB3oZHaqIiKBKuZMq5SKuS2Ole7S2OsgtqGBTtp3sgxW0OhwMjQskKyOKEYNDsVlVPe/tNFZEukaVchERMYzZbCJ9UAjpg0I4UdvAxzl2NmUX8efle/DxtDmr55HB3kaHKiLS76hS3k6VchHXpbHSc1odDvYeqmRjtp1dX5bT0uogKTaACRlRjE4KxWa1GB2iXASNFZGuUaVcRERcitlkYnhCMMMTgqmub2RLbhGbdtlZ9O5e/rHWSubwCLLSo4gOPfsvERER6R5KykVEBAB/bzduuCqOaVcOYP+RE2zMtrNh53HWfV7IoGh/sjKiGJ0chrtN1XMRke6mpFxERDowm0wMHRjE0IFB1JxsZGtuMRuz7fz1vX38Y92XZKaEMyE9igHhvkaHKiLSZygpFxGRc/LzcmPalQO4bkwsB45VsTG7bXHohzuPEx/pR1ZGFGOGhuHhpl8nIiKXQgs922mhp4jr0lhxLXWnmti2u616bi+vx8PNwlXDwpmQEcXACD+jw+vXNFZEukYLPUVEpNfz8bQx5YpYJo+OIf94DRt3HWfr7mI+2mUnLtyXrIworhwWjqe7fsWIiHSVKuXtVCkXcV0aK67v5Okmtu0pYeMuO4VldbjZzFw5tK16nhDph8lkMjrEfkFjRaRrVCkXEZE+ycvDxqRRMVw7MppDRbVs3HWc7ftK2ZxTREyoD1kZUWSmhOPlYTM6VBERl6RKeTtVykVcl8ZK73SqoZlP97VVz48U1+JmNXNFchgTMqIYFO2v6nkP0FgR6RpVykVEpN/wdLdyTUY012REc6S4lo3Zdj7ZU8yW3cVEhXgzIT2Kq4dH4OOp6rmIiCrl7VQpF3FdGit9x+nGZrbvK2VTtp0Cew1Wi5nRSaFkZUQxJDZA1fNLpLEi0jWqlIuISL/m4WZlQnoUE9KjOFZax6ZddrbuKeaTvSWEB3mRlR7F1akR+Hm5GR2qiMhlpUp5O1XKRVyXxkrf1tDUwud5pWzMtnOwsBqL2cTIIW3V8+S4QMyqnneZxopI16hSLiIi8g3uNgtjUyMZmxrJ8fL6tur57iI+yyslLMCT8emRjEuNxN/H3ehQRUR6jCrl7VQpF3FdGiv9T1NzCzv2l7Fxl539x6qwmE1kDAohKyOKYfFBqp6fg8aKSNeoUi4iItIFNquFq1IiuColgqKKejZnF/FxbhE7DpQR7OfBhPRIxqVFEeir6rmI9A2qlLdTpVzEdWmsCEBTcytffFnGpmw7ew+fwGSC9MS26nlqQjBms6rnGisiXaNKuYiIyLdks5oZMzScMUPDKT1xkk3t1fNdB8sJ9HVnfFok49OiCPb3MDpUEZGLpkp5O1XKRVyXxoqcS3NLK9kHy9mYbWdPQSUAqYnBTEiPIi0xGKvFbHCEl5fGikjXqFL+DY2NjTz//PMsX76cmpoakpOTeeihh8jMzDzvdQsWLOCFF17odDwkJIQtW7b0VLgiIuJirBYzo5LCGJUURnnVKTblFPFxjp0Xlubi7+PmrJ6HBngaHaqIyHkZmpQ/+uijrFmzhjvuuIO4uDiWLVvG3LlzWbx4MSNGjLjg9b/97W/x8Pjqz5Rf/7eIiPQvIQGefG9CAjPHDSQnv4JNu+y8t+0I7209wrD4ILLSo8gYHNLvquci0jsYlpTn5OTw3nvv8dhjj/GjH/0IgJtuuonp06czf/58Xn/99Qve4/rrr8fPz6+HIxURkd7EYjYzYnAoIwaHUllzmo9zitiUY+eld3bj52VjbGokE9KjCA/yMjpUEREnw5LyVatWYbPZuPnmm53H3N3dmTVrFs8++yylpaWEhYWd9x4Oh4O6ujq8vb0xqWetiIh8Q5CfB98ZF8/0qwey+1AFG3fZWb39GB98epTkAQFkZUQzckgoNquq5yJiLMOS8n379hEfH4+3t3eH42lpaTgcDvbt23fBpPyaa67h5MmTeHt7c9111/HII48QEBDQk2GLiEgvZDabSEsMIS0xhBO1DWzJLWJTtp2XV+zBx9PG1cMjyMqIIjLY+8I3ExHpAYYl5WVlZYSHh3c6HhoaCkBpaek5r/Xz82POnDmkp6djs9n45JNPePPNN9m7dy9LlizBzc2tx+IWEZHeLdDXnelXD+SGzDj2Hq5k0y4763cUsuazYwyJ8ScrI5pRSaG42SxGhyoi/YhhSfnp06ex2Wydjru7t+3O1tDQcM5r77zzzg5fT5s2jcGDB/Pb3/6Wd955hx/84AcXHc+52tP0tNBQX0OeK9LbaKxITwgP82PimIGcqD3Nh58dY/WnR1i0ci//XG9j4uhYrrsyjrjI3rV2SWNFpGtcbawYlpR7eHjQ1NTU6fiZZPxMct5Vt9xyC08//TTbtm37Vkm5+pSLuC6NFbkcJqRGMG54OPuPVrFx13E+2HqIdzcXkBjtR1Z6NFcMDcPdxavnGisiXaM+5V8TGhp61ikqZWVlABecT/5NZrOZ8PBwqquruyU+ERHpf8wmE0PjAhkaF0jtyUa27i5m4y47f3t/H/9cf4CrUiLISo9iQLhrVdhEpPczLClPTk5m8eLF1NfXd1jsmZ2d7Xz9YjQ1NVFUVMTw4cO7NU4REemffL3cuG7MAKZeEcuXhdVs3HWczdlFbNh5nPhIX7IyohkzNAwPN0O3/BCRPsKwHlDTpk2jqamJJUuWOI81NjaydOlSRo4c6VwEarfbyc/P73BtZWVlp/v99a9/paGhgfHjx/ds4CIi0q+YTCaGxAYwd0YKz9w/llsmD6axuZW/f5DHQy9s4dVVeRwurjE6TBHp5Qz7eJ+ens60adOYP38+ZWVlDBgwgGXLlmG32/nDH/7gPO+RRx5h+/bt7N+/33ls4sSJ3HDDDQwZMgQ3Nzc+/fRTVq9ezahRo5g+fboRb0dERPoBH08bU0bHMnlUDPn2GjbtsrOtfYrLgHAfstKjuHJYBF4eqp6LyMUx9KfGU089xXPPPcfy5cuprq4mKSmJv/zlL4waNeq8182YMYOdO3eyatUqmpqaiI6O5r777uOnP/0pVqt+EIqISM8ymUwMivZnULQ/P5w0mE/2tiXmi9cc4M0NBxkzNJys9CgSovy0uZ2IdInJ4XBc3pYjLkrdV0Rcl8aK9AYOh4PDxbVs3GXn070lNDS1EBPqzYT0KDKHR+Dt0bkNcHfTWBHpGlfsvqKkvJ2SchHXpbEivc2phma27yth4y47h4trsVnNXJEcxoT0KAbH+PdY9VxjRaRrXDEp11wPERGRbubpbiUrI5qsjGiOFNeyKdvOtj3FbN1dTGSwF1npUVydGomPZ89Xz0Wkd1ClvJ0q5SKuS2NF+oKGxha255WwaZedfHsNVouJUUlhZKVHkTQgoFuq5xorIl2jSrmIiEg/5e5mYXxaFOPToigsrWNjdlvnlk/3lhAe6MmEjCjGDo/Ez9vN6FBFxACqlLdTpVzEdWmsSF/V2NTC5/tL2bjLzpeF1VjMJkYMCSUrI4qhcYGYL7J6rrEi0jWqlIuIiIiTm83C1cMjuXp4JPbyejZl29mSW8TneaWEBngwIT2KsamRBPi4Gx2qiPQwVcrbqVIu4ro0VqQ/aWpuYceBMjbtspN3tAqzyUTG4BAmpEcxPD4Is/nc1XONFZGuUaVcREREzstmtXDVsAiuGhZBceVJZ/V854Eygv3cGZ/eNi890FfVc5G+RJXydqqUi7gujRXp75pbWvniy3I27TrOnsMnMJkgPbGtep6aGMT2faUs3ZhPZU0DQX7ufC8rkcyUCKPDFnFZqpSLiIjIRbNa2jYfuiI5jNKqU2zOtvNxThG7Dpbj5W6hoamVlvbCUkVNA69+kAegxFykFzEbHYCIiIh0XViAJ9/PSuTp+65m3ndTaWpxOBPyMxqbW1m6Md+gCEXk21BSLiIi0gtZLWZGJYXS1Nx61tcrahr4x7oD7C6ooKm55TJHJyIXS9NXREREerFgP3cqaho6HbdZzWzcZWfd54W42cwMHRBIWmIwqQnBhAR4GhCpiJyPknIREZFe7HtZibz6QR6NX6uYu1nN3Hl9MqOGhJJ3tIrc/ApyCsrJzq8AIDLYi7TEYNISghkcG4DVoj+cixhN3VfaqfuKiOvSWBE5v217ii/YfcXhcFBy4hQ5+RXk5pez/1gVzS0O3N0spAwMIjUhiNSEYIL8PAx6FyKXjyt2X1FS3k5JuYjr0lgR6ZqLGSunG5vJO1JFTkFbkn5mCkxMqE9bFT0xmMRoPyxmVdGl73HFpFzTV0RERPohDzcrGYNDyBgcgsMxBHt5fXuCXsHq7Ud5/5MjeLpbSYkPIi0hmNSEIPx9tGGRSE9RUi4iItLPmUwmokN9iA714for4zjV0Mzew5Xk5FeQU1DB53mlAMRF+JKa0FZFT4j0w2w2GRy5SN+hpFxEREQ68HS3MiopjFFJYTgcDo6V1pFbUEFOfgXvbTvMyq2H8fawkpoQTGpiMMPjg/D1cjM6bJFeTUl5FzU1NVJbW0VzcyOtrd3T77W01Exr69n7y0rvZLFY8fEJwNPT2+hQRES6hclkYkC4LwPCfbkxcyD1p5vYc6iS3PwKcgsq+GRvCSYgPsqvbZpLYjBxEb6YTaqii1wMLfRsd76FnqdO1VNbewIfH3/c3T0xmy2YuuGHjdVqpvkcmz5I7+NwOGhqaqSqqgxf30Al5t1ICz1FuuZyj5VWh4MjxbXtLRcrOGSvwQH4edkY3j7NJSU+CG8P22WLSaQrtNCzl6qrqyYgIAQ3N7WJknMzmUy4ubkTEBBKdXW5knIR6fPMJhPxkX7ER/rxnXHx1JxsdFbRsw+Ws3V3MSYTDIr2d25cFBvm0y2FLZG+Rkl5F7S0NGGzacW5dI3N5kZLS7PRYYiIXHZ+Xm5kpkSQmRJBa6uDgqIaZxX97Y0FvL2xgAAfN+di0WEDg/B0VyoiAkrKu0yf6qWr9L0iItL2Z/pB0f4MivbnuxMSqK5rILegsq2by/4yNucUYTGbGBzjT2r77qJRId76GSr9lpJyERER6XH+Pu6MS4tkXFokzS2tFNhr2lou5lewZEM+SzbkE+znTmpiCKkJQQyNC8TDTWmK9B/6bpcedf/9PwHghRf+clmvFRER12W1mBkSG8CQ2ABmXZNIZc1pcgsqyC2oZNueYj764jhWi4mk2ABSE0NISwwmPNBTVXTp05SU91Pjxo3u0nlLlqwgMjKqh6MREZH+LMjPg6yMaLIyomluaeXLY1XktPdFf2P9l7yx/ktCAzxISwghNTGY5AEBuNksRoct0q3UErHd+VoiFhcfISIirtufaWRLxNWr3+/w9b/+9U9KSop44IGHOxyfMGEinp6e3/o5TU1NANhsF98O61KuNVpPfc/0V2qJKNI1fXGslFedcm5ctO/ICRqbW7FZzQyNC3RuXhQW8O1/T0n/pJaI4jKuu+6GDl9/9NF6qqurOh3/ptOnT+Ph0fXWkJeSUPfGZFxERLpXSIAnE0fGMHFkDE3NLew/2lZFz22fj85aiAjyamu5mBjMkJgAbFaz0WGLXDQl5XJO99//E+rq6vjlL3/FggXPsn9/Hrfddgd33/1TNm/+iBUrlnHgwH5qaqoJDQ3jhhtmMGfOXVgslg73gK/mhe/c+Tk/+9k9PPHEUxw6VMA777xNTU01qanp/OIXvyImJrZbrgV4++1/8cYbr1NRUU5iYiL33/8QixYt7HBPERHpPWxWC8MTghmeEAyToaTypDNB/3DncdZ8dgx3m4WhcYHOvujB/tpjRHoHJeUG2banmKWbCqioPk2wnzvfy0okMyXC6LA6qao6wS9/+RBTp05j2rQbCQ9vi/H991fi6enF7Nm34eXlyY4dn/PKK3+mvr6eefP+/YL3ffXVv2I2W7j11juora3hn/9czG9+8x8sWvRqt1y7bNlbPPvsU2RkjGT27FsoKiriscd+jq+vL6GhYd/+P0RERFxGeJAXU4K8mDI6loamFvKOnHAm6bsOlgMQHerd1hc9IZhBMf5YLaqii2tSUm6AbXuKefWDPBrb55NX1DTw6gd5AC6XmJeXl/Hoo//J9OkzOxx//PHf4e7+VfXhpptm8fTTv2fZsiXMnXsvbm5u571vc3Mzf/vbq1itbd+Cfn7+PP/8fAoKDpKQMOiSrm1qauKVVxaSkpLKc8+95Dxv0KDBPPHE40rKRUT6IHebhfRBIaQPCsHhcFBUcdI5F33tZ8dY9elRPN0tDBsYRFp7tT3QVxsDiutQUn4JtuQW8XFO0UVfl2+vprml46LSxuZW/vf9fWzaZb/o+41Li2RsauRFX9cVHh4eTJt2Y6fjX0/IT56sp7GxifT0ESxfvpQjRw4zePCQ8973xhu/40yWAdLTMwCw249fMCm/0LV5eXuprq7mvvu+2+G8KVOm8ac/PXPee4uISO9nMpmICvEmKsSb68YM4FRDM/uOnCAnv4Lcggp27C8DYECYT9vGRYnBJET5YTGrii7GUVJugG8m5Bc6bqTQ0LAOie0ZBQX5LFq0kJ07P6O+vr7Da/X1dRe875lpMGf4+voBUFt74ZXQF7q2uLjtg9I355hbrVYiI3vmw4uIiLguT3crI4eEMnJIKA6Hg+Nl9c6Wix98cpT3th3B28NKSnxQW0eXhGD8vM//F1+R7qak/BKMTf12FepfvLSFipqGTseD/dx55LaR3RFat/l6RfyM2tpaHnjgJ3h5+XD33fcQHR2Dm5sbBw7ksXDhAlpbL9zm0Ww+e3/ZrnTovJRrRUSkfzOZTMSE+RAT5sMNV8Vx8nQTew9/VUXfvq8UgPhIX2fLxfgIP8xmbVwkPUtJuQG+l5XYYU45gJvVzPeyEg2Mquu++GIH1dXVPPHE02RkfPUhoqjo4qfe9ISIiLYPSoWFx0hPH+E83tzcTFFREYmJ558eIyIi/YeXh43RyWGMTg6j1eHgWEkdOfnl5BZU8u7Ww6zYchgfTxupCUGkJgYzPD4YH0+17JXup6TcAGcWc/aG7itnY26fc/f1ynRTUxPLli0xKqQOkpOH4e/vz4oVy7juuhuc02/Wrl1FbW2NwdGJiIirMptMxEX4Ehfhy4yx8dSdamL3obZuLrkFlWzbU4LJBAlRfqQlBJOWGEJsuA9mk6rocumUlBskMyWC8elRhu3oeSlSU9Pw9fXjiSceZ9as2ZhMJlavfh9XmT1is9n48Y9/wrPPPs2DD97HxImTKCoq4oMP3iU6OgaTfniKiEgX+HjauGpYBFcNi6C11cHh4tr2KnoFyzYfYtnmQ/h7uzE8IYi0xBBSBgbi5aEqunw7Ssrlovn7B/DUU8/ywgvPsWjRQnx9/Zg69XpGjx7Dww/fb3R4AHz/+7NxOBy88cbrvPji8yQmDubJJ5/huefm4+amFlgiInJxzGYTCVF+JET5cdP4BKrrG9ld0DYPfdeX5WzJLcZsMjEoxt+5cVFMqLcKQdJlJodWxwFQUVFHa+vZ/yuKi48QERHX7c+0Ws29slLeW7W2tjJ9+hSysibyyCP/0aPP6qnvmf4qNNSXsrILd+YR6e80VozR0tpKgb2mbbFofgVHS9u6kAX6urdtXJQYzNC4QDzdVQt1FUaNFbPZRHCwz1lf03eH9EkNDQ24u3esiK9a9R41NdWMGDHKoKhERKQvspjNDI4JYHBMAN/PSuREbQO7CyrIKahg+74SNmXbsZhNDIkNcCbpkcFeqqJLB4Ym5Y2NjTz//PMsX76cmpoakpOTeeihh8jMzLyo+8ydO5dNmzZxxx138Otf/7qHopXeJCdnFwsXLuCaa67Fz8+fAwfyeO+9FSQkJDJx4mSjwxMRkT4s0Ned8elRbWvHWlo5WFjdtrtoQQX/2nCQf204SIi/B6nt01yGDgjE3e3s7X6l/zA0KX/00UdZs2YNd9xxB3FxcSxbtoy5c+eyePFiRowYceEbAB999BGff/55D0cqvU1UVDQhIaG89dab1NRU4+fnz7RpN3LPPfdjs2kRjoiIXB5Wi5nkuECS4wK5eeIgKqpPtyXo+RVszS1mw87jbecMCHDuLhoe6GV02GIAw+aU5+TkcPPNN/PYY4/xox/9CGibcjB9+nTCwsJ4/fXXL3iPxsZGZsyYwYwZM1iwYMElVco1p1y6k+aUdy/NkxXpGo2V3qWpuZUDhVXk5rcl6cWVJwEIC/Rsb7kYTNKAAGxWVdG7m+aUf82qVauw2WzcfPPNzmPu7u7MmjWLZ599ltLSUsLCws57j9dee43Tp09z9913s2DBgp4OWURERKTb2KxmUgYGkTIwiB9OGkxp1an2nugVbMy2s25HIW5WM0PjAtuq6AnBhAR4Gh229BDDkvJ9+/YRHx+Pt7d3h+NpaWk4HA727dt33qS8rKyMl156if/6r//C01PfoCIiItK7hQV4MmlUDJNGxdDY1ELe0fYqekE52fkVAEQGe5HWnqAPjg3AajEbHLV0F8OS8rIC1g7wAAAYwUlEQVSyMsLDwzsdDw0NBaC0tPS81z/zzDPEx8czc+bMHolPRERExChuNktb8p0YzK2OwZScONXecrGc9TsKWb39GO5uFobFBTr7ogf5eRgdtlwCw5Ly06dPn3XB3Zk2dg0NDee8Nicnh3feeYfFixd3Wzuhc83vASgtNWO19swn0Z66rxjLbDYTGuprdBh9iv4/RbpGY6VvCgvzIzWprZh5qqGZ3IPlfL6vhM/zSvjiy3IABkb6MSo5jNFDw0keGKQq+gW42lgxLCn38PCgqamp0/Ezyfg3e0yf4XA4eOKJJ5g6dSqjR4/utnjOt9CztbW1RxZkaqFn39Xa2qrFVt1Ii9dEukZjpf+ID/MmPiyBWRPisZfXk1PQtnHROxvzeXvDQTzdraTEB5GWEExqQhD+PtrN+uu00PNrQkNDzzpFpaysDOCc88nXrl1LTk4ODz30EIWFhR1eq6uro7CwkJCQEDw89CccERER6dtMJhPRoT5Eh/pw/ZVxnGpoZu/hSnLy2/qif57XlmvFhfs6Wy4mRPphNmvjIldjWFKenJzM4sWLqa+v77DYMzs72/n62djtdlpbW7nzzjs7vbZ06VKWLl3KokWLmDBhQs8ELiIiIuKiPN2tjEoKY1RSGA6Hg2Oldc6+6O9tO8zKrYfx9rCSmtA2Dz0lIQg/LzejwxYMTMqnTZvG3/72N5YsWeLsU97Y2MjSpUsZOXKkcxGo3W7n1KlTJCYmAnDttdcSExPT6X7z5s1j4sSJzJo1i5SUlMv2PkRERERckclkYkC4LwPCfbkxcyD1p5vYc6jS2Xbxk70lmID4KL+2aS6JwcRF+GLupvV6cnEMS8rT09OZNm0a8+fPp6ysjAEDBrBs2TLsdjt/+MMfnOc98sgjbN++nf379wMwYMAABgwYcNZ7xsbGMnmytlA3wvvvv8vvf/8blixZQWRkFACzZs1gxIhR/PrXj1/0tZdq587P+dnP7uFPf/ozI0d239oDERGR3srbw8aYoeGMGRpOq8PBkeLa9paLFSz/+BDvfHwIPy8bw9s3LkqJD8LbQ7tgXy6GJeUATz31FM899xzLly+nurqapKQk/vKXvzBq1Cgjw+oXfvnLh9i58zPefXftOfu8P/zw/ezZk8uKFWvOufDWaOvWraaysoIf/OBWo0MRERHpNcwmE/GRfsRH+vGdcfHUnGx0VtGzD5azdXcxJhMMivYntT1Jjw3z6baud9KZoUm5u7s7jzzyCI888sg5z1m8eHGX7nWmki5dM2XKdWzdupmPP97IlCnTOr1+4kQlO3Z8xtSp13/rhPwf/3gbs7ln2zGtX7+GL7880Ckpz8gYyfr1W87adlNEREQ68vNyIzMlgsyUCFpbHRQU1Tir6Es3FbB0UwEBPm7OBH3YwCA83Q1NI/sc/W/2U+PHX4Onpxfr1q0+a1L+4YfraGlpYerUzq91lZubcQtHzGazy1b3RUREXJnZbGJQtD+Dov357oQEqusayC2obOvmsr+MzTlFWMwmBsf4t3V0SQgmKsRbVfRLpKS8n/Lw8GD8+Cw2bFhHTU0Nfn5+HV5ft241wcHBxMbGMX/+k+zYsZ2SkhI8PDwYOXI08+b9+wXnf59tTnlBQT7PPfc0u3fn4u/vz8yZ3yMkJLTTtZs3f8SKFcs4cGA/NTXVhIaGccMNM5gz5y4sFgsA99//E3bt2gnAuHFt88YjIiJ56613zzmnfP36Nfzf//2dI0cO4+Xlzdix47n33p8REBDgPOf++39CXV0d//Vfv+WZZ55i3749+Pr6cfPNP+S22zp3/REREenL/H3cGZcWybi0SJpbWimw17S1XMyvYMmGfJZsyCfYz72to0tiMEPjAvFwU4p5sfQ/ZpDtxTt5t2AVlaerCHQP4DuJ0xgTMfKyxjBlyjTWrPmAjz5az3e+813n8eLiInbvzmHWrB+yb98edu/OYfLk6wgNDaOoyM4777zNAw/8lP/7vyUX1Q++oqKcn/3sHlpbW7n99jvx8PBkxYplZ61ov//+Sjw9vZg9+za8vDzZseNzXnnlz9TX1zNv3r8DcOedP+bUqVOUlBTxwAMPA+Dp6XXO559ZUJqSksq99/6M0tIS3n77Tfbt28OiRa91iKOmppr/9/9+xsSJk5g0aSobNqxj4cIFJCQMIjNzbJffs4iISF9itZgZEhvAkNgAZl2TSGXNaXILKsgtqGTb3hI+2mXHajGRFBtAamIIqQlBRAR5qYreBUrKDbC9eCf/yHubpta2HU1PNFTxj7y3AS5rYn7FFVcSEBDIunWrOyTl69atxuFwMGXKdSQmDmLixI4dbcaOncA999zFRx+tZ9q0G7v8vNdff5Xq6ipeeWUxSUltfeivv346t9zy3U7nPv7473B3/yrhv+mmWTz99O9ZtmwJc+fei5ubG1dccRVLly6hurqK66674bzPbm5uZuHCBQwaNIQFC152Tq1JSkrm8cd/zbvvLmPWrB86zy8tLeG///t3zqk906fPZNas6bz33nIl5SIiIu2C/DzIyogmKyOa5pZWvjxWRU57X/Q31n/JG+shNMCDtIQQUhODSR4QgJvNYnTYLklJ+SX4tGgH24o+u+jrDlUfpdnR3OFYU2sTr+97i6327Rd9v8zIK7gy8uI71litVq69djLvvPM25eXlhISEALBu3RpiYmIZNmx4h/Obm5upr68jJiYWHx9fDhzIu6ikfNu2LaSmpjsTcoDAwECmTLmeZcuWdDj36wn5yZP1NDY2kZ4+guXLl3LkyGEGDx5yUe81L28vJ05UOhP6M669dgovvvg8W7du6ZCU+/j4MHnydc6vbTYbQ4emYLcfv6jnioiI9BdWi5mhA4MYOjCI2dcOprzqlHPjos05dtbvLMRmNZM8IJC0xLapLmEBZ+8A1x8pKTfANxPyCx3vSVOmTGPp0iV8+OEafvCDWzl8+BAHDx7grrvmAtDQcJrFi//O+++/S1lZKQ6Hw3ltXV3dRT2rpKSY1NT0TscHDIjrdKygIJ9Fixayc+dn1NfXd3itvv7ingttU3LO9iyz2UxMTCwlJUUdjoeFhXf6U5uvrx/5+Qcv+tkiIiL9UUiAJxNHxjBxZAxNzS3sP9pWRc/Nr+D1tRWwFiKCvNoS9IRghsQGYLP2bNc2V6ak/BJcGTnqW1Wo/2PL7znRUNXpeKB7AA+OvKc7Quuy1NR0IiOjWbt2FT/4wa2sXbsKwDlt49lnn+b999/l5ptvYfjwVHx8fAATjz/+qw4Jeneqra3lgQd+gpeXD3fffQ/R0TG4ublx4EAeCxcuoLW1tUee+3Vm89n/tNZT71lERKQvs1ktDE8IZnhCMEyGksqTzgT9w53HWfPZMdxtFobGBTqT9GD/rq9b6wuUlBvgO4nTOswpB7CZbXwn8du3H7wUkydPZfHi/6Ww8Bjr168hKWmos6J8Zt74Aw885Dy/oaHhoqvkAOHhERQWHut0/OjRIx2+/uKLHVRXV/PEE0+TkfHVHPuiIvtZ7tq1hSMREZHOZ339ng6Hg8LCY8THJ3bpPiIiInLpwoO8mBLkxZTRsTQ0tZB35IQzSd91sByA6BBvZ8vFQTH+WC19u4qupNwAZxZzGt195YypU69n8eL/5YUXnqWw8FiHBPxsFeO3336TlpaWi35OZuZYlix5g/3785zzyk+cOMHatR90OO/MhkNfr0o3NTV1mncO4Onp2aUPCMnJwwgMDOKdd97i+uunOzcV2rBhPWVlpdx22x0X/X5ERETk0rnbLKQPCiF9UAgOh4OiipPOuehrPzvGqk+P4uluYdjAoLa2iwnBBPr2vb1IlJQbZEzESK6OGU1zc89PxbiQ+PgEBg0awscfb8JsNjNp0lcLHK++ehyrV7+Pt7cPAwfGs2dPLp9/vh1/f/+Lfs6tt97J6tXv8/DD85g164e4u3uwYsUywsMjqav70nleamoavr5+PPHE48yaNRuTycTq1e9ztpkjSUnJrFnzAQsWPENy8jA8Pb0YN25Cp/OsViv33vsAv//9b3jggZ8yefJUSktLeOutN0lISGTGjM4dYEREROTyMplMRIV4ExXizXVjBnCqoZl9R06Qk19BbkEFO/aXATAgzKetip4YTEKUH5Ye3kH8clBSLgBMnTqNgwcPMGLEKGcXFoB///efYzabWbv2AxoaGklNTee5517k4YcfuOhnhISE8Kc/vcyzzz7F4sV/77B50JNP/o/zPH//AJ566lleeOE5Fi1aiK+vH1OnXs/o0WN4+OH7O9xz5szvc+BAHu+/v5I33/wHERGRZ03KAW64YQZubm68/vqrvPji83h7ezNlyjTuuecB7f4pIiLigjzdrYwcEsrIIaE4HA6Ol9U7Wy5+8MlR3tt2BC93K8MTvqqi+3kbt6P4pTA5tHINgIqKOlpbz/5fUVx8hIiIzh1CLpXVanaJSrl0v576numvQkN9KSurNToMEZensSL9ycnTTew9/FUVvbq+EYD4SF/n7qLxEX6YzV+tP9u2p5ilG/OprGkgyM+d72UlkpkScdliNptNBAf7nPU1VcpFREREpNfx8rAxOjmM0clhtDocHCupIye/nNyCSt7depgVWw7j42kjNSGI1MRgTjW08Ob6L2lsL4hW1DTw6gd5AJc1MT8XJeUiIiIi0quZTSbiInyJi/Blxth46k41sftQWzeX3IJKtu0pOet1jc2tLN2Yr6RcRERERKS7+XjauGpYBFcNi6C11cHh4lp+99rnZz23oqbhMkd3dr1/qaqIiIiIyDmYzSYSovwI9jt7U4dzHb/clJSLiIiISJ/3vaxE3KwdU183q5nvZbnGBoKaviIiIiIifd6ZeeNGdl85HyXlXeRwODCZuralu/Rv6jIqIiLimjJTIshMiXDJ9qGavtIFFouNpibXWAQgrq+pqRGLRZ93RUREpOuUlHeBj48/VVXl1NfX0tLSrEqonJXD4aCxsYGqqjJ8fAKMDkdERER6EZXzusDT0xur1UZdXRX19dW0trZ0y33NZjOtrdrRsy+xWKz4+gbi6eltdCgiIiLSiygp7yKbzY3AwLBuvacrzmcSERERkctP01dERERERAympFxERERExGBKykVEREREDKakXERERETEYErKRUREREQMpu4r7cxmY3brNOq5Ir2NxopI12isiHSNEWPlfM80ObQTjoiIiIiIoTR9RURERETEYErKRUREREQMpqRcRERERMRgSspFRERERAympFxERERExGBKykVEREREDKakXERERETEYErKRUREREQMpqRcRERERMRgSspFRERERAxmNTqA/qa0tJTXXnuN7Oxsdu/ezcmTJ3nttde48sorjQ5NxGXk5OSwbNkyPv30U+x2OwEBAYwYMYIHH3yQuLg4o8MTcRm5ubn8+c9/Zu/evVRUVODr60tycjLz5s1j5MiRRocn4tIWLVrE/PnzSU5OZvny5UaHo6T8cjt06BCLFi0iLi6OpKQkvvjiC6NDEnE5r7zyCjt37mTatGkkJSVRVlbG66+/zk033cRbb71FYmKi0SGKuIRjx47R0tLCzTffTGhoKLW1tbz77rvcfvvtLFq0iLFjxxodoohLKisrY+HChXh5eRkdipPJ4XA4jA6iP6mrq6OpqYnAwEDWrVvHvHnzVCkX+YadO3cyfPhw3NzcnMcOHz7MjBkzuPHGG3nyyScNjE7EtZ06dYrJkyczfPhwXn75ZaPDEXFJjz76KHa7HYfDQU1NjUtUyjWn/DLz8fEhMDDQ6DBEXNrIkSM7JOQAAwcOZPDgweTn5xsUlUjv4OnpSVBQEDU1NUaHIuKScnJyWLFiBY899pjRoXSgpFxEegWHw0F5ebk+1IqcRV1dHZWVlRQUFPDMM89w4MABMjMzjQ5LxOU4HA7+53/+h5tuuomhQ4caHU4HmlMuIr3CihUrKCkp4aGHHjI6FBGX86tf/YrVq1cDYLPZ+OEPf8g999xjcFQiruedd97h4MGDvPjii0aH0omSchFxefn5+fz2t79l1KhRzJw50+hwRFzOvHnzmD17NsXFxSxfvpzGxkaampo6TQMT6c/q6ur44x//yE9+8hPCwsKMDqcTTV8REZdWVlbGT3/6U/z9/Xn++ecxm/VjS+SbkpKSGDt2LN///vf561//yp49e1xuvqyI0RYuXIjNZuOuu+4yOpSz0m83EXFZtbW1zJ07l9raWl555RVCQ0ONDknE5dlsNiZNmsSaNWs4ffq00eGIuITS0lJeffVVbr31VsrLyyksLKSwsJCGhgaampooLCykurra0Bg1fUVEXFJDQwP33HMPhw8f5u9//zsJCQlGhyTSa5w+fRqHw0F9fT0eHh5GhyNiuIqKCpqampg/fz7z58/v9PqkSZOYO3cuP//5zw2Iro2SchFxOS0tLTz44IPs2rWLl156iYyMDKNDEnFJlZWVBAUFdThWV1fH6tWriYyMJDg42KDIRFxLTEzMWRd3Pvfcc5w8eZJf/epXDBw48PIH9jVKyg3w0ksvATj7LS9fvpwdO3bg5+fH7bffbmRoIi7hySef5MMPP2TixIlUVVV12NTB29ubyZMnGxidiOt48MEHcXd3Z8SIEYSGhlJUVMTSpUspLi7mmWeeMTo8EZfh6+t71t8dr776KhaLxSV+r2hHTwMkJSWd9Xh0dDQffvjhZY5GxPXMmTOH7du3n/U1jRORr7z11lssX76cgwcPUlNTg6+vLxkZGfz4xz9mzJgxRocn4vLmzJnjMjt6KikXERERETGYuq+IiIiIiBhMSbmIiIiIiMGUlIuIiIiIGExJuYiIiIiIwZSUi4iIiIgYTEm5iIiIiIjBlJSLiIiIiBhMSbmIiBhmzpw5XHvttUaHISJiOKvRAYiISPf69NNPueOOO875usViYe/evZcxIhERuRAl5SIifdT06dOZMGFCp+Nms/5IKiLiapSUi4j0UcOGDWPmzJlGhyEiIl2gcomISD9VWFhIUlISCxYsYOXKlcyYMYPU1FSuueYaFixYQHNzc6dr8vLymDdvHldeeSWpqanccMMNLFq0iJaWlk7nlpWV8bvf/Y5JkyYxfPhwMjMzueuuu9iyZUunc0tKSnj44Ye54oorSE9P5+677+bQoUM98r5FRFyRKuUiIn3UqVOnqKys7HTczc0NHx8f59cffvghx44d47bbbiMkJIQPP/yQF154Abvdzh/+8Afnebm5ucyZMwer1eo8d8OGDcyfP5+8vDz++Mc/Os8tLCzklltuoaKigpkzZzJ8+HBOnTpFdnY2W7duZezYsc5zT548ye233056ejoPPfQQhYWFvPbaa9x3332sXLkSi8XSQ/9DIiKuQ0m5iEgftWDBAhYsWNDp+DXXXMPLL7/s/DovL4+33nqLlJQUAG6//Xbuv/9+li5dyuzZs8nIyADgiSeeoLGxkTfeeIPk5GTnuQ8++CArV65k1qxZZGZmAvCb3/yG0tJSXnnlFcaPH9/h+a2trR2+PnHiBHfffTdz5851HgsKCuLpp59m69atna4XEemLlJSLiPRRs2fPZtq0aZ2OBwUFdfj66quvdibkACaTiX/7t39j3bp1rF27loyMDCoqKvjiiy+YMmWKMyE/c+69997LqlWrWLt2LZmZmVRVVbF582bGjx9/1oT6mwtNzWZzp24xV111FQBHjhxRUi4i/YKSchGRPiouLo6rr776guclJiZ2OjZo0CAAjh07BrRNR/n68a9LSEjAbDY7zz169CgOh4Nhw4Z1Kc6wsDDc3d07HAsICACgqqqqS/cQEenttNBTREQMdb454w6H4zJGIiJiHCXlIiL9XH5+fqdjBw8eBCA2NhaAmJiYDse/rqCggNbWVue5AwYMwGQysW/fvp4KWUSkz1FSLiLSz23dupU9e/Y4v3Y4HLzyyisATJ48GYDg4GBGjBjBhg0bOHDgQIdz//KXvwAwZcoUoG3qyYQJE9i0aRNbt27t9DxVv0VEOtOcchGRPmrv3r0sX778rK+dSbYBkpOTufPOO7ntttsIDQ1l/fr1bN26lZkzZzJixAjneb/+9a+ZM2cOt912G7feeiuhoaFs2LCBjz/+mOnTpzs7rwD853/+J3v37mXu3LncdNNNpKSk0NDQQHZ2NtHR0fziF7/ouTcuItILKSkXEemjVq5cycqVK8/62po1a5xzua+99lri4+N5+eWXOXToEMHBwdx3333cd999Ha5JTU3ljTfe4E9/+hP//Oc/OXnyJLGxsfz85z/nxz/+cYdzY2Njefvtt3nxxRfZtGkTy5cvx8/Pj+TkZGbPnt0zb1hEpBczOfR3RBGRfqmwsJBJkyZx//3388ADDxgdjohIv6Y55SIiIiIiBlNSLiIiIiJiMCXlIiIiIiIG05xyERERERGDqVIuIiIiImIwJeUiIiIiIgZTUi4iIiIiYjAl5SIiIiIiBlNSLiIiIiJiMCXlIiIiIiIG+/8/pgEGrDPAPwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jXA7-YxHlRHc",
        "outputId": "77a094c2-c127-4338-9a8e-f420fcd29e2d"
      },
      "source": [
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"data/twitter-2016test-A.txt\", delimiter='\\t', header=None, names=['label', 'tweet','id'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Create sentence and label lists\n",
        "\n",
        "test_tweet = df.tweet.values\n",
        "y_test = df.label.values\n",
        "\n",
        "input_ids = []\n",
        "attention_masks = []\n",
        "\n",
        "labels=[]\n",
        "\n",
        "for label in y_test:\n",
        "  labels.append(label_dict[label])\n",
        "\n",
        "# Process test data  \n",
        "input_ids,attention_masks,labels = processdata(test_tweet,labels)\n",
        "\n",
        "# Set the batch size.  \n",
        "batch_size = 32  \n",
        "\n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(input_ids, attention_masks, labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of test sentences: 20,632\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVjq7-1kXKLe"
      },
      "source": [
        "from sklearn import metrics\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "\n",
        "def test(model,prediction_dataloader): \n",
        "\n",
        "  total_test_acc  = 0\n",
        "  total_F1_Score = 0\n",
        "  total_precision = 0\n",
        "  total_recall = 0\n",
        "\n",
        "  for batch_idx, (pair_token_ids, mask_ids,y) in enumerate(prediction_dataloader):\n",
        "    pair_token_ids = pair_token_ids.to(device)\n",
        "    mask_ids = mask_ids.to(device)\n",
        "    labels = y.to(device)\n",
        "\n",
        "    loss, prediction = model(pair_token_ids, \n",
        "                             token_type_ids=None, \n",
        "                             attention_mask=mask_ids, \n",
        "                             labels=labels).values()\n",
        "\n",
        "\n",
        "    acc = accuracy(prediction, labels)\n",
        "\n",
        "    f1 = metrics.f1_score(labels.cpu(), torch.argmax(prediction, -1).cpu(), labels=[0, 1, 2], average='macro')\n",
        "    precision = precision_score(labels.cpu(), torch.argmax(prediction, -1).cpu(),labels=[0, 1, 2], average='macro')\n",
        "    recall = recall_score(labels.cpu(), torch.argmax(prediction, -1).cpu(),labels=[0, 1, 2], average='macro')\n",
        "\n",
        "    total_test_acc  += acc.item()\n",
        "    total_F1_Score += f1\n",
        "    total_precision += precision\n",
        "    total_recall += recall\n",
        "  \n",
        "  test_acc  = total_test_acc/len(prediction_dataloader)\n",
        "  test_f1 = total_F1_Score/len(prediction_dataloader)\n",
        "  test_precision = total_precision/len(prediction_dataloader)\n",
        "  test_recall = total_recall/len(prediction_dataloader)\n",
        "\n",
        "\n",
        "  print(f'test_acc: {test_acc:.4f}')\n",
        "  print(f'f1 Score: {test_f1:.4f}')\n",
        "  print(f'precision: {test_precision:.4f}')\n",
        "  print(f'recall: {test_recall:.4f}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zP5PLHjQameD",
        "outputId": "db8947ae-3e6c-4a5c-9541-b73248a1f789"
      },
      "source": [
        "test(model,prediction_dataloader)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1515: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true nor predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  average, \"true nor predicted\", 'F-score is', len(true_sum)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "test_acc: 0.6337\n",
            "f1 Score: 0.5931\n",
            "precision: 0.6296\n",
            "recall: 0.6202\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MRNLvhI7Ggk7"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}