Sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "collapsed_sections": [
        "MXsu2HLI5tRh",
        "CmQxeGwK50xZ"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "fa315a42a89e4dde975dac86d0f5fb26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa6cc3c6fe2d43adad1273178bc373e2",
              "IPY_MODEL_abe6d34458ec498eb1031e565bde8e9b",
              "IPY_MODEL_9588845c2d1a439486c225b70f0cd43c"
            ],
            "layout": "IPY_MODEL_b6800aa585fd48b8b2b13e96b0dc7c34"
          }
        },
        "aa6cc3c6fe2d43adad1273178bc373e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b9b3213094f4196b605bca3a0620172",
            "placeholder": "​",
            "style": "IPY_MODEL_2e895f4861bf4a4b823ab62c1bd705e6",
            "value": "Downloading readme: 100%"
          }
        },
        "abe6d34458ec498eb1031e565bde8e9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4cc4dabf33424ba9a01cf1ea85243edd",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8586bcf0c933475e908adbf27180e057",
            "value": 22
          }
        },
        "9588845c2d1a439486c225b70f0cd43c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eaedfac59fd14d5aaf233602ec38eb85",
            "placeholder": "​",
            "style": "IPY_MODEL_d4fe70d56a55460c91491a7c5d26718e",
            "value": " 22.0/22.0 [00:00&lt;00:00, 1.37kB/s]"
          }
        },
        "b6800aa585fd48b8b2b13e96b0dc7c34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b9b3213094f4196b605bca3a0620172": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e895f4861bf4a4b823ab62c1bd705e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4cc4dabf33424ba9a01cf1ea85243edd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8586bcf0c933475e908adbf27180e057": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "eaedfac59fd14d5aaf233602ec38eb85": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4fe70d56a55460c91491a7c5d26718e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd67e52cd6d64dc395657d2ee038b1ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4873c3e2fe9f44ed9fab0de27c15737e",
              "IPY_MODEL_ccf0e00d720343d5aea6d98b55d03d50",
              "IPY_MODEL_718d5a4a106b434bbf1b10458b77db2b"
            ],
            "layout": "IPY_MODEL_7e5b8b0c20d046ac93f3d977325fb06a"
          }
        },
        "4873c3e2fe9f44ed9fab0de27c15737e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_514788897bfa4f76ae4c4874ec20b1bf",
            "placeholder": "​",
            "style": "IPY_MODEL_05752128e99f4f33b2f2e913e578a9e1",
            "value": "Downloading data files: 100%"
          }
        },
        "ccf0e00d720343d5aea6d98b55d03d50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_90787226e5af4aa68ee09fb7bb6d0305",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4305e0e8a84c4e24ae01c3eef3684259",
            "value": 2
          }
        },
        "718d5a4a106b434bbf1b10458b77db2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_313bd697bd524ed2bfa291d4fd7f6b2a",
            "placeholder": "​",
            "style": "IPY_MODEL_671e2d7a074c44d3bb938ab020bfca14",
            "value": " 2/2 [00:02&lt;00:00,  1.03s/it]"
          }
        },
        "7e5b8b0c20d046ac93f3d977325fb06a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "514788897bfa4f76ae4c4874ec20b1bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05752128e99f4f33b2f2e913e578a9e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90787226e5af4aa68ee09fb7bb6d0305": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4305e0e8a84c4e24ae01c3eef3684259": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "313bd697bd524ed2bfa291d4fd7f6b2a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "671e2d7a074c44d3bb938ab020bfca14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca198d6eaab5417bb85f09bdd827c63d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_84f3c6dbee8d4fd29720843b7516e3cb",
              "IPY_MODEL_5b7bbb10030e4b1190f5437b97e9dcbd",
              "IPY_MODEL_40509705beb4423db2d7c4dce120d288"
            ],
            "layout": "IPY_MODEL_a1a9147396ca4aec97834a6f3bb8941c"
          }
        },
        "84f3c6dbee8d4fd29720843b7516e3cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd300ef5ed304901b463767b5d526249",
            "placeholder": "​",
            "style": "IPY_MODEL_691f7c5be4c2462d9009297c6b404895",
            "value": "Downloading data: 100%"
          }
        },
        "5b7bbb10030e4b1190f5437b97e9dcbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7366eb9e63c04c91a6fbaa32b4ff609c",
            "max": 3633338,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0df93bd518df4c38b3a196ce6bab3d2d",
            "value": 3633338
          }
        },
        "40509705beb4423db2d7c4dce120d288": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84ee264011dc45ccb95c9b8d42c50393",
            "placeholder": "​",
            "style": "IPY_MODEL_581f388380a04bc993d60dcf1d4c08de",
            "value": " 3.63M/3.63M [00:00&lt;00:00, 6.85MB/s]"
          }
        },
        "a1a9147396ca4aec97834a6f3bb8941c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd300ef5ed304901b463767b5d526249": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "691f7c5be4c2462d9009297c6b404895": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7366eb9e63c04c91a6fbaa32b4ff609c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0df93bd518df4c38b3a196ce6bab3d2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "84ee264011dc45ccb95c9b8d42c50393": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "581f388380a04bc993d60dcf1d4c08de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "674b856d33154209aa8c9067cb044cf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_75c6afbd6c5d4d7f88a3dc21da6e6007",
              "IPY_MODEL_c8db0cbdb6b84e6cbea56676fcce5f1b",
              "IPY_MODEL_36887fdc940a46c68000a072313c7eb3"
            ],
            "layout": "IPY_MODEL_5fb83fd6ec404a33a22909a81833d10c"
          }
        },
        "75c6afbd6c5d4d7f88a3dc21da6e6007": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f34a5ea1babc4091b4fde940562fccd4",
            "placeholder": "​",
            "style": "IPY_MODEL_92ceaf2292794893bd25202a0feb345b",
            "value": "Downloading data: 100%"
          }
        },
        "c8db0cbdb6b84e6cbea56676fcce5f1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b101ba55d4c549a6baba1cefcb5d8e7a",
            "max": 465111,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_02a984bca8cf414daa8a5acffe414d65",
            "value": 465111
          }
        },
        "36887fdc940a46c68000a072313c7eb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_89257a1421044a02a9730f369e83af9d",
            "placeholder": "​",
            "style": "IPY_MODEL_005123f058854eb98e1e3c7b3ffb3b58",
            "value": " 465k/465k [00:00&lt;00:00, 15.9MB/s]"
          }
        },
        "5fb83fd6ec404a33a22909a81833d10c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f34a5ea1babc4091b4fde940562fccd4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92ceaf2292794893bd25202a0feb345b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b101ba55d4c549a6baba1cefcb5d8e7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02a984bca8cf414daa8a5acffe414d65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "89257a1421044a02a9730f369e83af9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "005123f058854eb98e1e3c7b3ffb3b58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b9a18fe7b9ff45b0be54cefaec75043b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e7f823171cc34644b670abf66242bad1",
              "IPY_MODEL_301aad102efd45a7a04bf6bf939f6d1e",
              "IPY_MODEL_ad66ddeb7766494ba4109bebf4b8e106"
            ],
            "layout": "IPY_MODEL_5ae7cd4448b341a29a9b155c93e31869"
          }
        },
        "e7f823171cc34644b670abf66242bad1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4b7c244823f4973958414539eb3fc7b",
            "placeholder": "​",
            "style": "IPY_MODEL_213cab146e4045d999c73a9b0ca26df0",
            "value": "Extracting data files: 100%"
          }
        },
        "301aad102efd45a7a04bf6bf939f6d1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_016174ab9dbf49d482212c81dbb981a7",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b7b30ca1dfd6469db7674100a7890103",
            "value": 2
          }
        },
        "ad66ddeb7766494ba4109bebf4b8e106": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8012464690414c639b204babbfdc19a4",
            "placeholder": "​",
            "style": "IPY_MODEL_a23967aefbd74a02a2851308c83faa34",
            "value": " 2/2 [00:00&lt;00:00, 100.90it/s]"
          }
        },
        "5ae7cd4448b341a29a9b155c93e31869": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4b7c244823f4973958414539eb3fc7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "213cab146e4045d999c73a9b0ca26df0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "016174ab9dbf49d482212c81dbb981a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b7b30ca1dfd6469db7674100a7890103": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8012464690414c639b204babbfdc19a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a23967aefbd74a02a2851308c83faa34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "25ff054bddf24315a1972aa2eef5d7c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_13b3f1986bd04067b679f1baf757128a",
              "IPY_MODEL_f2bb20f679ca42b9be09f6769fca2290",
              "IPY_MODEL_be46a78b903d4d1cab2f176cb923998d"
            ],
            "layout": "IPY_MODEL_ae9baeeb5c3e4939a84c688535241c6b"
          }
        },
        "13b3f1986bd04067b679f1baf757128a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_afb1d895736d49deba179344de13fb57",
            "placeholder": "​",
            "style": "IPY_MODEL_8c89101526a14c5cb98964b3477bf999",
            "value": "Generating train split: "
          }
        },
        "f2bb20f679ca42b9be09f6769fca2290": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf08a13eddbb4dc6b29817d5b5992887",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_106b3892ffef41a9afd0d894048fb2a6",
            "value": 1
          }
        },
        "be46a78b903d4d1cab2f176cb923998d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cce1fa1f22674728a2648acf60528ca5",
            "placeholder": "​",
            "style": "IPY_MODEL_46661fc816814188968cb1595f2a952b",
            "value": " 27481/0 [00:00&lt;00:00, 214972.37 examples/s]"
          }
        },
        "ae9baeeb5c3e4939a84c688535241c6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "afb1d895736d49deba179344de13fb57": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c89101526a14c5cb98964b3477bf999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cf08a13eddbb4dc6b29817d5b5992887": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "106b3892ffef41a9afd0d894048fb2a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cce1fa1f22674728a2648acf60528ca5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46661fc816814188968cb1595f2a952b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0bc843baf5794f87ba87fdcf85840971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_11cfd5db51a14894b229a1d3fad868c5",
              "IPY_MODEL_075b86b342ed4e3c9c1cfc387939ca68",
              "IPY_MODEL_87f20690b7084c1999d3b64510048b63"
            ],
            "layout": "IPY_MODEL_4953746e7b9d4c5fb8d1204c4536ea18"
          }
        },
        "11cfd5db51a14894b229a1d3fad868c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c2792bb1efc4d29a9bb2424253c85ce",
            "placeholder": "​",
            "style": "IPY_MODEL_5d82a82e18b74292ba2a8762ae34bf8c",
            "value": "Generating test split: "
          }
        },
        "075b86b342ed4e3c9c1cfc387939ca68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "info",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f541c5a6a007405d9d1bacec8801c3e8",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_50b509d16a4c4f258bc3698f82fc3a16",
            "value": 1
          }
        },
        "87f20690b7084c1999d3b64510048b63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d764e4a32b0549e6b781c6187fb286ef",
            "placeholder": "​",
            "style": "IPY_MODEL_ababfdc1cb034c10a029e484f215dfa3",
            "value": " 0/0 [00:00&lt;?, ? examples/s]"
          }
        },
        "4953746e7b9d4c5fb8d1204c4536ea18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "0c2792bb1efc4d29a9bb2424253c85ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d82a82e18b74292ba2a8762ae34bf8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f541c5a6a007405d9d1bacec8801c3e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "50b509d16a4c4f258bc3698f82fc3a16": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d764e4a32b0549e6b781c6187fb286ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ababfdc1cb034c10a029e484f215dfa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1515c9aac6db4aa895b3407e4bce5387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_348e91cd883d43c9afb81986917f4f92",
              "IPY_MODEL_de686e6daed747799b53dec0ef565876",
              "IPY_MODEL_1420b058c09e40acb7314c87f6a5240c"
            ],
            "layout": "IPY_MODEL_1caa3c8ea0834734aad5132e7a755afb"
          }
        },
        "348e91cd883d43c9afb81986917f4f92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6cb89871487e47299a3c71b1d8389d8c",
            "placeholder": "​",
            "style": "IPY_MODEL_1a78ad818c9c498ea4165fcb1efd755e",
            "value": "100%"
          }
        },
        "de686e6daed747799b53dec0ef565876": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6445afbab9af465db73e2cecaf862dfd",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e5d1b05b7c844b1fa0e1eabcb1f04d14",
            "value": 2
          }
        },
        "1420b058c09e40acb7314c87f6a5240c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_082d5ffed60942fe9f0c559dd23350cb",
            "placeholder": "​",
            "style": "IPY_MODEL_294354385df74f1b86b2033ac8730aea",
            "value": " 2/2 [00:00&lt;00:00, 53.86it/s]"
          }
        },
        "1caa3c8ea0834734aad5132e7a755afb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6cb89871487e47299a3c71b1d8389d8c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a78ad818c9c498ea4165fcb1efd755e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6445afbab9af465db73e2cecaf862dfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5d1b05b7c844b1fa0e1eabcb1f04d14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "082d5ffed60942fe9f0c559dd23350cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "294354385df74f1b86b2033ac8730aea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c04a28f0107846099e4a9fd8cd757927": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_82693147e592427696e2b30291217344",
              "IPY_MODEL_c109dd449d4a47e99ee16aaace8cfd1e",
              "IPY_MODEL_f4ecf83e19ae4bf8989d890034c3b082"
            ],
            "layout": "IPY_MODEL_e76e401670a749998bb827c9e64d1109"
          }
        },
        "82693147e592427696e2b30291217344": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_958997ab5a844dc298917f6dc6fa4282",
            "placeholder": "​",
            "style": "IPY_MODEL_a0eb882007bb422b92828a441fabf534",
            "value": "config.json: 100%"
          }
        },
        "c109dd449d4a47e99ee16aaace8cfd1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3724d1a6f396425f8ac300fce8b9ef0f",
            "max": 1404,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_17e7590e9be740f5a6c7c98701e5345f",
            "value": 1404
          }
        },
        "f4ecf83e19ae4bf8989d890034c3b082": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e308beca84404b7a801c1475db8ee224",
            "placeholder": "​",
            "style": "IPY_MODEL_25b45337013e4aa1899f68b1743f81f5",
            "value": " 1.40k/1.40k [00:00&lt;00:00, 98.7kB/s]"
          }
        },
        "e76e401670a749998bb827c9e64d1109": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "958997ab5a844dc298917f6dc6fa4282": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0eb882007bb422b92828a441fabf534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3724d1a6f396425f8ac300fce8b9ef0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17e7590e9be740f5a6c7c98701e5345f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e308beca84404b7a801c1475db8ee224": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25b45337013e4aa1899f68b1743f81f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8d44537a13d24dd7bcad23a09c44430c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f4712c6529344627b1ce2ed2916616de",
              "IPY_MODEL_1e1ad980551648e79e44c51e56c4279b",
              "IPY_MODEL_c1033af6365744d18031677d22e3cdd8"
            ],
            "layout": "IPY_MODEL_50453614545b423798f13927de392cca"
          }
        },
        "f4712c6529344627b1ce2ed2916616de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81bdc0ace86b4edaabd287acf25d9e76",
            "placeholder": "​",
            "style": "IPY_MODEL_46f7015b0b2e461b984f9d4b163a5c17",
            "value": "model.safetensors: 100%"
          }
        },
        "1e1ad980551648e79e44c51e56c4279b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84d4704e07ba452d9175f956fbc1f108",
            "max": 990345061,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6aadf6f6d5494751b1e82bc7608acc46",
            "value": 990345061
          }
        },
        "c1033af6365744d18031677d22e3cdd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a73e6e041f2346bd8257f9d3845ee9d1",
            "placeholder": "​",
            "style": "IPY_MODEL_ea86fb104fa14e4994d367ded8adb995",
            "value": " 990M/990M [00:04&lt;00:00, 150MB/s]"
          }
        },
        "50453614545b423798f13927de392cca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81bdc0ace86b4edaabd287acf25d9e76": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46f7015b0b2e461b984f9d4b163a5c17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84d4704e07ba452d9175f956fbc1f108": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6aadf6f6d5494751b1e82bc7608acc46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a73e6e041f2346bd8257f9d3845ee9d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea86fb104fa14e4994d367ded8adb995": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a3f5115349064533844473a0d013a794": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_59d27aa07c96424ea11c07f979b5fc5f",
              "IPY_MODEL_d31f9c19bef74bc5abc79cde9d98a2e2",
              "IPY_MODEL_ee92bf7081c94bb495564eea0d6bdcc1"
            ],
            "layout": "IPY_MODEL_8d09e7956ae8432eb2eac998afc4e486"
          }
        },
        "59d27aa07c96424ea11c07f979b5fc5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64cee5a704f94ccd93ff8d35f753f0f8",
            "placeholder": "​",
            "style": "IPY_MODEL_436816ee5f96461da2930ee2a4beebc7",
            "value": "generation_config.json: 100%"
          }
        },
        "d31f9c19bef74bc5abc79cde9d98a2e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8633a7fbb3d4a17a395aac4bb385bc1",
            "max": 147,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5057a710468844f38e0621d3465c2d9f",
            "value": 147
          }
        },
        "ee92bf7081c94bb495564eea0d6bdcc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd155ec4b4e94cb3bdb07d2f2e8b8124",
            "placeholder": "​",
            "style": "IPY_MODEL_5fdbfada98ac4587acfa4df77aedcdec",
            "value": " 147/147 [00:00&lt;00:00, 3.97kB/s]"
          }
        },
        "8d09e7956ae8432eb2eac998afc4e486": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64cee5a704f94ccd93ff8d35f753f0f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "436816ee5f96461da2930ee2a4beebc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b8633a7fbb3d4a17a395aac4bb385bc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5057a710468844f38e0621d3465c2d9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd155ec4b4e94cb3bdb07d2f2e8b8124": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5fdbfada98ac4587acfa4df77aedcdec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "259b48cef71f4e9eae7c2e90913a68e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3bf47f86b4754111ae9dc6f8160b373a",
              "IPY_MODEL_6b6e537c13874ccea66138f1425f8741",
              "IPY_MODEL_d95aac342e564b60b9044f6e9a51c878"
            ],
            "layout": "IPY_MODEL_bd76230047964a6fa458804dc72d8dd6"
          }
        },
        "3bf47f86b4754111ae9dc6f8160b373a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5a439fc30b84059a3cc544be9f08457",
            "placeholder": "​",
            "style": "IPY_MODEL_1f45d629088b4297b07631e316bd9db4",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "6b6e537c13874ccea66138f1425f8741": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f15103066944f639eecf79d2e4c5a35",
            "max": 2537,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ed5693073e84432a99bb3b5f5cf02b51",
            "value": 2537
          }
        },
        "d95aac342e564b60b9044f6e9a51c878": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_68c71841225442f0b971c3fe8d2afbd5",
            "placeholder": "​",
            "style": "IPY_MODEL_d5f4703c62a84592a9c243b72a189501",
            "value": " 2.54k/2.54k [00:00&lt;00:00, 48.7kB/s]"
          }
        },
        "bd76230047964a6fa458804dc72d8dd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5a439fc30b84059a3cc544be9f08457": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f45d629088b4297b07631e316bd9db4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f15103066944f639eecf79d2e4c5a35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed5693073e84432a99bb3b5f5cf02b51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "68c71841225442f0b971c3fe8d2afbd5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5f4703c62a84592a9c243b72a189501": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "00dc6ed907c84963a1da385b940d32e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f7186ea58ca4ce298d901e367d2f22f",
              "IPY_MODEL_983074c1ba4645e09a60b6626aafb56d",
              "IPY_MODEL_adc9d728aa9d4bc0a616f3a9f7d53ebf"
            ],
            "layout": "IPY_MODEL_5180cb9fd5834f02b875e04fe73395a9"
          }
        },
        "4f7186ea58ca4ce298d901e367d2f22f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cad42931aa544ae380d8161b13717dae",
            "placeholder": "​",
            "style": "IPY_MODEL_cb49263d767a4d41aeb91e725e32afed",
            "value": "spiece.model: 100%"
          }
        },
        "983074c1ba4645e09a60b6626aafb56d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a7fe4f390e6b49d3b9e5602ea49f0f35",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fa6d0e9597224b4c80b2e5fe3035fb71",
            "value": 791656
          }
        },
        "adc9d728aa9d4bc0a616f3a9f7d53ebf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c32b998fc44460c8741c8a42741d0ba",
            "placeholder": "​",
            "style": "IPY_MODEL_844e6a4873d647f48a2ae140cb678834",
            "value": " 792k/792k [00:00&lt;00:00, 11.9MB/s]"
          }
        },
        "5180cb9fd5834f02b875e04fe73395a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cad42931aa544ae380d8161b13717dae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb49263d767a4d41aeb91e725e32afed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a7fe4f390e6b49d3b9e5602ea49f0f35": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa6d0e9597224b4c80b2e5fe3035fb71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3c32b998fc44460c8741c8a42741d0ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "844e6a4873d647f48a2ae140cb678834": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3243d6c5151e432db2b4cc0a7a074cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_27c523ef742a4c8e9d6d2d08b443c2d1",
              "IPY_MODEL_ddbd2cfc385d4f479cfbc5290eacd7dd",
              "IPY_MODEL_4d1720c2bdf04a4a9a3009f2c717bff0"
            ],
            "layout": "IPY_MODEL_aaaecb19907f4800a5f0fda1e02c64a0"
          }
        },
        "27c523ef742a4c8e9d6d2d08b443c2d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d678d1af21ea4f7aa98d83147982a548",
            "placeholder": "​",
            "style": "IPY_MODEL_5ea4aa09209c458c8e301c449423e245",
            "value": "tokenizer.json: 100%"
          }
        },
        "ddbd2cfc385d4f479cfbc5290eacd7dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_84328b93859743a492a9854e9d725aec",
            "max": 2424064,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d4c47f06da60453ea3bd8e5bfbc32d9d",
            "value": 2424064
          }
        },
        "4d1720c2bdf04a4a9a3009f2c717bff0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6aa638dad108411cab9085aaea04b977",
            "placeholder": "​",
            "style": "IPY_MODEL_9c6a1a729256471288d4fc744ce77cad",
            "value": " 2.42M/2.42M [00:01&lt;00:00, 2.27MB/s]"
          }
        },
        "aaaecb19907f4800a5f0fda1e02c64a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d678d1af21ea4f7aa98d83147982a548": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ea4aa09209c458c8e301c449423e245": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "84328b93859743a492a9854e9d725aec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4c47f06da60453ea3bd8e5bfbc32d9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6aa638dad108411cab9085aaea04b977": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c6a1a729256471288d4fc744ce77cad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64e1646abe6049fa836cc87433066dc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b3aad4a6391b4b71b2e3d825ff5deb68",
              "IPY_MODEL_ec850d668030446d87ace2ba7723c89c",
              "IPY_MODEL_1cca4e1784fa4f7c87219971c0179c26"
            ],
            "layout": "IPY_MODEL_61588c54d0234d5ba298695b1791a2d6"
          }
        },
        "b3aad4a6391b4b71b2e3d825ff5deb68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e162bdaa9014495be31c241b68b45d2",
            "placeholder": "​",
            "style": "IPY_MODEL_53c9dd79c1db418197250f4e5f3abde5",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "ec850d668030446d87ace2ba7723c89c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5baea4792e443ee92b85a897b59577e",
            "max": 2201,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c8238751a79841e1a1e380eef5579794",
            "value": 2201
          }
        },
        "1cca4e1784fa4f7c87219971c0179c26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f6da7d3b6954275889b9faae85eb73a",
            "placeholder": "​",
            "style": "IPY_MODEL_597c37d9d48b4c8ea2cabd23032a29f3",
            "value": " 2.20k/2.20k [00:00&lt;00:00, 51.4kB/s]"
          }
        },
        "61588c54d0234d5ba298695b1791a2d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e162bdaa9014495be31c241b68b45d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "53c9dd79c1db418197250f4e5f3abde5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5baea4792e443ee92b85a897b59577e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8238751a79841e1a1e380eef5579794": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6f6da7d3b6954275889b9faae85eb73a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "597c37d9d48b4c8ea2cabd23032a29f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7d1941d77cee464b99b6e1ec22652356": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c17ff21fbadf416fbcf33e29931430a4",
              "IPY_MODEL_fe4fb6fb669143e1ab64f68ba73fd069",
              "IPY_MODEL_d02e589963c14bdb80fd437c8dc787b7"
            ],
            "layout": "IPY_MODEL_9d7187b6dded4ca6b4f447aee2282b68"
          }
        },
        "c17ff21fbadf416fbcf33e29931430a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2483db5361cd4009af9a9fc36299de16",
            "placeholder": "​",
            "style": "IPY_MODEL_c50a280ac3154f218750079d1bd057f9",
            "value": "Map: 100%"
          }
        },
        "fe4fb6fb669143e1ab64f68ba73fd069": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9004c730369a4bd1825bc3092f3b3131",
            "max": 27481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_907cf2503521408082e3e035f635e006",
            "value": 27481
          }
        },
        "d02e589963c14bdb80fd437c8dc787b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97df134d8e294756ad1ebf2be81a2e4b",
            "placeholder": "​",
            "style": "IPY_MODEL_01edcec6037542e98815091b9135c2c9",
            "value": " 27481/27481 [00:12&lt;00:00, 2372.35 examples/s]"
          }
        },
        "9d7187b6dded4ca6b4f447aee2282b68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "2483db5361cd4009af9a9fc36299de16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c50a280ac3154f218750079d1bd057f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9004c730369a4bd1825bc3092f3b3131": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "907cf2503521408082e3e035f635e006": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "97df134d8e294756ad1ebf2be81a2e4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01edcec6037542e98815091b9135c2c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "395665fc5f844aa0890792b51fc73542": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1a1599a7fca547cc961eca6bc4db4702",
              "IPY_MODEL_2d5c8c790d70468ba03bc0bba9424cc4",
              "IPY_MODEL_914313f0d7584c5d80c6fe13c7cd3527"
            ],
            "layout": "IPY_MODEL_7b1136e6579d4b07ae12772651ebf64a"
          }
        },
        "1a1599a7fca547cc961eca6bc4db4702": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5bb44ff635814e0191bd577f52a02ea6",
            "placeholder": "​",
            "style": "IPY_MODEL_fefcdb2079614dbf9efedaa4cca11f53",
            "value": "Map: 100%"
          }
        },
        "2d5c8c790d70468ba03bc0bba9424cc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9f6ab57d69e4e83bcf5c35e6819b7e9",
            "max": 3534,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f3b8926b3f1342739287e2435ef92c71",
            "value": 3534
          }
        },
        "914313f0d7584c5d80c6fe13c7cd3527": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9d00af3444f49a68b5d58c95967dde8",
            "placeholder": "​",
            "style": "IPY_MODEL_4b5657e9cf0a4117893eca4a4a54ae36",
            "value": " 3534/3534 [00:01&lt;00:00, 2865.29 examples/s]"
          }
        },
        "7b1136e6579d4b07ae12772651ebf64a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "5bb44ff635814e0191bd577f52a02ea6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fefcdb2079614dbf9efedaa4cca11f53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9f6ab57d69e4e83bcf5c35e6819b7e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f3b8926b3f1342739287e2435ef92c71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f9d00af3444f49a68b5d58c95967dde8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b5657e9cf0a4117893eca4a4a54ae36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b09ee8f4e0d8477e8de4bbb322b88c1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0af60aa599814a7c8e1e2f6482db5fb2",
              "IPY_MODEL_bb53b029cce8462490184eff62036ae4",
              "IPY_MODEL_23a8af087ce54fb5a877aabd22878ccd"
            ],
            "layout": "IPY_MODEL_13e61905a7e34694b8b919c0b19edcbe"
          }
        },
        "0af60aa599814a7c8e1e2f6482db5fb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1205008e62a5412ca720131248911091",
            "placeholder": "​",
            "style": "IPY_MODEL_b6e5cfd9de694609bb360f511277e1f9",
            "value": "Filter: 100%"
          }
        },
        "bb53b029cce8462490184eff62036ae4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b06e6951fbec46d0bb5561e835d68e11",
            "max": 27481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_292c734c20964d299b6bbec7b9d088e6",
            "value": 27481
          }
        },
        "23a8af087ce54fb5a877aabd22878ccd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5f0570fb278c4fe7aa742ac40fe2e0dc",
            "placeholder": "​",
            "style": "IPY_MODEL_6952242dc85f49cf95b20796564d81dc",
            "value": " 27481/27481 [00:13&lt;00:00, 1446.71 examples/s]"
          }
        },
        "13e61905a7e34694b8b919c0b19edcbe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "1205008e62a5412ca720131248911091": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b6e5cfd9de694609bb360f511277e1f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b06e6951fbec46d0bb5561e835d68e11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "292c734c20964d299b6bbec7b9d088e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5f0570fb278c4fe7aa742ac40fe2e0dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6952242dc85f49cf95b20796564d81dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f19c4e3657764979be54ae160b15c317": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_eb3860488e334dbd9fc42ee0a40aa167",
              "IPY_MODEL_f411c3f9291f42f9909be260e69de775",
              "IPY_MODEL_3feb5bafda1c409bb1d7715f6d035f71"
            ],
            "layout": "IPY_MODEL_e3a7c3e182cc48ddbc46db4adda24164"
          }
        },
        "eb3860488e334dbd9fc42ee0a40aa167": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c862d57d0f1b40c9aefcab273dc3bd1a",
            "placeholder": "​",
            "style": "IPY_MODEL_682739da17f049f89a02ecef101faabb",
            "value": "Filter: 100%"
          }
        },
        "f411c3f9291f42f9909be260e69de775": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2436cb1cc484062ae70eae1d2dea7c8",
            "max": 3534,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_945694b20c0e41a1ad7707e0830a4b9c",
            "value": 3534
          }
        },
        "3feb5bafda1c409bb1d7715f6d035f71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9da0be432fc40e081a1bc6524eeb037",
            "placeholder": "​",
            "style": "IPY_MODEL_1dc8a33f68ac4cb9a3201d2ccf0fec6c",
            "value": " 3534/3534 [00:01&lt;00:00, 2014.07 examples/s]"
          }
        },
        "e3a7c3e182cc48ddbc46db4adda24164": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "c862d57d0f1b40c9aefcab273dc3bd1a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "682739da17f049f89a02ecef101faabb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2436cb1cc484062ae70eae1d2dea7c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "945694b20c0e41a1ad7707e0830a4b9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f9da0be432fc40e081a1bc6524eeb037": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1dc8a33f68ac4cb9a3201d2ccf0fec6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b487512585824bbe85ba3971177cd295": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_82776a3ee8234f598ad4d874250b9086",
              "IPY_MODEL_5779147f16c7422f9c685a1406a0fa49",
              "IPY_MODEL_13842ad44c4a46b99c46be0055f249bf"
            ],
            "layout": "IPY_MODEL_b5061c8e233949869dfe8ae2b00106fa"
          }
        },
        "82776a3ee8234f598ad4d874250b9086": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b90502f06ca541bea8994bd8f51f3211",
            "placeholder": "​",
            "style": "IPY_MODEL_a2b5d813a32a435cb4866af3c201738b",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "5779147f16c7422f9c685a1406a0fa49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_48b0ae690a4441fea4e95503fe8d6287",
            "max": 2537,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bf53071aaa384114854e4e7fb8cbbe75",
            "value": 2537
          }
        },
        "13842ad44c4a46b99c46be0055f249bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f8dae7f260c343ac8c40c2fa59b7a6f3",
            "placeholder": "​",
            "style": "IPY_MODEL_4ffca50e1aa741e6b6ea1552080a7cd0",
            "value": " 2.54k/2.54k [00:00&lt;00:00, 165kB/s]"
          }
        },
        "b5061c8e233949869dfe8ae2b00106fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b90502f06ca541bea8994bd8f51f3211": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2b5d813a32a435cb4866af3c201738b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "48b0ae690a4441fea4e95503fe8d6287": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf53071aaa384114854e4e7fb8cbbe75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f8dae7f260c343ac8c40c2fa59b7a6f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ffca50e1aa741e6b6ea1552080a7cd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9811eff0f3b042f294ec5fdf98122c5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e1518b42f3b0426086d0e18b28a3b447",
              "IPY_MODEL_9565180c73b8409aa4050ab0e693a2ec",
              "IPY_MODEL_eda09b9de67348bfb58e5e4d0a10e48b"
            ],
            "layout": "IPY_MODEL_beea6e223bf8434aa7ead688555a1f86"
          }
        },
        "e1518b42f3b0426086d0e18b28a3b447": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c17d0f92f46a42838a7fa0bf2bf03d11",
            "placeholder": "​",
            "style": "IPY_MODEL_722ada0e93064d90899d36fcb015b4e1",
            "value": "spiece.model: 100%"
          }
        },
        "9565180c73b8409aa4050ab0e693a2ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_40a1c955463f41aba8081de32320b3c0",
            "max": 791656,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f80e3c870c174e67bb2132fc1ba153b6",
            "value": 791656
          }
        },
        "eda09b9de67348bfb58e5e4d0a10e48b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f85812f47eeb46f6baf07f6b832d297a",
            "placeholder": "​",
            "style": "IPY_MODEL_ba1130807c4840338e6f50d14d5d5897",
            "value": " 792k/792k [00:00&lt;00:00, 38.0MB/s]"
          }
        },
        "beea6e223bf8434aa7ead688555a1f86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c17d0f92f46a42838a7fa0bf2bf03d11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "722ada0e93064d90899d36fcb015b4e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "40a1c955463f41aba8081de32320b3c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f80e3c870c174e67bb2132fc1ba153b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f85812f47eeb46f6baf07f6b832d297a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba1130807c4840338e6f50d14d5d5897": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0d87c3ea49364d418940a21ce4852500": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aed034790b134350bcf3231f54b724ae",
              "IPY_MODEL_0aa1967736c4441b93fb335ae22bd465",
              "IPY_MODEL_55f80eaecce94f18b8b81fe0cd86c6fa"
            ],
            "layout": "IPY_MODEL_9a68adc65f9341cd8e021963cf33617a"
          }
        },
        "aed034790b134350bcf3231f54b724ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0b0fb9b691bc451e939200eca050cea1",
            "placeholder": "​",
            "style": "IPY_MODEL_7a38e14610a14d3fad231be756a6657f",
            "value": "tokenizer.json: 100%"
          }
        },
        "0aa1967736c4441b93fb335ae22bd465": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ec78b7130014ce79aa9971b6172112a",
            "max": 2422164,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_32227dd9cc9f45358b0e8a8c96692cee",
            "value": 2422164
          }
        },
        "55f80eaecce94f18b8b81fe0cd86c6fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_da14c6bd122040d7b1123aba2972217d",
            "placeholder": "​",
            "style": "IPY_MODEL_5ba2bb188f574123a7657a4b3fa96dfd",
            "value": " 2.42M/2.42M [00:00&lt;00:00, 9.38MB/s]"
          }
        },
        "9a68adc65f9341cd8e021963cf33617a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b0fb9b691bc451e939200eca050cea1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a38e14610a14d3fad231be756a6657f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ec78b7130014ce79aa9971b6172112a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "32227dd9cc9f45358b0e8a8c96692cee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "da14c6bd122040d7b1123aba2972217d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ba2bb188f574123a7657a4b3fa96dfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0e9955a9b4d44460b19f46a73c189514": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_41e6191348c14dc3a078f46e23ca551d",
              "IPY_MODEL_4555d0ccd2ba494d9fd5bb3281635021",
              "IPY_MODEL_b29158ae7a9e40d998234b4c93e7e8b0"
            ],
            "layout": "IPY_MODEL_03d3533b81614153b017ae6d31a90ae0"
          }
        },
        "41e6191348c14dc3a078f46e23ca551d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4c8ac0ee212f4bfa8fc90101bc473d00",
            "placeholder": "​",
            "style": "IPY_MODEL_5c127dc1afde4a579fd0b05226915e9e",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "4555d0ccd2ba494d9fd5bb3281635021": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b5cf5fe5eba44a9d95ae1c4451ba4a88",
            "max": 2201,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3805bfeaa3c3402888ff0391ede18eea",
            "value": 2201
          }
        },
        "b29158ae7a9e40d998234b4c93e7e8b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0efada7fd17041c1910bb96dd8aba9a6",
            "placeholder": "​",
            "style": "IPY_MODEL_0c6ab74eeb4e46218c9259d871d2fe58",
            "value": " 2.20k/2.20k [00:00&lt;00:00, 95.7kB/s]"
          }
        },
        "03d3533b81614153b017ae6d31a90ae0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c8ac0ee212f4bfa8fc90101bc473d00": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c127dc1afde4a579fd0b05226915e9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5cf5fe5eba44a9d95ae1c4451ba4a88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3805bfeaa3c3402888ff0391ede18eea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0efada7fd17041c1910bb96dd8aba9a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c6ab74eeb4e46218c9259d871d2fe58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "488ff28e86a34713be5332eb06e0f00d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bd35dc7bca9d4c218da6379529daea44",
              "IPY_MODEL_8d810525fcdb4ae8adb7b3e429bd82d9",
              "IPY_MODEL_5f490c826150418b9557eec9530559b4"
            ],
            "layout": "IPY_MODEL_cf7d1e43569642ceaf3094e0e8f0ec37"
          }
        },
        "bd35dc7bca9d4c218da6379529daea44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_45e8dfa9c3284ed8b9541cc06deb0a91",
            "placeholder": "​",
            "style": "IPY_MODEL_c8b7bfbe5c6a4f3291974f079cba6a82",
            "value": "config.json: 100%"
          }
        },
        "8d810525fcdb4ae8adb7b3e429bd82d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5839ab26f354dbdbcf70091b90afdfb",
            "max": 1528,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_78a97a70cf434d7d93580b76612c8429",
            "value": 1528
          }
        },
        "5f490c826150418b9557eec9530559b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28e913aa6d234d5697a11b6bad15fe7b",
            "placeholder": "​",
            "style": "IPY_MODEL_0d5d4bd186084a669375a3f6d9e133d6",
            "value": " 1.53k/1.53k [00:00&lt;00:00, 84.3kB/s]"
          }
        },
        "cf7d1e43569642ceaf3094e0e8f0ec37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45e8dfa9c3284ed8b9541cc06deb0a91": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8b7bfbe5c6a4f3291974f079cba6a82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5839ab26f354dbdbcf70091b90afdfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78a97a70cf434d7d93580b76612c8429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "28e913aa6d234d5697a11b6bad15fe7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0d5d4bd186084a669375a3f6d9e133d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0389f8778d6e465c81c37b46fa052ab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b9362ac6a16b4273902e263545d6c2a7",
              "IPY_MODEL_75c8959d820542aba2d12b9a4f21adaa",
              "IPY_MODEL_2fa0fc8d8c1442b6a7ffc9c2bf7e938a"
            ],
            "layout": "IPY_MODEL_03d26d66fbf8414b8456dc969f1665d1"
          }
        },
        "b9362ac6a16b4273902e263545d6c2a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4034dcd63de74bcf8bb3291fd78d876a",
            "placeholder": "​",
            "style": "IPY_MODEL_41f4a615779a4179943c870ebd443db9",
            "value": "pytorch_model.bin: 100%"
          }
        },
        "75c8959d820542aba2d12b9a4f21adaa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4f068e2532a247c684eec557100ec4d7",
            "max": 990406605,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d4098b9628884e5bab9d3e1840317a54",
            "value": 990406605
          }
        },
        "2fa0fc8d8c1442b6a7ffc9c2bf7e938a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1754222def1d4f0d9df1c8b3bfdec225",
            "placeholder": "​",
            "style": "IPY_MODEL_7af63470071e43afa7db30acf4be33aa",
            "value": " 990M/990M [01:01&lt;00:00, 15.0MB/s]"
          }
        },
        "03d26d66fbf8414b8456dc969f1665d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4034dcd63de74bcf8bb3291fd78d876a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "41f4a615779a4179943c870ebd443db9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f068e2532a247c684eec557100ec4d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4098b9628884e5bab9d3e1840317a54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1754222def1d4f0d9df1c8b3bfdec225": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7af63470071e43afa7db30acf4be33aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "29990dec26b14bfaad79aa51eef2f550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f92c91c375d84ed6ac04d199fe78c73a",
              "IPY_MODEL_4ae102c7d51346e8a3359b6e8d14e584",
              "IPY_MODEL_111b52b3fa944a5d833404e44dae996c"
            ],
            "layout": "IPY_MODEL_8495f79bd14948a396322793135b1bc7"
          }
        },
        "f92c91c375d84ed6ac04d199fe78c73a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2cd3678ea1a34af38775535d882e2df5",
            "placeholder": "​",
            "style": "IPY_MODEL_d9c8445f3cd04b9fb328eede838ebedb",
            "value": "Downloading builder script: 100%"
          }
        },
        "4ae102c7d51346e8a3359b6e8d14e584": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_972757171ecb493d8c30e2af1ec2e08a",
            "max": 6270,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_61cd1dc950f44d0888d09d893e9881a2",
            "value": 6270
          }
        },
        "111b52b3fa944a5d833404e44dae996c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67801014c2d24f2bb4062e7da01f4c12",
            "placeholder": "​",
            "style": "IPY_MODEL_2b235e38d57b49b9bd02a8934d111a6e",
            "value": " 6.27k/6.27k [00:00&lt;00:00, 295kB/s]"
          }
        },
        "8495f79bd14948a396322793135b1bc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cd3678ea1a34af38775535d882e2df5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d9c8445f3cd04b9fb328eede838ebedb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "972757171ecb493d8c30e2af1ec2e08a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61cd1dc950f44d0888d09d893e9881a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67801014c2d24f2bb4062e7da01f4c12": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b235e38d57b49b9bd02a8934d111a6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5375818c120d48e0a595372254c38a5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a967b78a161d4382b0815d570dd1b14a",
              "IPY_MODEL_cde4d86e76ad4b88863d8c1cd3ab06a4",
              "IPY_MODEL_06d481e527504722803437906aa334aa"
            ],
            "layout": "IPY_MODEL_22fc417159d74e37835241c8cc3031e3"
          }
        },
        "a967b78a161d4382b0815d570dd1b14a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_953686abe5374d81a5444f2d49a9dfa7",
            "placeholder": "​",
            "style": "IPY_MODEL_f2d245b178e2401a89fef0759277b1f7",
            "value": "Filter:  69%"
          }
        },
        "cde4d86e76ad4b88863d8c1cd3ab06a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9d11bc9015a4bdca2ad25d55dbfe4c3",
            "max": 27481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c67af42a7eb740ee95b402693b746f7a",
            "value": 27481
          }
        },
        "06d481e527504722803437906aa334aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_30fe5bf925fc49a4b18f3e5323678a06",
            "placeholder": "​",
            "style": "IPY_MODEL_f5bb05b04b8c4d9f84a64511129440b9",
            "value": " 19000/27481 [00:00&lt;00:00, 156096.47 examples/s]"
          }
        },
        "22fc417159d74e37835241c8cc3031e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "953686abe5374d81a5444f2d49a9dfa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2d245b178e2401a89fef0759277b1f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b9d11bc9015a4bdca2ad25d55dbfe4c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c67af42a7eb740ee95b402693b746f7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "30fe5bf925fc49a4b18f3e5323678a06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f5bb05b04b8c4d9f84a64511129440b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "215e4ff9678948648ef8d30362315020": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f6e2a356684a4649ac14f03bc3c5263c",
              "IPY_MODEL_71b3a8504810408c9524e6f78af3bb8d",
              "IPY_MODEL_9096e9620a324cec8bd3acfdc4ec14c2"
            ],
            "layout": "IPY_MODEL_5f958662a19648e79b0a41ee5d005483"
          }
        },
        "f6e2a356684a4649ac14f03bc3c5263c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb0569828b0248ebbfa7d87fe464c274",
            "placeholder": "​",
            "style": "IPY_MODEL_ac653d35316341879f1dff4be7601d4c",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "71b3a8504810408c9524e6f78af3bb8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_690dfd6fd5df4041989f63b6df9f68b4",
            "max": 1106,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d2341aca405c41f4a420c4bbe8fa0420",
            "value": 1106
          }
        },
        "9096e9620a324cec8bd3acfdc4ec14c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72142749796149c1bfe96446a590f592",
            "placeholder": "​",
            "style": "IPY_MODEL_d49aeac061d946a1968711636c38847d",
            "value": " 1.11k/1.11k [00:00&lt;00:00, 73.8kB/s]"
          }
        },
        "5f958662a19648e79b0a41ee5d005483": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb0569828b0248ebbfa7d87fe464c274": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac653d35316341879f1dff4be7601d4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "690dfd6fd5df4041989f63b6df9f68b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2341aca405c41f4a420c4bbe8fa0420": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72142749796149c1bfe96446a590f592": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d49aeac061d946a1968711636c38847d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d067de736c604e529df81241a3061e7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b7df1d0926fc43469c4431e131b11074",
              "IPY_MODEL_dcbfd87d665f420a8505c44ea05a17f2",
              "IPY_MODEL_bf3d9e4b09b540038c586f8b17eaedf3"
            ],
            "layout": "IPY_MODEL_797891b7e649429e8c0c50bc9e0571ba"
          }
        },
        "b7df1d0926fc43469c4431e131b11074": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfece662a0454a81988ee61971646cb6",
            "placeholder": "​",
            "style": "IPY_MODEL_bdd655e2b6f24470ab7d25baeecd18bf",
            "value": "vocab.json: 100%"
          }
        },
        "dcbfd87d665f420a8505c44ea05a17f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4527f9a9122c459cab5e6d6b43d8960c",
            "max": 898822,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5417e9962e0c439f88f41264cd18972a",
            "value": 898822
          }
        },
        "bf3d9e4b09b540038c586f8b17eaedf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f68b6020dbae4ea9b944aa8c0ff54729",
            "placeholder": "​",
            "style": "IPY_MODEL_016c65063e6640069210fab18c529e2a",
            "value": " 899k/899k [00:00&lt;00:00, 24.2MB/s]"
          }
        },
        "797891b7e649429e8c0c50bc9e0571ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfece662a0454a81988ee61971646cb6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bdd655e2b6f24470ab7d25baeecd18bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4527f9a9122c459cab5e6d6b43d8960c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5417e9962e0c439f88f41264cd18972a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f68b6020dbae4ea9b944aa8c0ff54729": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "016c65063e6640069210fab18c529e2a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "681d1db7a53945139586bd3bf6ec858f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5a4a0a6521c844a0b536430884aa4bd9",
              "IPY_MODEL_03a08c52cc804e769bcdb83be7f21668",
              "IPY_MODEL_554bd6fe1bad4405bde8dc1acc6bbab4"
            ],
            "layout": "IPY_MODEL_7926f30128394b26bd833d3193b999d3"
          }
        },
        "5a4a0a6521c844a0b536430884aa4bd9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_26251873aea648abb0a0f65c3043d04a",
            "placeholder": "​",
            "style": "IPY_MODEL_fc9fde2733e04346a6d2df2a343407b8",
            "value": "merges.txt: 100%"
          }
        },
        "03a08c52cc804e769bcdb83be7f21668": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d312eae439e848119f2447563af97fa1",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_55c2f7f4289040f4bbb3ac8c6dbffe37",
            "value": 456318
          }
        },
        "554bd6fe1bad4405bde8dc1acc6bbab4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a25d598baa6a4708940c308fea786c11",
            "placeholder": "​",
            "style": "IPY_MODEL_5e2fb42ef2394ae888e3f392dafd9c14",
            "value": " 456k/456k [00:00&lt;00:00, 22.2MB/s]"
          }
        },
        "7926f30128394b26bd833d3193b999d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26251873aea648abb0a0f65c3043d04a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc9fde2733e04346a6d2df2a343407b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d312eae439e848119f2447563af97fa1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55c2f7f4289040f4bbb3ac8c6dbffe37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a25d598baa6a4708940c308fea786c11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e2fb42ef2394ae888e3f392dafd9c14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53efb35d0aa644e49bcccc721095b253": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d6774fc272f4cd994999b09124817a0",
              "IPY_MODEL_2d2084fe21224e3fa9c5892e3824929c",
              "IPY_MODEL_92676ee648904f929fd913ae123a4cec"
            ],
            "layout": "IPY_MODEL_bee79abe1a7847cdb71258417cc0f729"
          }
        },
        "3d6774fc272f4cd994999b09124817a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f203076c09f4243aeb3fdabcdcb2b09",
            "placeholder": "​",
            "style": "IPY_MODEL_e75e4467ca464898a1f6c7bf65d02a76",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "2d2084fe21224e3fa9c5892e3824929c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f76613090a0543e6b42ee02956771248",
            "max": 239,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d2b63578ce204f5189eae5f87fa93f28",
            "value": 239
          }
        },
        "92676ee648904f929fd913ae123a4cec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4c1167611f84f3583a90d963a178a2d",
            "placeholder": "​",
            "style": "IPY_MODEL_dcb8e682b79241a4932c5486d8d2b825",
            "value": " 239/239 [00:00&lt;00:00, 9.27kB/s]"
          }
        },
        "bee79abe1a7847cdb71258417cc0f729": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9f203076c09f4243aeb3fdabcdcb2b09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e75e4467ca464898a1f6c7bf65d02a76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f76613090a0543e6b42ee02956771248": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2b63578ce204f5189eae5f87fa93f28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b4c1167611f84f3583a90d963a178a2d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dcb8e682b79241a4932c5486d8d2b825": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d7053f780b24c2d9bc27a6015a7bdfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a2371df166474311b6af7a36df59ef08",
              "IPY_MODEL_c3eb7f089fec4dbfb28b712c114f5ac0",
              "IPY_MODEL_48d0f640414f417f9617227f1195e14f"
            ],
            "layout": "IPY_MODEL_ef32d82d387a403c8b646c3243796374"
          }
        },
        "a2371df166474311b6af7a36df59ef08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3680bda4106b404b999777927a7d1cae",
            "placeholder": "​",
            "style": "IPY_MODEL_0dbc66caa907436c9d2635a44235446f",
            "value": "config.json: 100%"
          }
        },
        "c3eb7f089fec4dbfb28b712c114f5ac0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f82464171012489886db18c721b0a80c",
            "max": 816,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_10bb8d045d9f47c19c35d7f3b2dc65d6",
            "value": 816
          }
        },
        "48d0f640414f417f9617227f1195e14f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b9478f7e8fe470d84f4c889c8aa57e1",
            "placeholder": "​",
            "style": "IPY_MODEL_d829703081444e2fb27ba654f5d723e7",
            "value": " 816/816 [00:00&lt;00:00, 36.9kB/s]"
          }
        },
        "ef32d82d387a403c8b646c3243796374": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3680bda4106b404b999777927a7d1cae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dbc66caa907436c9d2635a44235446f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f82464171012489886db18c721b0a80c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10bb8d045d9f47c19c35d7f3b2dc65d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8b9478f7e8fe470d84f4c889c8aa57e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d829703081444e2fb27ba654f5d723e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "74862d731eb94de084b62658ecad8e72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5880d5ac9c884dd291cf43f5da74b02a",
              "IPY_MODEL_1a349f9a05ab4765ba9cf6d90d5dd1d8",
              "IPY_MODEL_83d99a3c1b794a93a8a210d327a2f1a3"
            ],
            "layout": "IPY_MODEL_07cbe58b06ce439f916c8efedb389204"
          }
        },
        "5880d5ac9c884dd291cf43f5da74b02a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67895bec89cd4e588b9499543df87b50",
            "placeholder": "​",
            "style": "IPY_MODEL_afeac9c8a73c4db5aba3f1509aab6af1",
            "value": "model.safetensors: 100%"
          }
        },
        "1a349f9a05ab4765ba9cf6d90d5dd1d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4557123618e4639baf18b805cfa9682",
            "max": 498617024,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c4985daf6059440eb95d2af90cca3a88",
            "value": 498617024
          }
        },
        "83d99a3c1b794a93a8a210d327a2f1a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2610ee92b9204b09ae548cf7e34c62f8",
            "placeholder": "​",
            "style": "IPY_MODEL_c61a627edba04f8fbe60ca41d1314803",
            "value": " 499M/499M [00:03&lt;00:00, 154MB/s]"
          }
        },
        "07cbe58b06ce439f916c8efedb389204": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "67895bec89cd4e588b9499543df87b50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afeac9c8a73c4db5aba3f1509aab6af1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f4557123618e4639baf18b805cfa9682": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4985daf6059440eb95d2af90cca3a88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2610ee92b9204b09ae548cf7e34c62f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c61a627edba04f8fbe60ca41d1314803": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p5RUTq8ueIeY"
      },
      "source": [
        "# Crash Course in Generative AI Worked Examples - Sentiment Analysis with Flan-T5\n",
        "\n",
        "*Authors: Yash Gopalji Pankhania, Jeel Kanzaria, Megha Patel*"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Introduction"
      ],
      "metadata": {
        "id": "a0Wbsq6T5rib"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Welcome to this hands-on Colab notebook, where we will explore the dynamic realm of sentiment analysis using the power of Generative AI. Throughout this interactive journey, you will gain insights into the intricate relationship between input text and model output, mastering the art of prompt engineering to tailor responses for specific tasks.\n",
        "\n",
        "In our exploration, we will compare zero-shot, one-shot, and few-shot inferences, unraveling the potential of prompt engineering to elevate the generative capabilities of Large Language Models (LLMs). To elevate our inferences, we'll employ a comprehensive fine-tuning approach and evaluate the results using ROUGE metrics. This meticulous examination will shed light on the nuanced improvements achievable through tailored model training. Diving deeper into efficiency, we'll explore Parameter Efficient Fine-Tuning (PEFT). Despite potential slightly-lower performance metrics, we'll uncover how the benefits of PEFT outweigh these trade-offs, providing valuable insights into resource-efficient fine-tuning."
      ],
      "metadata": {
        "id": "HYkP1W9x7Ujm"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Overview of the Labs"
      ],
      "metadata": {
        "id": "MXsu2HLI5tRh"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###**Lab 1: Generative AI use cases, project lifecycle, and model pre-training**\n",
        "\n",
        "Learning Objectives:\n",
        "\n",
        "**1. Discuss model pre-training and the value of continued pre-training vs fine-tuning:**\n",
        "\n",
        "Model Pre-training:\n",
        "\n",
        "Definition:\n",
        "\n",
        "Model pre-training is the initial phase in training a large language model. During this stage, the model is exposed to a diverse and extensive dataset containing a wide range of linguistic patterns and structures. The primary goal is for the model to learn general language patterns, grammar, and semantics, creating a foundation of linguistic knowledge.\n",
        "\n",
        "Purpose:\n",
        "\n",
        "The purpose of model pre-training is to build a versatile language model that can understand and generate coherent text across various language-related tasks. By exposing the model to a broad dataset, it learns to capture general language features, enabling it to perform well on a range of applications.\n",
        "\n",
        "Continued Pre-training vs. Fine-tuning:\n",
        "\n",
        "Continued Pre-training:\n",
        "\n",
        "Definition: Continued pre-training involves further training the pre-trained model on a specific dataset or within a particular domain.\n",
        "\n",
        "Purpose: This approach is beneficial when more domain-specific knowledge is required. It allows the model to delve deeper into the intricacies of a specific context, capturing nuances and patterns that might be crucial for tasks within that domain.\n",
        "\n",
        "Fine-tuning:\n",
        "\n",
        "Definition: Fine-tuning refers to adapting the pre-trained model for a specific task or dataset.\n",
        "\n",
        "Purpose: Fine-tuning is effective when there is limited task-specific data available. Instead of training the model from scratch, fine-tuning leverages the general language knowledge gained during pre-training and refines the model's parameters to make it more suitable for the targeted application.\n",
        "\n",
        "Decision Factors:\n",
        "\n",
        "Data Specificity: Continued pre-training is advantageous when there's a substantial amount of domain-specific data available. Fine-tuning is preferred when task-specific data is limited.\n",
        "\n",
        "Task Requirements: If the target task is closely related to the pre-training data, fine-tuning may suffice. If the task requires specialized knowledge, continued pre-training might be more beneficial.\n",
        "\n",
        "Computational Resources: Fine-tuning is generally more computationally efficient than continued pre-training, making it preferable in resource-constrained scenarios.\n",
        "\n",
        "In summary, the choice between continued pre-training and fine-tuning depends on factors such as the availability of task-specific data, the relevance of pre-training data to the target task, and the computational resources at hand. The decision is a balance between leveraging general language knowledge and adapting the model for specific applications.\n",
        "\n",
        "**2.Define the terms Generative AI, large language models, prompt, and describe the transformer architecture that powers LLMs:**\n",
        "\n",
        "**Generative AI:**\n",
        "\n",
        "Definition:\n",
        "\n",
        "Generative AI refers to a category of artificial intelligence systems designed to generate new, original content. These systems, instead of being explicitly programmed for specific tasks, are trained on data and can produce outputs such as text, images, or other forms of creative content. Generative AI models are known for their ability to autonomously create content that resembles the patterns and styles present in their training data.\n",
        "\n",
        "**Large Language Models (LLMs):**\n",
        "\n",
        "Definition:\n",
        "\n",
        "Large Language Models are advanced neural network architectures, often containing millions or even billions of parameters, designed to understand and generate human-like language. These models, like OpenAI's GPT (Generative Pre-trained Transformer) series, are pre-trained on massive datasets, enabling them to learn complex language patterns, semantics, and contextual relationships. LLMs are versatile and can be fine-tuned for various natural language processing tasks.\n",
        "\n",
        "**Prompt:**\n",
        "\n",
        "Definition:\n",
        "\n",
        "A prompt is a specific set of instructions or input provided to a generative AI model, especially large language models. It guides the model in generating a desired output. For instance, in the context of text generation, a prompt could be a sentence or a series of words that instruct the model on the type of text or information to generate in response.\n",
        "\n",
        "**Transformer Architecture:**\n",
        "\n",
        "Description:\n",
        "\n",
        "The transformer architecture is a type of neural network architecture introduced in the paper \"Attention is All You Need.\" It revolutionized natural language processing tasks, providing an efficient way to capture long-range dependencies in sequential data. Transformers use self-attention mechanisms to weigh the importance of different parts of the input sequence, enabling parallelized processing. The transformer architecture is the backbone of many state-of-the-art models, including large language models like GPT-3. Its ability to capture contextual relationships efficiently makes it well-suited for tasks requiring an understanding of language semantics and structure.\n",
        "\n",
        "**3.Define chain-of-thought prompting and describe how it can be used to improve LLMs reasoning and planning abilities:**\n",
        "\n",
        "**LLM-Based Generative AI Model Lifecycle:**\n",
        "\n",
        "**1.Data Gathering:**\n",
        "\n",
        "Objective: Collecting diverse and relevant datasets to train the LLM.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Data Quality: The quality of training data significantly impacts model performance.\n",
        "\n",
        "Data Diversity: Ensuring the dataset covers a broad range of language patterns and contexts.\n",
        "\n",
        "Ethical Considerations: Avoiding biases and ensuring fair representation in the training data.\n",
        "\n",
        "**2.Model Selection:**\n",
        "\n",
        "Objective: Choosing an appropriate LLM architecture based on the task requirements.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Model Size: Balancing between model complexity and computational resources.\n",
        "\n",
        "Task-specific Features: Selecting a model that aligns with the specific requirements of the intended tasks.\n",
        "\n",
        "Pre-training Data: Considering models pre-trained on relevant or diverse datasets.\n",
        "\n",
        "**3.Training:**\n",
        "\n",
        "Objective: Pre-training the selected model on a large dataset to learn general language patterns.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Computational Resources: Availability of GPUs or TPUs for efficient training.\n",
        "\n",
        "Training Time: Balancing between training time and the depth of model pre-training.\n",
        "\n",
        "Data Volume: Utilizing a sufficiently large dataset for effective pre-training.\n",
        "\n",
        "**4.Performance Evaluation:**\n",
        "\n",
        "Objective: Assessing the model's performance on validation datasets.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Task-specific Metrics: Choosing metrics relevant to the intended tasks.\n",
        "\n",
        "Generalization: Ensuring the model performs well on diverse data.\n",
        "\n",
        "Overfitting: Identifying and mitigating overfitting issues.\n",
        "\n",
        "**5.Fine-tuning (Optional):**\n",
        "\n",
        "Objective: Adapting the pre-trained model for specific tasks or domains.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Task-specific Data: The availability and quality of data for fine-tuning.\n",
        "\n",
        "Computational Resources: The capacity to fine-tune the model efficiently.\n",
        "\n",
        "Task Complexity: The intricacy of the target tasks influencing the need for fine-tuning.\n",
        "\n",
        "**6.Deployment:**\n",
        "\n",
        "Objective: Integrating the trained model into real-world applications.\n",
        "\n",
        "Factors:\n",
        "\n",
        "Inference Speed: Balancing between model accuracy and real-time inference requirements.\n",
        "\n",
        "Scalability: Ensuring the model can handle varying workloads.\n",
        "\n",
        "Ethical and Regulatory Compliance: Complying with ethical guidelines and legal requirements.\n",
        "\n",
        "**Constraining Factors:**\n",
        "\n",
        "**1.Resource Constraints:**\n",
        "\n",
        "Impact: Limited computational resources can restrict the size and complexity of the chosen model.\n",
        "\n",
        "Decision Influence: Determines the feasibility of training large models or fine-tuning for specific tasks.\n",
        "\n",
        "**2.Data Limitations:**\n",
        "\n",
        "Impact: Insufficient or biased training data can hinder model generalization.\n",
        "\n",
        "Decision Influence: Influences the effectiveness of model training and its ability to perform well across diverse scenarios.\n",
        "\n",
        "**3.Task Complexity:**\n",
        "\n",
        "Impact: Complex tasks may require more extensive pre-training or fine-tuning efforts.\n",
        "\n",
        "Decision Influence: Determines the depth and specificity of training needed for optimal performance.\n",
        "\n",
        "**4.Ethical Considerations:**\n",
        "\n",
        "Impact: Biased or unfair representation in training data can lead to ethical concerns.\n",
        "\n",
        "Decision Influence: Guides decisions in data gathering, model training, and deployment to ensure fairness and avoid ethical pitfalls.\n",
        "\n",
        "**5.Inference Requirements:**\n",
        "\n",
        "Impact: Real-time applications may require trade-offs between model complexity and inference speed.\n",
        "\n",
        "Decision Influence: Shapes decisions during model selection, fine-tuning, and deployment to meet real-time demands.\n",
        "\n",
        "The lifecycle of an LLM-based generative AI model is a dynamic process, and the decisions made at each step are influenced by a combination of technical, resource, and ethical considerations. Striking the right balance is crucial for developing effective and responsible generative AI systems.\n",
        "\n",
        "**4.Discuss the challenges that LLMs face with knowledge cut-offs, and explain how information retrieval and augmentation techniques can overcome these challenges:**\n",
        "\n",
        "**Computational Challenges during Model Pre-training:**\n",
        "\n",
        "1. **High Memory Usage:**\n",
        "  - **Challenge:** Pre-training large language models (LLMs) with millions or billions of parameters demands substantial GPU or TPU memory.\n",
        "  - **Impact:** Limited availability of memory can hinder the size and complexity of models that can be pre-trained.\n",
        "2. **Long Training Time:**\n",
        "  - **Challenge:** Pre-training on extensive datasets can be time-consuming, affecting overall training efficiency.\n",
        "  - **Impact:** Lengthy training times may limit the frequency of model experimentation and development.\n",
        "3. **Resource Intensiveness:**\n",
        "  - **Challenge:** Training large models requires significant computational resources, including powerful GPUs or TPUs.\n",
        "  - **Impact:** Resource-intensive pre-training can be costly and may be impractical for smaller research or development environments.\n",
        "\n",
        "**Strategies to Optimize Memory Usage and Reduce Footprint:**\n",
        "\n",
        "1. **Gradient Accumulation:**\n",
        "  - **Description:** Instead of updating model parameters after processing the entire batch, accumulate gradients over several smaller batches.\n",
        "  - **Benefits:** Reduces the effective batch size, allowing models to be trained on larger datasets with limited memory.\n",
        "2. **Model Parallelism:**\n",
        "  - **Description:** Distribute the model's parameters across multiple GPUs, enabling parallel processing.\n",
        "  - **Benefits:** Reduces the memory requirement on individual GPUs and facilitates training larger models.\n",
        "3. **Data Parallelism:**\n",
        "  - **Description:** Distribute training data across multiple devices and synchronize updates to the model.\n",
        "  - **Benefits:** Enables the use of larger batch sizes without increasing memory requirements on each device.\n",
        "4. **Mixed-Precision Training:**\n",
        "  - **Description:** Use lower-precision data types (e.g., float16) for model parameters during training.\n",
        "  - **Benefits:** Reduces memory usage without significant loss of model accuracy.\n",
        "5. **Gradient Checkpointing:**\n",
        "  - **Description:** Store intermediate activations during backward passes to trade computation for memory.\n",
        "  - **Benefits:** Enables training of deeper models with lower memory requirements at the cost of increased computation.\n",
        "6. **Memory-Efficient Architectures:**\n",
        "  - **Description:** Choose model architectures designed for memory efficiency without compromising performance.\n",
        "  - **Benefits:** Allows for the training of larger models within constrained memory environments.\n",
        "7. **Reduced Model Size:**\n",
        "  - **Description:** Prune or compress model parameters to reduce overall memory usage.\n",
        "  - **Benefits:** Facilitates pre-training of models on less powerful hardware or in resource-constrained environments.\n",
        "8. **Progressive Loading of Data:**\n",
        "  - **Description:** Load and process data progressively, reducing the need to store the entire dataset in memory.\n",
        "  - **Benefits:** Allows pre-training on datasets that do not fit entirely in memory.\n",
        "\n",
        "Efficiently addressing the computational challenges during model pre-training involves a combination of architectural choices, distributed computing strategies, and memory optimization techniques. By employing these strategies, researchers and practitioners can navigate resource constraints and pre-train large language models more effectively.\n",
        "\n",
        "**Define the term scaling law and describe the laws related to LLMs:**\n",
        "\n",
        "Scaling Law:\n",
        "\n",
        "Definition: A scaling law is a mathematical principle that describes how the performance of a system changes as a function of certain factors, such as size, volume, or resources. In the context of machine learning, scaling laws provide insights into how the performance of models, particularly large language models (LLMs), is affected by changes in factors like dataset size, computational resources, and inference requirements.\n",
        "\n",
        "Scaling Laws Related to LLMs:\n",
        "\n",
        "1. Dataset Size Scaling:\n",
        "  - Principle: Increasing the size of the training dataset can improve model performance, but with diminishing returns.\n",
        "  - Insights: Initially, as more data is added, models benefit from increased diversity and coverage. However, beyond a certain point, the marginal improvement decreases, and the computational cost of processing larger datasets becomes significant.\n",
        "2. Computational Resource Scaling:\n",
        "  - Principle: Increasing computational resources, such as GPU or TPU power, allows training larger and more complex models, improving performance.\n",
        "  - Insights: Larger models with more parameters tend to benefit from additional computation. However, the relationship is not always linear, and there's a point of diminishing returns where further increases in resources may not proportionally improve performance.\n",
        "3. Model Size Scaling:\n",
        "  - Principle: Enlarging the size of the model, measured by the number of parameters, can enhance performance.\n",
        "  - Insights: Larger models have the capacity to capture more complex patterns and relationships in the data. However, the computational cost increases, and there is a trade-off between model size and training efficiency.\n",
        "4. Inference Scaling:\n",
        "  - Principle: The computational requirements for model inference, especially for deployment in real-world applications, can be influenced by model size.\n",
        "  - Insights: Smaller models are often preferred for inference due to lower computational requirements. Efficient model architectures and optimization techniques become crucial for deployment in resource-constrained environments.\n",
        "\n",
        "Key Insights from Scaling Laws:\n",
        "\n",
        "1. Diminishing Returns:\n",
        "  - Increasing certain factors, such as dataset size or model complexity, may lead to diminishing returns in terms of improved performance.\n",
        "2. Trade-offs:\n",
        "  - There are trade-offs between factors like model size, computational resources, and inference speed. Finding the right balance is crucial for practical implementation.\n",
        "3. Efficiency Concerns:\n",
        "  - Efficient use of resources becomes paramount, especially when dealing with large datasets and complex models. Optimization techniques and algorithmic improvements play a significant role.\n",
        "4. Domain-Specific Considerations:\n",
        "  - The optimal scaling strategy may vary based on the specific requirements of the task or domain. Understanding the nuances is essential for effective model development.\n",
        "\n",
        "Scaling laws guide practitioners and researchers in optimizing large language models for efficiency and effectiveness, considering factors that impact performance across different stages of model development and deployment."
      ],
      "metadata": {
        "id": "Xgnrl51l8D9Q"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "###**Lab 2: Fine-tuning and evaluating large language models**\n",
        "\n",
        "Learning Objectives:\n",
        "\n",
        "**1.Describe how fine-tuning with instructions using prompt datasets can improve performance on one or more tasks:**\n",
        "\n",
        "**Fine-tuning with Instructions Using Prompt Datasets:**\n",
        "\n",
        "**Objective:** The objective of fine-tuning with instructions using prompt datasets is to adapt a pre-trained large language model (LLM) to perform specific tasks or excel in certain domains. By providing task-specific instructions through carefully crafted prompts during the fine-tuning process, the model can be guided to generate contextually relevant outputs for the desired applications.\n",
        "\n",
        "**Key Concepts:**\n",
        "\n",
        "1. **Fine-tuning:**\n",
        "  - **Definition:** Fine-tuning involves taking a pre-trained model and adjusting its parameters on a task-specific dataset.\n",
        "  - **Purpose:** It allows the model to specialize for particular applications, leveraging the general language knowledge gained during pre-training.\n",
        "2. **Instructions using Prompt Datasets:**\n",
        "  - **Definition:** A prompt is a set of input instructions given to the model to guide its output generation.\n",
        "  - **Purpose:** By providing specific instructions through prompts during fine-tuning, the model can be tailored to respond effectively to particular tasks or domains.\n",
        "3. **Enhancing Performance:**\n",
        "  - **Mechanism:** The prompts act as cues for the model, influencing its understanding and generation of contextually relevant content.\n",
        "  - **Benefits:**\n",
        "    - **Task-Specific Adaptation:** The model becomes more adept at tasks outlined in the prompts.\n",
        "    - **Improved Contextual Understanding:** Fine-tuning helps the model grasp nuances and specific patterns related to the given instructions.\n",
        "4. **Versatility:**\n",
        "  - **Applicability:** This fine-tuning approach makes large language models versatile, allowing them to be used for a wide range of tasks beyond their initial pre-training objectives.\n",
        "  - **Customization:** Tailoring the model through prompt-based fine-tuning enables it to adapt to diverse application scenarios.\n",
        "5. **Examples of Tasks:**\n",
        "  - **Text Completion:** Instructing the model to complete sentences or paragraphs based on given prompts.\n",
        "  - **Summarization:** Fine-tuning for summarizing longer texts with specific guidelines.\n",
        "  - **Translation:** Adapting the model for translating text from one language to another.\n",
        "  - **Question-Answering:** Training the model to generate accurate responses to specific types of questions.\n",
        "\n",
        "**Significance:**\n",
        "\n",
        "- **Efficiency:** Fine-tuning with prompt datasets is an efficient way to leverage pre-trained models for task-specific applications without starting the training process from scratch.\n",
        "- **Domain Adaptation:** This approach is particularly powerful for adapting models to specialized domains, where the language nuances may differ from the general pre-training data.\n",
        "- **Transfer Learning:** Fine-tuning extends the principles of transfer learning, allowing models to transfer knowledge gained in one context to excel in another.\n",
        "\n",
        "In summary, fine-tuning with instructions using prompt datasets is a crucial step in the lifecycle of large language models, enabling them to perform effectively on specific tasks or domains by providing targeted guidance during the training process.\n",
        "\n",
        "**2. Define catastrophic forgetting and explain techniques that can be used to overcome it:**\n",
        "\n",
        "**Catastrophic Forgetting:**\n",
        "\n",
        "**Definition:** Catastrophic forgetting is a phenomenon observed in machine learning, particularly in neural networks, where a model tends to forget previously learned information when it is trained on new tasks. This can be problematic when the model needs to continually adapt to new data without losing knowledge of its previously acquired skills.\n",
        "\n",
        "**Techniques to Overcome Catastrophic Forgetting:**\n",
        "\n",
        "1. **Rehearsal or Experience Replay:**\n",
        "  - **Description:** Store and periodically revisit old data during training.\n",
        "  - **Purpose:** By interleaving new and old data, the model is forced to retain knowledge of both tasks over time.\n",
        "2. **Elastic Weight Consolidation (EWC):**\n",
        "  - **Description:** Introduce a regularization term in the loss function that penalizes changes to important weights learned during previous tasks.\n",
        "  - **Purpose:** Prioritizes the preservation of important knowledge while allowing adaptation to new tasks.\n",
        "3. **Dual Memory Networks:**\n",
        "  - **Description:** Introduce a separate memory module dedicated to storing information from the past.\n",
        "  - **Purpose:** Allows the model to selectively access and update information in the memory module, preventing interference with previous knowledge.\n",
        "4. **Progressive Neural Networks (PNN):**\n",
        "  - **Description:** Maintain a separate network for each task, and share parameters between tasks.\n",
        "  - **Purpose:** Ensures that the knowledge learned for each task is retained in its dedicated network, preventing interference with other tasks.\n",
        "5. **Meta-learning or Learning to Learn:**\n",
        "  - **Description:** Train the model to quickly adapt to new tasks with minimal forgetting.\n",
        "  - **Purpose:** Enables the model to efficiently learn and adapt to new information while retaining knowledge of previous tasks.\n",
        "6. **Parameter Regularization:**\n",
        "  - **Description:** Regularize the model parameters during training to constrain changes that may lead to forgetting.\n",
        "  - **Purpose:** Encourages the model to adapt to new information without excessively modifying parameters crucial for retaining previous knowledge.\n",
        "7. **Knowledge Distillation:**\n",
        "  - **Description:** Transfer knowledge from an existing model (teacher) to the model being fine-tuned (student).\n",
        "  - **Purpose:** Helps the model retain knowledge from the teacher model while adapting to new tasks.\n",
        "8. **Task-specific Heads or Modules:**\n",
        "  - **Description:** Design the model with task-specific heads or modules for different tasks.\n",
        "  - **Purpose:** Allows the model to focus on learning new tasks without disrupting the representations associated with previous tasks.\n",
        "\n",
        "**Key Insights:**\n",
        "\n",
        "- Catastrophic forgetting is a challenge, especially in scenarios where a model needs to adapt to new tasks without losing knowledge of previously learned tasks.\n",
        "- Techniques to overcome catastrophic forgetting aim to strike a balance between adapting to new information and preserving knowledge from the past.\n",
        "- The choice of a specific technique depends on the nature of the tasks, the architecture of the model, and the available computational resources.\n",
        "\n",
        "By incorporating these techniques, practitioners can address the issue of catastrophic forgetting and build models that continually learn and adapt to new tasks without compromising their previously acquired knowledge.\n",
        "\n",
        "**3.**** Define the term Parameter-efficient Fine Tuning (PEFT):**\n",
        "\n",
        "**Parameter-efficient Fine Tuning (PEFT):**\n",
        "\n",
        "**Definition:** Parameter-efficient Fine Tuning (PEFT) is a fine-tuning methodology that emphasizes optimizing computational efficiency during the adaptation of pre-trained models for specific tasks. The primary goal of PEFT is to achieve effective fine-tuning with minimal updates to the model parameters, thereby reducing the computational resources required for the fine-tuning process.\n",
        "\n",
        "**Key Concepts:**\n",
        "\n",
        "1. **Fine-tuning with Minimal Parameter Updates:**\n",
        "  - **Objective:** PEFT aims to fine-tune models with only essential updates to the pre-trained parameters, avoiding extensive modifications.\n",
        "  - **Rationale:** By minimizing parameter updates, PEFT seeks to retain the valuable knowledge gained during pre-training while adapting the model to task-specific requirements.\n",
        "2. **Efficient Use of Computational Resources:**\n",
        "  - **Objective:** Focus on achieving fine-tuning with reduced computational cost.\n",
        "  - **Rationale:** Computational efficiency is a critical consideration, especially when dealing with resource-constrained environments or when deploying models at scale.\n",
        "3. **Balancing Task-specific Adaptation and Model Preservation:**\n",
        "  - **Objective:** Strike a balance between adapting the model to new tasks and preserving knowledge from pre-training.\n",
        "  - **Rationale:** PEFT seeks to avoid overfitting to task-specific data while ensuring that the model maintains its generalization capabilities acquired during pre-training.\n",
        "4. **Optimizing Memory Usage:**\n",
        "  - **Objective:** Reduce the memory footprint required for fine-tuning.\n",
        "  - **Rationale:** Optimizing memory usage is crucial for efficient training, especially when dealing with large models or limited computational resources.\n",
        "5. **Adaptability to Various Tasks:**\n",
        "  - **Objective:** Design PEFT to be adaptable across a range of tasks.\n",
        "  - **Rationale:** The method should be versatile, allowing models to be efficiently fine-tuned for different applications without the need for extensive computational resources.\n",
        "\n",
        "**Significance:**\n",
        "\n",
        "- **Resource Efficiency:** PEFT is particularly valuable in scenarios where computational resources are limited, enabling the fine-tuning of models without excessive demands on hardware.\n",
        "- **Scalability:** The efficiency of PEFT makes it scalable for deploying models in large-scale applications, where the cost of fine-tuning and model adaptation is a critical factor.\n",
        "- **Rapid Model Deployment:** PEFT facilitates quick adaptation of pre-trained models to new tasks, enabling rapid deployment in real-world applications.\n",
        "- **Versatility:** PEFT is applicable to a variety of tasks and domains, making it a versatile approach for fine-tuning models for different applications while considering resource constraints.\n",
        "\n",
        "Parameter-efficient Fine Tuning addresses the need for adapting pre-trained models with a focus on computational efficiency, making it a valuable methodology in scenarios where resource utilization is a crucial consideration.\n",
        "\n",
        "**4.**** Explain how PEFT decreases computational cost and overcomes catastrophic forgetting:**\n",
        "\n",
        "**How Parameter-efficient Fine Tuning (PEFT) Decreases Computational Cost:**\n",
        "\n",
        "1. **Minimal Parameter Updates:** PEFT aims to fine-tune models with minimal updates to the pre-trained parameters. This reduces the amount of computation required during the fine-tuning process. Instead of making extensive modifications to the model, PEFT selectively adjusts parameters relevant to the new task.\n",
        "2. **Efficient Memory Usage:** PEFT optimizes memory usage during the fine-tuning process. By minimizing the changes to the model's parameters, it helps in efficiently utilizing available memory resources. This is crucial, especially when dealing with large models that may have memory constraints.\n",
        "3. **Adaptability Across Tasks:** PEFT is designed to be adaptable across various tasks. This adaptability allows a single pre-trained model to be fine-tuned for different applications without the need for separate resource-intensive fine-tuning processes. This versatility contributes to overall computational efficiency.\n",
        "4. **Reduced Computational Overhead:** Since PEFT focuses on retaining the knowledge gained during pre-training while adapting to new tasks, it avoids unnecessary computations that may lead to increased computational overhead. This results in a more resource-efficient fine-tuning process.\n",
        "\n",
        "**How PEFT Overcomes Catastrophic Forgetting:**\n",
        "\n",
        "1. **Balancing Task-specific Adaptation:** PEFT seeks to strike a balance between adapting the model to new tasks and preserving knowledge from pre-training. This prevents the model from overly specializing on the new task, reducing the risk of catastrophic forgetting.\n",
        "2. **Regularization Techniques:** PEFT may incorporate regularization techniques that penalize drastic changes to important parameters learned during pre-training. This helps in retaining essential knowledge and preventing catastrophic forgetting when adapting to new tasks.\n",
        "3. **Efficient Transfer of Knowledge:** PEFT focuses on efficient transfer of knowledge from pre-training to fine-tuning. By emphasizing minimal parameter updates, it ensures that the model maintains a significant portion of its previously acquired knowledge while adapting to new information.\n",
        "4. **Selective Parameter Updates:** PEFT selectively updates parameters that are relevant to the new task, minimizing interference with existing knowledge. This selective approach prevents catastrophic forgetting by ensuring that the model retains its ability to perform well on a range of tasks.\n",
        "5. **Task-specific Modules:** If applicable, PEFT may involve the use of task-specific modules or heads, allowing the model to focus on learning new tasks without disrupting representations associated with previous tasks. This modular approach contributes to preventing catastrophic forgetting.\n",
        "\n",
        "In summary, PEFT decreases computational costs by optimizing memory usage, minimizing parameter updates, and providing adaptability across tasks. Simultaneously, it overcomes catastrophic forgetting by carefully balancing task-specific adaptation and preserving knowledge from pre-training through selective updates and regularization techniques.\n",
        "\n",
        "**5.**** Explain how fine-tuning with instructions using prompt datasets can increase LLM performance on one or more tasks:**\n",
        "\n",
        "**Fine-tuning with Instructions using Prompt Datasets: Enhancing LLM Performance**\n",
        "\n",
        "1. **Task-Specific Adaptation:**\n",
        "\n",
        "- **Importance:** Fine-tuning with instructions using prompt datasets enables the model to adapt to specific tasks by providing explicit guidance through prompts.\n",
        "- **Effect:** The model learns to generate contextually relevant responses based on the task-specific instructions, enhancing its performance for targeted applications.\n",
        "\n",
        "2. **Guided Context Understanding:**\n",
        "\n",
        "- **Importance:** Prompts serve as cues, guiding the model's understanding of the desired context for generating outputs.\n",
        "- **Effect:** This guided context understanding ensures that the model produces outputs that align more closely with the intended meaning of the given instructions, leading to improved task performance.\n",
        "\n",
        "3. **Customization for Diverse Tasks:**\n",
        "\n",
        "- **Importance:** Fine-tuning with prompt datasets allows large language models (LLMs) to be customized for a variety of tasks beyond their initial pre-training objectives.\n",
        "- **Effect:** The model becomes versatile, capable of adapting to diverse applications by leveraging task-specific instructions, making it a valuable tool for a broad range of natural language processing tasks.\n",
        "\n",
        "4. **Improved Contextual Relevance:**\n",
        "\n",
        "- **Importance:** The inclusion of task-specific instructions refines the model's contextual understanding, improving the relevance of generated content.\n",
        "- **Effect:** The model produces more accurate and contextually appropriate responses, addressing the specific requirements outlined in the prompts and enhancing its overall performance on various tasks.\n",
        "\n",
        "5. **Nuanced Adaptation to Instructions:**\n",
        "\n",
        "- **Importance:** Fine-tuning allows the model to capture nuanced instructions and patterns within the prompts, leading to more nuanced adaptations.\n",
        "- **Effect:** The model learns to generate outputs that align not only with explicit instructions but also with subtle nuances present in the provided prompts, making it more adept at understanding complex language patterns.\n",
        "\n",
        "6. **Reduced Ambiguity:**\n",
        "\n",
        "- **Importance:** Task-specific instructions help clarify ambiguous scenarios and guide the model in generating more precise responses.\n",
        "- **Effect:** The model becomes less prone to generating ambiguous or generic outputs, resulting in improved clarity and accuracy tailored to the specific task at hand.\n",
        "\n",
        "7. **Efficient Model Adaptation:**\n",
        "\n",
        "- **Importance:** Fine-tuning with prompt datasets provides an efficient way to adapt pre-trained models to new tasks without retraining from scratch.\n",
        "- **Effect:** The model leverages its pre-existing language knowledge and adapts more quickly and efficiently to new tasks, making the fine-tuning process resource-efficient.\n",
        "\n",
        "8. **Enhanced Generalization:**\n",
        "\n",
        "- **Importance:** The fine-tuning process with specific instructions contributes to the model's ability to generalize well across a range of tasks.\n",
        "- **Effect:** The model retains its adaptability to different tasks, ensuring that the learned knowledge from prompts contributes to improved generalization capabilities.\n",
        "\n",
        "In conclusion, fine-tuning with instructions using prompt datasets is crucial for enhancing the performance of large language models by providing task-specific guidance, refining contextual understanding, and enabling efficient adaptation to diverse applications. The incorporation of explicit instructions through prompts is a powerful approach to tailoring models for specific tasks and domains, contributing to their versatility and effectiveness."
      ],
      "metadata": {
        "id": "S14wxYC99X3Q"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## **Lab 3: Reinforcement learning and LLM-powered applications**\n",
        "\n",
        "Learning Objectives:\n",
        "\n",
        "**1.Describe how RLHF uses human feedback to improve the performance and alignment of large language models:**\n",
        "\n",
        "**1. Reinforcement Learning from Human Feedback (RLHF):**\n",
        "\n",
        "**Definition:** Reinforcement Learning from Human Feedback (RLHF) is a methodology that leverages human-generated feedback to enhance the performance and alignment of large language models (LLMs). It involves using human-provided signals, such as comparisons or rankings, to refine the model's behavior over time.\n",
        "\n",
        "**Key Concepts:**\n",
        "\n",
        "1. **Performance Improvement:**\n",
        "  - **Objective:** RLHF aims to improve the model's performance by iteratively incorporating feedback from human evaluators.\n",
        "  - **Process:** The model receives feedback signals based on its generated outputs, allowing it to adapt and refine its behavior in subsequent iterations.\n",
        "2. **Alignment with Human Intent:**\n",
        "  - **Objective:** RLHF seeks to align the model's outputs more closely with human preferences and intentions.\n",
        "  - **Human Feedback Types:** Evaluators provide feedback in the form of rankings, preferences, or comparisons, guiding the model to generate content that better meets human expectations.\n",
        "3. **Iterative Learning Process:**\n",
        "  - **Process:** RLHF involves multiple iterations where the model generates outputs, receives human feedback, and updates its parameters to improve alignment and performance.\n",
        "  - **Feedback Refinement:** The iterative process allows the model to learn from nuanced human feedback, progressively refining its understanding and generating more desirable content.\n",
        "4. **Diversity of Human Feedback:**\n",
        "  - **Importance:** RLHF benefits from diverse human perspectives and preferences.\n",
        "  - **Adaptability:** The model can adapt to a wide range of preferences, ensuring that it aligns well with the varied expectations of users.\n",
        "5. **Reward Signal Design:**\n",
        "  - **Objective:** Designing an effective reward signal is crucial in RLHF.\n",
        "  - **Feedback Types:** Rewards can be based on binary preferences, numerical scores, or more complex signals, depending on the nature of the task and the quality of generated outputs.\n",
        "6. **Addressing Ethical Considerations:**\n",
        "  - **Consideration:** RLHF should be designed with ethical considerations to avoid biases and ensure fair representation in the feedback process.\n",
        "  - **Transparency:** Transparent communication about the model's learning process and the use of human feedback is essential for ethical RLHF.\n",
        "7. **Application to LLMs:**\n",
        "  - **Relevance:** RLHF is particularly relevant for large language models, where aligning generated text with human preferences is crucial for diverse applications.\n",
        "  - **Versatility:** LLMs can be fine-tuned and adapted for various tasks using RLHF, making them more versatile and capable of delivering contextually relevant content.\n",
        "\n",
        "**Significance:**\n",
        "\n",
        "- **Human-Centric Improvement:** RLHF puts human evaluators at the center of the improvement process, ensuring that the model's outputs align with human expectations and preferences.\n",
        "- **Adaptability to Dynamic Preferences:** RLHF allows models to adapt to evolving user preferences and expectations, making them more responsive to changes in language use and cultural context.\n",
        "- **Ethical Considerations:** RLHF emphasizes ethical considerations in the training process, promoting fairness, transparency, and responsible AI practices.\n",
        "\n",
        "In summary, RLHF is a powerful approach to enhance the performance and alignment of large language models by leveraging human feedback, facilitating iterative learning, and ensuring adaptability to diverse user preferences.\n",
        "\n",
        "**2.**  **Explain how data gathered from human labelers is used to train a reward model for RLHF:**\n",
        "\n",
        "**Training a Reward Model for RLHF using Data from Human Labelers:**\n",
        "\n",
        "Reinforcement Learning from Human Feedback (RLHF) involves training a reward model that guides the learning process of a model based on human-provided signals. The process typically involves collecting data from human labelers who evaluate the model's outputs, and this data is then used to construct a reward model. Here's an overview of how this process works:\n",
        "\n",
        "1. **Data Collection:**\n",
        "\n",
        "- **Human Labelers:** Gather a group of human labelers who can assess the quality of the model's outputs. These labelers may provide rankings, preferences, or comparisons based on their judgment.\n",
        "\n",
        "2. **Task Definition:**\n",
        "\n",
        "- **Define the Task:** Clearly define the task for which the model is seeking improvement. This could be generating coherent text, answering questions, or any other task relevant to the model's purpose.\n",
        "\n",
        "3. **Model Output Generation:**\n",
        "\n",
        "- **Model Inference:** Have the model generate outputs for a set of inputs relevant to the defined task. These outputs serve as the candidate solutions that will be evaluated by human labelers.\n",
        "\n",
        "4. **Human Feedback Collection:**\n",
        "\n",
        "- **Feedback Signals:** Human labelers assess the model-generated outputs and provide feedback in the form of comparisons, rankings, or preferences. For example, they may rank different outputs in terms of quality or indicate preferences between pairs of generated content.\n",
        "\n",
        "5. **Reward Model Construction:**\n",
        "\n",
        "- **Mapping Human Feedback to Rewards:** Convert the human-provided feedback into a reward signal. This involves mapping the feedback types (e.g., preferences, rankings) to numerical values that represent the desirability of the generated outputs.\n",
        "- **Reward Function:** Define a reward function that captures the desired characteristics of the outputs based on the human feedback. For instance, if the task is language generation, a reward function might assign higher scores to outputs that are more coherent, contextually relevant, or grammatically correct.\n",
        "\n",
        "6. **Supervised Learning:**\n",
        "\n",
        "- **Training the Reward Model:** Utilize the collected data and the constructed reward model to train the model in a supervised learning setting. The model learns to generate outputs that align with the reward signals derived from human feedback.\n",
        "\n",
        "7. **Iterative Feedback Loop:**\n",
        "\n",
        "- **Iterative Learning Process:** Repeat the process iteratively. Generate new outputs, collect feedback from human labelers, update the reward model, and retrain the model. This iterative loop allows the model to progressively improve its performance based on refined human guidance.\n",
        "\n",
        "8. **Ethical Considerations:**\n",
        "\n",
        "- **Bias Mitigation:** Pay attention to potential biases in the human feedback and take steps to mitigate them. Ensure diverse perspectives are considered and that the training process adheres to ethical guidelines.\n",
        "\n",
        "9. **Validation and Testing:**\n",
        "\n",
        "- **Validation Set:** Reserve a portion of the collected data as a validation set to assess the generalization of the model to new, unseen inputs.\n",
        "- **Testing Set:** Use a separate testing set to evaluate the model's performance on inputs that were not part of the training or validation sets.\n",
        "\n",
        "10. **Deployment and Monitoring:**\n",
        "\n",
        "- **Deployment:** Once the model achieves satisfactory performance, deploy it for the intended task.\n",
        "- **Monitoring:** Continuously monitor the model's outputs and gather ongoing feedback from users to address any issues and further improve its performance over time.\n",
        "\n",
        "11. **Documentation and Transparency:**\n",
        "\n",
        "- **Documentation:** Document the entire process, including the task definition, feedback collection, reward model construction, and training details.\n",
        "- **Transparency:** Be transparent about the use of human feedback, the reward model, and the iterative learning process to build trust and ensure accountability.\n",
        "\n",
        "12. **Feedback Refinement:**\n",
        "\n",
        "- **Refining Feedback Mechanisms:** Based on the model's ongoing performance, refine the feedback mechanisms used by human labelers. Continuous improvement in feedback collection contributes to the model's adaptability.\n",
        "\n",
        "**3.**  **Define chain-of-thought prompting and describe how it can be used to improve LLMs reasoning and planning abilities:**\n",
        "\n",
        "**Chain-of-Thought Prompting:**\n",
        "\n",
        "**Definition:** Chain-of-Thought Prompting is a technique used in natural language processing, particularly in the context of large language models (LLMs), to guide the model's reasoning and planning abilities. This approach involves structuring a sequence of prompts or queries in a way that encourages the model to build upon its own generated responses iteratively, creating a coherent chain of thought.\n",
        "\n",
        "**Key Concepts:**\n",
        "\n",
        "1. **Sequential Prompting:**\n",
        "  - **Approach:** Provide a series of prompts or questions to the model in a sequential manner.\n",
        "  - **Intent:** Each prompt builds upon the context established by the previous one, guiding the model to maintain a coherent and logical chain of thought.\n",
        "2. **Progressive Expansion of Ideas:**\n",
        "  - **Purpose:** Encourage the model to progressively expand on its initial responses, developing a more nuanced and detailed line of reasoning.\n",
        "  - **Continuity:** The prompts maintain continuity in the thought process, facilitating the generation of logically connected ideas.\n",
        "3. **Contextual Understanding:**\n",
        "  - **Emphasis:** Chain-of-thought prompting emphasizes the development of a deeper contextual understanding within the model.\n",
        "  - **Context Retention:** The model is guided to retain and build upon the context established in earlier prompts, contributing to more informed and contextually relevant responses.\n",
        "4. **Reasoning and Planning:**\n",
        "  - **Application:** The technique is particularly useful for enhancing the model's reasoning and planning abilities.\n",
        "  - **Structured Thought Process:** By guiding the model through a structured sequence of prompts, it learns to reason through problems and plan responses in a more organized manner.\n",
        "5. **Task-Specific Adaptation:**\n",
        "  - **Flexibility:** Chain-of-thought prompting is adaptable to various tasks, allowing the model to reason and plan in domains such as problem-solving, decision-making, and narrative generation.\n",
        "  - **Task-Specific Context:** The prompts can be tailored to elicit responses aligned with specific task requirements.\n",
        "\n",
        "**Application to Improve LLMs Reasoning and Planning Abilities:**\n",
        "\n",
        "1. **Scenario-Based Problem Solving:**\n",
        "  - **Prompt Sequence:** Present the model with a scenario or problem in the initial prompt. Subsequent prompts can ask for step-by-step solutions or elaborations, guiding the model through a reasoning process.\n",
        "  - **Result:** The model learns to approach complex problems systematically, demonstrating improved reasoning abilities.\n",
        "2. **Long-Form Narrative Generation:**\n",
        "  - **Prompt Sequence:** Initiate the narrative with an initial scenario or idea. Subsequent prompts guide the model to add details, develop characters, and progress the storyline.\n",
        "  - **Result:** The model produces coherent and logically structured long-form narratives, showcasing enhanced planning capabilities.\n",
        "3. **Strategic Decision-Making:**\n",
        "  - **Prompt Sequence:** Present a decision-making scenario in the initial prompt. Follow up with prompts that seek justifications, alternative choices, and potential outcomes.\n",
        "  - **Result:** The model learns to reason through strategic decisions, considering various factors and consequences, leading to improved planning abilities.\n",
        "4. **Argumentative Essay Construction:**\n",
        "  - **Prompt Sequence:** Begin with a prompt introducing a topic. Subsequent prompts guide the model to present arguments, counterarguments, and evidence.\n",
        "  - **Result:** The model develops a well-structured argumentative essay, demonstrating its ability to reason and plan the presentation of ideas cohesively.\n",
        "\n",
        "**Benefits:**\n",
        "\n",
        "- **Contextual Continuity:** Chain-of-thought prompting ensures that the model maintains contextual continuity, reducing the risk of generating disjointed or inconsistent responses.\n",
        "- **Structured Reasoning:** The technique promotes structured reasoning, aiding the model in breaking down complex tasks into a series of logical steps.\n",
        "- **Adaptability:** Chain-of-thought prompting is adaptable to various applications, allowing LLMs to improve their reasoning and planning abilities across diverse tasks.\n",
        "\n",
        "By employing chain-of-thought prompting, developers and researchers can guide large language models through a structured thinking process, enhancing their reasoning and planning capabilities in a wide range of applications.\n",
        "\n",
        "**4.**  **Discuss the challenges that LLMs face with knowledge cut-offs, and explain how information retrieval and augmentation techniques can overcome these challenges:**\n",
        "\n",
        "**Challenges of Knowledge Cut-offs in Large Language Models (LLMs):**\n",
        "\n",
        "1. **Temporal Limitations:**\n",
        "  - **Challenge:** LLMs have a knowledge cut-off, meaning they are trained on a fixed dataset up to a certain point in time. They may lack awareness of events or information that occurred after the cut-off, leading to temporal limitations in their knowledge.\n",
        "2. **Dynamic Information:**\n",
        "  - **Challenge:** Real-world information is dynamic and continuously evolving. LLMs may not be aware of the latest developments, leading to outdated or inaccurate responses when queried about recent events.\n",
        "3. **Limited Domain Expertise:**\n",
        "  - **Challenge:** LLMs may not have expertise in specialized or rapidly evolving domains. Their knowledge is confined to the general patterns learned during training, and they may struggle with in-depth understanding of specific topics.\n",
        "4. **Emerging Trends and Terminology:**\n",
        "  - **Challenge:** New trends, technologies, and terminology may emerge after the knowledge cut-off, causing LLMs to be unaware of or misinterpret current terminology and concepts.\n",
        "\n",
        "**Information Retrieval and Augmentation Techniques to Overcome Knowledge Cut-offs:**\n",
        "\n",
        "1. **Continuous Training:**\n",
        "  - **Approach:** Periodically update the LLM by retraining it on more recent data to extend its knowledge beyond the initial cut-off.\n",
        "  - **Benefit:** Allows the model to stay informed about evolving information and adapt to changes in language use.\n",
        "2. **External Knowledge Bases:**\n",
        "  - **Approach:** Integrate information retrieval from external knowledge bases or databases during inference.\n",
        "  - **Benefit:** Enables the model to access real-time information and domain-specific knowledge beyond its training data, overcoming cut-off limitations.\n",
        "3. **Pre-training on Broad Datasets:**\n",
        "  - **Approach:** Pre-train LLMs on diverse and extensive datasets to capture a broader understanding of language patterns.\n",
        "  - **Benefit:** Increases the chances that the model has encountered a wide range of topics during training, improving its ability to generate contextually relevant responses even after the knowledge cut-off.\n",
        "4. **Federated Learning:**\n",
        "  - **Approach:** Implement federated learning where models are trained collaboratively on decentralized data sources.\n",
        "  - **Benefit:** By aggregating knowledge from multiple sources, the model gains insights from a broader set of data, reducing the impact of individual knowledge cut-offs.\n",
        "5. **Active Learning:**\n",
        "  - **Approach:** Allow the model to actively seek user feedback and incorporate it into its knowledge base.\n",
        "  - **Benefit:** Users can correct and update the model's knowledge, helping it adapt to changes and overcome limitations imposed by the cut-off.\n",
        "6. **Fine-tuning on Task-Specific Data:**\n",
        "  - **Approach:** Fine-tune the LLM on task-specific datasets relevant to the domain in question.\n",
        "  - **Benefit:** Improves the model's performance in specific domains or tasks by providing targeted training data beyond the initial cut-off.\n",
        "7. **Semantic Augmentation:**\n",
        "  - **Approach:** Introduce semantic augmentation techniques that generate paraphrased versions of queries or responses.\n",
        "  - **Benefit:** Helps the model generalize its understanding and address gaps in knowledge by presenting information in varied ways.\n",
        "8. **Active Context Integration:**\n",
        "  - **Approach:** Allow the model to maintain an active context window, dynamically updating it with recent information during conversations.\n",
        "  - **Benefit:** Keeps the model informed about the context of ongoing interactions, compensating for knowledge cut-offs in a conversational context."
      ],
      "metadata": {
        "id": "ywS0yI4lWJpa"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Sentiment Analysis\n"
      ],
      "metadata": {
        "id": "CmQxeGwK50xZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Upon successfully navigating the lab focused on generating summaries from dialogues, our curiosity naturally led us to explore the realm of sentiment analysis using the powerful Flan-T5 model. In each section of our investigation, we aimed to extract sentiment, categorizing it as positive, negative, or neutral. Employing pretrained models, we meticulously replicated the process on a Twitter dataset, fine-tuning our approach by refining prompts to guide the model in discerning our specific criteria. Let's demonstrate this in the next section as we walkthrough the labs again."
      ],
      "metadata": {
        "id": "01qHNMEV_2wV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Walkthrough of the Lab Using Sentiment Analysis"
      ],
      "metadata": {
        "id": "T998rMdqNI9F"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this notebook, we will do sentiment analysis using generative AI. You will\n",
        "explore how the input text affects the output of the model, and perform prompt engineering to direct it towards our task. By comparing zero shot, one shot, and few shot inferences, we will take the first step towards prompt engineering and see how it can enhance the generative output of Large Language Models."
      ],
      "metadata": {
        "id": "TdolQqGPMtZS"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "odM32eE2OoEm"
      },
      "source": [
        "<a name='1.1'></a>\n",
        "## 1 - Install Required Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BGsCh9OFRNmY",
        "outputId": "d20615ec-9bb8-46ff-94a8-4dd2bef791d2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (23.1.2)\n",
            "Collecting pip\n",
            "  Downloading pip-23.3.1-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: pip\n",
            "  Attempting uninstall: pip\n",
            "    Found existing installation: pip 23.1.2\n",
            "    Uninstalling pip-23.1.2:\n",
            "      Successfully uninstalled pip-23.1.2\n",
            "Successfully installed pip-23.3.1\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m887.5/887.5 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.6/4.6 MB\u001b[0m \u001b[31m113.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m849.3/849.3 kB\u001b[0m \u001b[31m58.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m557.1/557.1 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m317.1/317.1 MB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.0/21.0 MB\u001b[0m \u001b[31m67.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchaudio 2.1.0+cu118 requires torch==2.1.0, but you have torch 1.13.1 which is incompatible.\n",
            "torchtext 0.16.0 requires torch==2.1.0, but you have torch 1.13.1 which is incompatible.\n",
            "torchtext 0.16.0 requires torchdata==0.7.0, but you have torchdata 0.5.1 which is incompatible.\n",
            "torchvision 0.16.0+cu118 requires torch==2.1.0, but you have torch 1.13.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m64.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m468.7/468.7 kB\u001b[0m \u001b[31m42.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m92.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m21.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for rouge_score (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ],
      "source": [
        "%pip install --upgrade pip\n",
        "%pip install --disable-pip-version-check \\\n",
        "    torch==1.13.1 \\\n",
        "    torchdata==0.5.1 --quiet\n",
        "\n",
        "%pip install \\\n",
        "    transformers==4.27.2 \\\n",
        "    datasets==2.11.0 --quiet\\\n",
        "    evaluate==0.4.0 \\\n",
        "    rouge_score==0.1.2 \\\n",
        "    loralib==0.1.1 \\\n",
        "    peft==0.3.0 --quiet"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/lvwerra/trl.git@25fa1bd\n",
        "!pip install tensorflow"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FTz_u20iBGNR",
        "outputId": "48b7fb61-f601-41d3-f771-0bc5d8b109a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/lvwerra/trl.git@25fa1bd\n",
            "  Cloning https://github.com/lvwerra/trl.git (to revision 25fa1bd) to /tmp/pip-req-build-g55a65vb\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/lvwerra/trl.git /tmp/pip-req-build-g55a65vb\n",
            "\u001b[33m  WARNING: Did not find branch or tag '25fa1bd', assuming revision or ref.\u001b[0m\u001b[33m\n",
            "\u001b[0m  Running command git checkout -q 25fa1bd\n",
            "  Resolved https://github.com/lvwerra/trl.git to commit 25fa1bd\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from trl==0.4.2.dev0) (1.13.1)\n",
            "Requirement already satisfied: transformers>=4.18.0 in /usr/local/lib/python3.10/dist-packages (from trl==0.4.2.dev0) (4.27.2)\n",
            "Requirement already satisfied: numpy>=1.18.2 in /usr/local/lib/python3.10/dist-packages (from trl==0.4.2.dev0) (1.23.5)\n",
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (from trl==0.4.2.dev0) (0.24.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from trl==0.4.2.dev0) (2.11.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl==0.4.2.dev0) (4.5.0)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl==0.4.2.dev0) (11.7.99)\n",
            "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl==0.4.2.dev0) (8.5.0.96)\n",
            "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl==0.4.2.dev0) (11.10.3.66)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch>=1.4.0->trl==0.4.2.dev0) (11.7.99)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.4.0->trl==0.4.2.dev0) (67.7.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.4.0->trl==0.4.2.dev0) (0.41.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (0.19.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (2.31.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (0.13.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.18.0->trl==0.4.2.dev0) (4.66.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate->trl==0.4.2.dev0) (5.9.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (0.3.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (1.5.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (0.70.14)\n",
            "Requirement already satisfied: fsspec>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]>=2021.11.1->datasets->trl==0.4.2.dev0) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (3.8.6)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.10/dist-packages (from datasets->trl==0.4.2.dev0) (0.18.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (3.3.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->trl==0.4.2.dev0) (1.3.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.18.0->trl==0.4.2.dev0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.18.0->trl==0.4.2.dev0) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers>=4.18.0->trl==0.4.2.dev0) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->trl==0.4.2.dev0) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->trl==0.4.2.dev0) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets->trl==0.4.2.dev0) (1.16.0)\n",
            "Building wheels for collected packages: trl\n",
            "  Building wheel for trl (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for trl: filename=trl-0.4.2.dev0-py3-none-any.whl size=67532 sha256=f5933ee651713ae901a7e4d465fef4d47e992ceacf13960577d48563b05d871b\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-lciq9t1l/wheels/24/b4/20/2fa3a1e47c0411c39e198029315e3af2a2c1d59132913f136f\n",
            "Successfully built trl\n",
            "Installing collected packages: trl\n",
            "Successfully installed trl-0.4.2.dev0\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0mRequirement already satisfied: tensorflow in /usr/local/lib/python3.10/dist-packages (2.14.0)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.5.26)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.5.4)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.9.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (16.0.6)\n",
            "Requirement already satisfied: ml-dtypes==0.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: numpy>=1.23.5 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.23.5)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow) (23.2)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.20.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow) (67.7.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.16.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.3.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (4.5.0)\n",
            "Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.14.1)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.34.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.59.2)\n",
            "Requirement already satisfied: tensorboard<2.15,>=2.14 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.14.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.15,>=2.14.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.14.0)\n",
            "Requirement already satisfied: keras<2.15,>=2.14.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.14.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow) (0.41.3)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (3.5.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (2.31.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.15,>=2.14->tensorflow) (3.0.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (5.3.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorboard<2.15,>=2.14->tensorflow) (2023.7.22)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.15,>=2.14->tensorflow) (2.1.3)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.15,>=2.14->tensorflow) (0.5.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard<2.15,>=2.14->tensorflow) (3.2.2)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "uCtnTZkkOoEp"
      },
      "source": [
        "Load the datasets, Large Language Model (LLM), tokenizer, and configurator."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from transformers import AutoModelForSeq2SeqLM\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import GenerationConfig\n",
        "import torch"
      ],
      "metadata": {
        "id": "JarlsenaRWLW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "4tjBiyjR4lAQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4ub7WCageIei"
      },
      "source": [
        "<a name='1.2'></a>\n",
        "## 2 - Sentiment Analysis without Prompt Engineering\n",
        "\n",
        "In this use case, you will be generating the sentiment of a text with the pre-trained Large Language Model (LLM) FLAN-T5 from Hugging Face. The list of available models in the Hugging Face `transformers` package can be found [here](https://huggingface.co/docs/transformers/index).\n",
        "\n",
        "Let's upload some simple dialogues from the [DialogSum](https://huggingface.co/datasets/knkarthick/dialogsum) Hugging Face dataset. This dataset contains 10,000+ dialogues with the corresponding manually labeled summaries and topics."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "huggingface_dataset_name = \"mteb/tweet_sentiment_extraction\"\n",
        "\n",
        "dataset = load_dataset(huggingface_dataset_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 264,
          "referenced_widgets": [
            "fa315a42a89e4dde975dac86d0f5fb26",
            "aa6cc3c6fe2d43adad1273178bc373e2",
            "abe6d34458ec498eb1031e565bde8e9b",
            "9588845c2d1a439486c225b70f0cd43c",
            "b6800aa585fd48b8b2b13e96b0dc7c34",
            "0b9b3213094f4196b605bca3a0620172",
            "2e895f4861bf4a4b823ab62c1bd705e6",
            "4cc4dabf33424ba9a01cf1ea85243edd",
            "8586bcf0c933475e908adbf27180e057",
            "eaedfac59fd14d5aaf233602ec38eb85",
            "d4fe70d56a55460c91491a7c5d26718e",
            "bd67e52cd6d64dc395657d2ee038b1ea",
            "4873c3e2fe9f44ed9fab0de27c15737e",
            "ccf0e00d720343d5aea6d98b55d03d50",
            "718d5a4a106b434bbf1b10458b77db2b",
            "7e5b8b0c20d046ac93f3d977325fb06a",
            "514788897bfa4f76ae4c4874ec20b1bf",
            "05752128e99f4f33b2f2e913e578a9e1",
            "90787226e5af4aa68ee09fb7bb6d0305",
            "4305e0e8a84c4e24ae01c3eef3684259",
            "313bd697bd524ed2bfa291d4fd7f6b2a",
            "671e2d7a074c44d3bb938ab020bfca14",
            "ca198d6eaab5417bb85f09bdd827c63d",
            "84f3c6dbee8d4fd29720843b7516e3cb",
            "5b7bbb10030e4b1190f5437b97e9dcbd",
            "40509705beb4423db2d7c4dce120d288",
            "a1a9147396ca4aec97834a6f3bb8941c",
            "dd300ef5ed304901b463767b5d526249",
            "691f7c5be4c2462d9009297c6b404895",
            "7366eb9e63c04c91a6fbaa32b4ff609c",
            "0df93bd518df4c38b3a196ce6bab3d2d",
            "84ee264011dc45ccb95c9b8d42c50393",
            "581f388380a04bc993d60dcf1d4c08de",
            "674b856d33154209aa8c9067cb044cf3",
            "75c6afbd6c5d4d7f88a3dc21da6e6007",
            "c8db0cbdb6b84e6cbea56676fcce5f1b",
            "36887fdc940a46c68000a072313c7eb3",
            "5fb83fd6ec404a33a22909a81833d10c",
            "f34a5ea1babc4091b4fde940562fccd4",
            "92ceaf2292794893bd25202a0feb345b",
            "b101ba55d4c549a6baba1cefcb5d8e7a",
            "02a984bca8cf414daa8a5acffe414d65",
            "89257a1421044a02a9730f369e83af9d",
            "005123f058854eb98e1e3c7b3ffb3b58",
            "b9a18fe7b9ff45b0be54cefaec75043b",
            "e7f823171cc34644b670abf66242bad1",
            "301aad102efd45a7a04bf6bf939f6d1e",
            "ad66ddeb7766494ba4109bebf4b8e106",
            "5ae7cd4448b341a29a9b155c93e31869",
            "f4b7c244823f4973958414539eb3fc7b",
            "213cab146e4045d999c73a9b0ca26df0",
            "016174ab9dbf49d482212c81dbb981a7",
            "b7b30ca1dfd6469db7674100a7890103",
            "8012464690414c639b204babbfdc19a4",
            "a23967aefbd74a02a2851308c83faa34",
            "25ff054bddf24315a1972aa2eef5d7c6",
            "13b3f1986bd04067b679f1baf757128a",
            "f2bb20f679ca42b9be09f6769fca2290",
            "be46a78b903d4d1cab2f176cb923998d",
            "ae9baeeb5c3e4939a84c688535241c6b",
            "afb1d895736d49deba179344de13fb57",
            "8c89101526a14c5cb98964b3477bf999",
            "cf08a13eddbb4dc6b29817d5b5992887",
            "106b3892ffef41a9afd0d894048fb2a6",
            "cce1fa1f22674728a2648acf60528ca5",
            "46661fc816814188968cb1595f2a952b",
            "0bc843baf5794f87ba87fdcf85840971",
            "11cfd5db51a14894b229a1d3fad868c5",
            "075b86b342ed4e3c9c1cfc387939ca68",
            "87f20690b7084c1999d3b64510048b63",
            "4953746e7b9d4c5fb8d1204c4536ea18",
            "0c2792bb1efc4d29a9bb2424253c85ce",
            "5d82a82e18b74292ba2a8762ae34bf8c",
            "f541c5a6a007405d9d1bacec8801c3e8",
            "50b509d16a4c4f258bc3698f82fc3a16",
            "d764e4a32b0549e6b781c6187fb286ef",
            "ababfdc1cb034c10a029e484f215dfa3",
            "1515c9aac6db4aa895b3407e4bce5387",
            "348e91cd883d43c9afb81986917f4f92",
            "de686e6daed747799b53dec0ef565876",
            "1420b058c09e40acb7314c87f6a5240c",
            "1caa3c8ea0834734aad5132e7a755afb",
            "6cb89871487e47299a3c71b1d8389d8c",
            "1a78ad818c9c498ea4165fcb1efd755e",
            "6445afbab9af465db73e2cecaf862dfd",
            "e5d1b05b7c844b1fa0e1eabcb1f04d14",
            "082d5ffed60942fe9f0c559dd23350cb",
            "294354385df74f1b86b2033ac8730aea"
          ]
        },
        "id": "ZV6kvMw4Rfv1",
        "outputId": "ebaf1eb8-6e59-4e9b-809b-419512b9329c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading readme:   0%|          | 0.00/22.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fa315a42a89e4dde975dac86d0f5fb26"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading and preparing dataset json/mteb--tweet_sentiment_extraction to /root/.cache/huggingface/datasets/mteb___json/mteb--tweet_sentiment_extraction-19a608df77d581ad/0.0.0/fe5dd6ea2639a6df622901539cb550cf8797e5a6b2dd7af1cf934bed8e233e6e...\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd67e52cd6d64dc395657d2ee038b1ea"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/3.63M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ca198d6eaab5417bb85f09bdd827c63d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading data:   0%|          | 0.00/465k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "674b856d33154209aa8c9067cb044cf3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Extracting data files:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b9a18fe7b9ff45b0be54cefaec75043b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "25ff054bddf24315a1972aa2eef5d7c6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating test split: 0 examples [00:00, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0bc843baf5794f87ba87fdcf85840971"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset json downloaded and prepared to /root/.cache/huggingface/datasets/mteb___json/mteb--tweet_sentiment_extraction-19a608df77d581ad/0.0.0/fe5dd6ea2639a6df622901539cb550cf8797e5a6b2dd7af1cf934bed8e233e6e. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1515c9aac6db4aa895b3407e4bce5387"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Here we print a couple of texts with their set sentiments."
      ],
      "metadata": {
        "id": "bZmN8Pl4OWnQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "example_indices = [50, 78]\n",
        "\n",
        "dash_line = '-'.join('' for x in range(100))\n",
        "\n",
        "for i, index in enumerate(example_indices):\n",
        "    print(dash_line)\n",
        "    print('Example ', i + 1)\n",
        "    print(dash_line)\n",
        "    print('Text:')\n",
        "    print(dataset['train'][index]['text'])\n",
        "    print(dash_line)\n",
        "    print('Sentiment:')\n",
        "    print(dataset['train'][index]['label_text'])\n",
        "    print(dash_line)\n",
        "    print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D9F8gOCUSMyI",
        "outputId": "7a62a3ed-f9f5-49b6-ba70-66bbebede0c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "Example  1\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Text:\n",
            " Then you should check out http://twittersucks.com and connect with other tweeple who hate twitter\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Sentiment:\n",
            "neutral\n",
            "---------------------------------------------------------------------------------------------------\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Example  2\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Text:\n",
            "I am sooo tired\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Sentiment:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l7Z9OrrxeIek"
      },
      "source": [
        "Load the [FLAN-T5 model](https://huggingface.co/docs/transformers/model_doc/flan-t5), creating an instance of the `AutoModelForSeq2SeqLM` class with the `.from_pretrained()` method."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iAYlS40Z3l-v",
        "tags": [],
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "c04a28f0107846099e4a9fd8cd757927",
            "82693147e592427696e2b30291217344",
            "c109dd449d4a47e99ee16aaace8cfd1e",
            "f4ecf83e19ae4bf8989d890034c3b082",
            "e76e401670a749998bb827c9e64d1109",
            "958997ab5a844dc298917f6dc6fa4282",
            "a0eb882007bb422b92828a441fabf534",
            "3724d1a6f396425f8ac300fce8b9ef0f",
            "17e7590e9be740f5a6c7c98701e5345f",
            "e308beca84404b7a801c1475db8ee224",
            "25b45337013e4aa1899f68b1743f81f5",
            "8d44537a13d24dd7bcad23a09c44430c",
            "f4712c6529344627b1ce2ed2916616de",
            "1e1ad980551648e79e44c51e56c4279b",
            "c1033af6365744d18031677d22e3cdd8",
            "50453614545b423798f13927de392cca",
            "81bdc0ace86b4edaabd287acf25d9e76",
            "46f7015b0b2e461b984f9d4b163a5c17",
            "84d4704e07ba452d9175f956fbc1f108",
            "6aadf6f6d5494751b1e82bc7608acc46",
            "a73e6e041f2346bd8257f9d3845ee9d1",
            "ea86fb104fa14e4994d367ded8adb995",
            "a3f5115349064533844473a0d013a794",
            "59d27aa07c96424ea11c07f979b5fc5f",
            "d31f9c19bef74bc5abc79cde9d98a2e2",
            "ee92bf7081c94bb495564eea0d6bdcc1",
            "8d09e7956ae8432eb2eac998afc4e486",
            "64cee5a704f94ccd93ff8d35f753f0f8",
            "436816ee5f96461da2930ee2a4beebc7",
            "b8633a7fbb3d4a17a395aac4bb385bc1",
            "5057a710468844f38e0621d3465c2d9f",
            "cd155ec4b4e94cb3bdb07d2f2e8b8124",
            "5fdbfada98ac4587acfa4df77aedcdec"
          ]
        },
        "outputId": "e1461cfb-12c9-4fde-8fe6-130cc9d0cbfe"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.40k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c04a28f0107846099e4a9fd8cd757927"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/990M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8d44537a13d24dd7bcad23a09c44430c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/147 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a3f5115349064533844473a0d013a794"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "model_name='google/flan-t5-base'\n",
        "\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HX1BCQVO99vm",
        "outputId": "4a628d63-c99e-413d-ac1d-bdd6eca6c2d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5ForConditionalGeneration(\n",
              "  (shared): Embedding(32128, 768)\n",
              "  (encoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sPqQA3TT3l_I",
        "tags": []
      },
      "source": [
        "To perform encoding and decoding, you need to work with text in a tokenized form. **Tokenization** is the process of splitting texts into smaller units that can be processed by the LLM models.\n",
        "\n",
        "Download the tokenizer for the FLAN-T5 model using `AutoTokenizer.from_pretrained()` method. Parameter `use_fast` switches on fast tokenizer. At this stage, there is no need to go into the details of that, but you can find the tokenizer parameters in the [documentation](https://huggingface.co/docs/transformers/v4.28.1/en/model_doc/auto#transformers.AutoTokenizer)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "259b48cef71f4e9eae7c2e90913a68e4",
            "3bf47f86b4754111ae9dc6f8160b373a",
            "6b6e537c13874ccea66138f1425f8741",
            "d95aac342e564b60b9044f6e9a51c878",
            "bd76230047964a6fa458804dc72d8dd6",
            "b5a439fc30b84059a3cc544be9f08457",
            "1f45d629088b4297b07631e316bd9db4",
            "0f15103066944f639eecf79d2e4c5a35",
            "ed5693073e84432a99bb3b5f5cf02b51",
            "68c71841225442f0b971c3fe8d2afbd5",
            "d5f4703c62a84592a9c243b72a189501",
            "00dc6ed907c84963a1da385b940d32e2",
            "4f7186ea58ca4ce298d901e367d2f22f",
            "983074c1ba4645e09a60b6626aafb56d",
            "adc9d728aa9d4bc0a616f3a9f7d53ebf",
            "5180cb9fd5834f02b875e04fe73395a9",
            "cad42931aa544ae380d8161b13717dae",
            "cb49263d767a4d41aeb91e725e32afed",
            "a7fe4f390e6b49d3b9e5602ea49f0f35",
            "fa6d0e9597224b4c80b2e5fe3035fb71",
            "3c32b998fc44460c8741c8a42741d0ba",
            "844e6a4873d647f48a2ae140cb678834",
            "3243d6c5151e432db2b4cc0a7a074cee",
            "27c523ef742a4c8e9d6d2d08b443c2d1",
            "ddbd2cfc385d4f479cfbc5290eacd7dd",
            "4d1720c2bdf04a4a9a3009f2c717bff0",
            "aaaecb19907f4800a5f0fda1e02c64a0",
            "d678d1af21ea4f7aa98d83147982a548",
            "5ea4aa09209c458c8e301c449423e245",
            "84328b93859743a492a9854e9d725aec",
            "d4c47f06da60453ea3bd8e5bfbc32d9d",
            "6aa638dad108411cab9085aaea04b977",
            "9c6a1a729256471288d4fc744ce77cad",
            "64e1646abe6049fa836cc87433066dc5",
            "b3aad4a6391b4b71b2e3d825ff5deb68",
            "ec850d668030446d87ace2ba7723c89c",
            "1cca4e1784fa4f7c87219971c0179c26",
            "61588c54d0234d5ba298695b1791a2d6",
            "5e162bdaa9014495be31c241b68b45d2",
            "53c9dd79c1db418197250f4e5f3abde5",
            "b5baea4792e443ee92b85a897b59577e",
            "c8238751a79841e1a1e380eef5579794",
            "6f6da7d3b6954275889b9faae85eb73a",
            "597c37d9d48b4c8ea2cabd23032a29f3"
          ]
        },
        "id": "2fhgu07iS9LS",
        "outputId": "d18e1751-53af-43c1-bcc1-b43d39564ba8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "259b48cef71f4e9eae7c2e90913a68e4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "00dc6ed907c84963a1da385b940d32e2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3243d6c5151e432db2b4cc0a7a074cee"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "64e1646abe6049fa836cc87433066dc5"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "jBUCukBteIel"
      },
      "source": [
        "Test the tokenizer encoding and decoding a simple sentence:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"What time is it, Tom?\"\n",
        "\n",
        "sentence_encoded = tokenizer(sentence, return_tensors='pt')\n",
        "\n",
        "sentence_decoded = tokenizer.decode(\n",
        "        sentence_encoded[\"input_ids\"][0],\n",
        "        skip_special_tokens=True\n",
        "    )\n",
        "\n",
        "print('ENCODED SENTENCE:')\n",
        "print(sentence_encoded[\"input_ids\"][0])\n",
        "print('\\nDECODED SENTENCE:')\n",
        "print(sentence_decoded)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m53CPsqpS_Mo",
        "outputId": "4255f65d-e53e-4029-a725-f8db907627f3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ENCODED SENTENCE:\n",
            "tensor([ 363,   97,   19,   34,    6, 3059,   58,    1])\n",
            "\n",
            "DECODED SENTENCE:\n",
            "What time is it, Tom?\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9fcfSr8QeIel"
      },
      "source": [
        "Now it's time to explore how well the base LLM assigns sentiment without any prompt engineering. **Prompt engineering** is an act of a human changing the **prompt** (input) to improve the response for a given task."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i, index in enumerate(example_indices):\n",
        "    text = dataset['train'][index]['text']\n",
        "    sentiment = dataset['train'][index]['label_text']\n",
        "\n",
        "    inputs = tokenizer(text, return_tensors='pt').to(device)\n",
        "    output = tokenizer.decode(\n",
        "        model.generate(\n",
        "            inputs[\"input_ids\"],\n",
        "            max_new_tokens=50,\n",
        "        )[0],\n",
        "        skip_special_tokens=True\n",
        "    )\n",
        "\n",
        "    print(dash_line)\n",
        "    print('Example ', i + 1)\n",
        "    print(dash_line)\n",
        "    print(f'INPUT PROMPT:\\n{text}')\n",
        "    print(dash_line)\n",
        "    print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}')\n",
        "    print(dash_line)\n",
        "    print(f'MODEL GENERATION - WITHOUT PROMPT ENGINEERING:\\n{output}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IaZYUaMgTM_y",
        "outputId": "9cc39c17-8e59-4810-fd40-3e7791203c88"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "Example  1\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            " Then you should check out http://twittersucks.com and connect with other tweeple who hate twitter\n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "neutral\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - WITHOUT PROMPT ENGINEERING:\n",
            "If you're not a twitter user, then you should check out http://twitterucks.com and connect with other twitter users who hate twitter.\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Example  2\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            "I am sooo tired\n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - WITHOUT PROMPT ENGINEERING:\n",
            "I am so tired.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nwd9h9vLeIem"
      },
      "source": [
        "You can see that the guesses of the model make some sense, but it doesn't seem to be sure what task it is supposed to accomplish. Seems it just makes up the next sentence in the dialogue. Prompt engineering can help here."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3'></a>\n",
        "## 3 - Summarize Dialogue with an Instruction Prompt\n",
        "\n",
        "Prompt engineering is an important concept in using foundation models for text generation. You can check out [this blog](https://www.amazon.science/blog/emnlp-prompt-engineering-is-the-new-feature-engineering) from Amazon Science for a quick introduction to prompt engineering."
      ],
      "metadata": {
        "id": "fbLHrMW0ULgZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3.1'></a>\n",
        "### 3.1 - Zero Shot Inference with an Instruction Prompt\n",
        "\n",
        "In order to instruct the model to perform a task - summarize a dialogue - you can take the dialogue and convert it into an instruction prompt. This is often called **zero shot inference**.  You can check out [this blog from AWS](https://aws.amazon.com/blogs/machine-learning/zero-shot-prompting-for-the-flan-t5-foundation-model-in-amazon-sagemaker-jumpstart/) for a quick description of what zero shot learning is and why it is an important concept to the LLM model.\n",
        "\n",
        "Wrap the dialogue in a descriptive instruction and see how the generated text will change:"
      ],
      "metadata": {
        "id": "-__MxGddUOcH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for i, index in enumerate(example_indices):\n",
        "    text = dataset['train'][index]['text']\n",
        "    sentiment = dataset['train'][index]['label_text']\n",
        "\n",
        "    prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral\n",
        "\n",
        "{text}\n",
        "\n",
        "Sentiment:\n",
        "    \"\"\"\n",
        "\n",
        "    # Input constructed prompt instead of the dialogue.\n",
        "    inputs = tokenizer(prompt, return_tensors='pt').to(device)\n",
        "    output = tokenizer.decode(\n",
        "        model.generate(\n",
        "            inputs[\"input_ids\"],\n",
        "            max_new_tokens=50,\n",
        "        )[0],\n",
        "        skip_special_tokens=True\n",
        "    )\n",
        "\n",
        "    print(dash_line)\n",
        "    print('Example ', i + 1)\n",
        "    print(dash_line)\n",
        "    print(f'INPUT PROMPT:\\n{prompt}')\n",
        "    print(dash_line)\n",
        "    print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}')\n",
        "    print(dash_line)\n",
        "    print(f'MODEL GENERATION - ZERO SHOT:\\n{output}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NcQUy2DeUQcf",
        "outputId": "5e70a266-7771-4fc9-8e13-317577e4686d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "Example  1\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral\n",
            "\n",
            " Then you should check out http://twittersucks.com and connect with other tweeple who hate twitter\n",
            "\n",
            "Sentiment:\n",
            "    \n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "neutral\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - ZERO SHOT:\n",
            "negative\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Example  2\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral\n",
            "\n",
            "I am sooo tired\n",
            "\n",
            "Sentiment:\n",
            "    \n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - ZERO SHOT:\n",
            "negative\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e8Fa44fveIen"
      },
      "source": [
        "This is much better! But the model still does not pick up on the nuance of the conversations though."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3.2'></a>\n",
        "### 3.2 - Zero Shot Inference with the Prompt Template from FLAN-T5\n",
        "\n",
        "Let's use a slightly different prompt. FLAN-T5 has many prompt templates that are published for certain tasks [here](https://github.com/google-research/FLAN/tree/main/flan/v2). In the following code, you will use one of the [pre-built FLAN-T5 prompts](https://github.com/google-research/FLAN/blob/main/flan/v2/templates.py):"
      ],
      "metadata": {
        "id": "A2wOgIvxU8M4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for i, index in enumerate(example_indices):\n",
        "    text = dataset['train'][index]['text']\n",
        "    sentiment = dataset['train'][index]['label_text']\n",
        "\n",
        "    prompt = f\"\"\"\n",
        "Text:\n",
        "\n",
        "{text}\n",
        "\n",
        "Provide sentiment of the following text as either positive, negative or neutral\n",
        "\"\"\"\n",
        "\n",
        "    inputs = tokenizer(prompt, return_tensors='pt').to(device)\n",
        "    output = tokenizer.decode(\n",
        "        model.generate(\n",
        "            inputs[\"input_ids\"],\n",
        "            max_new_tokens=50,\n",
        "        )[0],\n",
        "        skip_special_tokens=True\n",
        "    )\n",
        "\n",
        "    print(dash_line)\n",
        "    print('Example ', i + 1)\n",
        "    print(dash_line)\n",
        "    print(f'INPUT PROMPT:\\n{prompt}')\n",
        "    print(dash_line)\n",
        "    print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}\\n')\n",
        "    print(dash_line)\n",
        "    print(f'MODEL GENERATION - ZERO SHOT:\\n{output}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zDeNqAWjU9Ta",
        "outputId": "0ff33440-136c-4252-d294-cdcdf82d816e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "Example  1\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            "\n",
            "Text:\n",
            "\n",
            " Then you should check out http://twittersucks.com and connect with other tweeple who hate twitter\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "neutral\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - ZERO SHOT:\n",
            "negative\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "Example  2\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INPUT PROMPT:\n",
            "\n",
            "Text:\n",
            "\n",
            "I am sooo tired\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - ZERO SHOT:\n",
            "negative\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wp7V19dJeIeo"
      },
      "source": [
        "Notice that this prompt from FLAN-T5 did help a bit, but still struggles to pick up on the nuance of the conversation. This is what you will try to solve with the few shot inferencing."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='4'></a>\n",
        "## 4 - Summarize Dialogue with One Shot and Few Shot Inference\n",
        "\n",
        "**One shot and few shot inference** are the practices of providing an LLM with either one or more full examples of prompt-response pairs that match your task - before your actual prompt that you want completed. This is called \"in-context learning\" and puts your model into a state that understands your specific task.  You can read more about it in [this blog from HuggingFace](https://huggingface.co/blog/few-shot-learning-gpt-neo-and-inference-api)."
      ],
      "metadata": {
        "id": "UiT8bumNWXQR"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='4.1'></a>\n",
        "### 4.1 - One Shot Inference\n",
        "\n",
        "Let's build a function that takes a list of `example_indices_full`, generates a prompt with full examples, then at the end appends the prompt which you want the model to complete (`example_index_to_summarize`).  You will use the same FLAN-T5 prompt template from section [3.2](#3.2)."
      ],
      "metadata": {
        "id": "ScFo5WOmWa-v"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def make_prompt(example_indices_full, example_index_to_summarize):\n",
        "    prompt = ''\n",
        "    for index in example_indices_full:\n",
        "        text = dataset['train'][index]['text']\n",
        "        sentiment = dataset['train'][index]['label_text']\n",
        "\n",
        "        # The stop sequence '{summary}\\n\\n\\n' is important for FLAN-T5. Other models may have their own preferred stop sequence.\n",
        "        prompt += f\"\"\"\n",
        "Text:\n",
        "\n",
        "{text}\n",
        "\n",
        "Provide sentiment of the following text as either positive, negative or neutral\n",
        "{sentiment}\n",
        "\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "    text = dataset['test'][example_index_to_summarize]['text']\n",
        "\n",
        "    prompt += f\"\"\"\n",
        "Text:\n",
        "\n",
        "{text}\n",
        "\n",
        "Provide sentiment of the following text as either positive, negative or neutral\n",
        "\"\"\"\n",
        "\n",
        "    return prompt"
      ],
      "metadata": {
        "id": "NRiNjKaYWfL7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "70pLWB_NeIep"
      },
      "source": [
        "Construct the prompt to perform one shot inference:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example_indices_full = [40]\n",
        "example_index_to_summarize = 200\n",
        "\n",
        "one_shot_prompt = make_prompt(example_indices_full, example_index_to_summarize)\n",
        "\n",
        "print(one_shot_prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jBYcNLP9W-FS",
        "outputId": "e3c45be7-f41b-4f16-d064-321976e85fd9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Text:\n",
            "\n",
            " Car not happy, big big dent in boot! Hoping theyre not going to write it off, crossing fingers and waiting\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "neutral\n",
            "\n",
            "\n",
            "\n",
            "Text:\n",
            "\n",
            "St joe is dirty.\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "UBvYT3e0eIep"
      },
      "source": [
        "Now pass this prompt to perform the one shot inference:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentiment = dataset['train'][example_index_to_summarize]['label_text']\n",
        "\n",
        "inputs = tokenizer(one_shot_prompt, return_tensors='pt').to(device)\n",
        "output = tokenizer.decode(\n",
        "    model.generate(\n",
        "        inputs[\"input_ids\"],\n",
        "        max_new_tokens=50,\n",
        "    )[0],\n",
        "    skip_special_tokens=True\n",
        ")\n",
        "\n",
        "print(dash_line)\n",
        "print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}\\n')\n",
        "print(dash_line)\n",
        "print(f'MODEL GENERATION - ONE SHOT:\\n{output}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2eT-XkLpXDLR",
        "outputId": "a5917b84-d541-4b09-a746-bdb9ff77a576"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - ONE SHOT:\n",
            "negative\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='4.2'></a>\n",
        "### 4.2 - Few Shot Inference\n",
        "\n",
        "Let's explore few shot inference by adding two more full dialogue-summary pairs to your prompt."
      ],
      "metadata": {
        "id": "pRBGZjndXQkM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "example_indices_full = [40, 80, 120]\n",
        "example_index_to_summarize = 200\n",
        "\n",
        "few_shot_prompt = make_prompt(example_indices_full, example_index_to_summarize)\n",
        "\n",
        "print(few_shot_prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wSTRq4_AXRkw",
        "outputId": "c814d164-98d3-4635-adc7-a76d248096fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Text:\n",
            "\n",
            " Car not happy, big big dent in boot! Hoping theyre not going to write it off, crossing fingers and waiting\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "neutral\n",
            "\n",
            "\n",
            "\n",
            "Text:\n",
            "\n",
            " THANK YYYYYYYYYOOOOOOOOOOUUUUU!\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "positive\n",
            "\n",
            "\n",
            "\n",
            "Text:\n",
            "\n",
            " I had it! On my itunes, but then I lost all my songs.\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral \n",
            "neutral\n",
            "\n",
            "\n",
            "\n",
            "Text:\n",
            "\n",
            "St joe is dirty.\n",
            "\n",
            "Provide sentiment of the following text as either positive, negative or neutral\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "ZbnuZeLdeIev"
      },
      "source": [
        "Now pass this prompt to perform a few shot inference:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentiment = dataset['train'][example_index_to_summarize]['label_text']\n",
        "\n",
        "inputs = tokenizer(few_shot_prompt, return_tensors='pt').to(device)\n",
        "output = tokenizer.decode(\n",
        "    model.generate(\n",
        "        inputs[\"input_ids\"],\n",
        "        max_new_tokens=50,\n",
        "    )[0],\n",
        "    skip_special_tokens=True\n",
        ")\n",
        "\n",
        "print(dash_line)\n",
        "print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}\\n')\n",
        "print(dash_line)\n",
        "print(f'MODEL GENERATION - FEW SHOT:\\n{output}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7_o0lg-gXXlB",
        "outputId": "55458ee9-9dd7-4a3f-acb2-2e1963bd9fc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "\n",
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - FEW SHOT:\n",
            "negative\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "QwShav5geIew"
      },
      "source": [
        "In this case, few shot did not provide much of an improvement over one shot inference.  And, anything above 5 or 6 shot will typically not help much, either.  Also, you need to make sure that you do not exceed the model's input-context length which, in our case, if 512 tokens.  Anything above the context length will be ignored.\n",
        "\n",
        "However, you can see that feeding in at least one full example (one shot) provides the model with more information and qualitatively improves the sentiment generated overall."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='5'></a>\n",
        "## 5 - Generative Configuration Parameters for Inference"
      ],
      "metadata": {
        "id": "ZJ1vxADAXg-o"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "tMSbxqp3eIew"
      },
      "source": [
        "You can change the configuration parameters of the `generate()` method to see a different output from the LLM. So far the only parameter that you have been setting was `max_new_tokens=50`, which defines the maximum number of tokens to generate. A full list of available parameters can be found in the [Hugging Face Generation documentation](https://huggingface.co/docs/transformers/v4.29.1/en/main_classes/text_generation#transformers.GenerationConfig).\n",
        "\n",
        "A convenient way of organizing the configuration parameters is to use `GenerationConfig` class."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# generation_config = GenerationConfig(max_new_tokens=50)\n",
        "# generation_config = GenerationConfig(max_new_tokens=10)\n",
        "# generation_config = GenerationConfig(max_new_tokens=50, do_sample=True, temperature=0.1)\n",
        "# generation_config = GenerationConfig(max_new_tokens=50, do_sample=True, temperature=0.5)\n",
        "generation_config = GenerationConfig(max_new_tokens=50, do_sample=True, temperature=1.0)\n",
        "\n",
        "inputs = tokenizer(few_shot_prompt, return_tensors='pt').to(device)\n",
        "output = tokenizer.decode(\n",
        "    model.generate(\n",
        "        inputs[\"input_ids\"],\n",
        "        generation_config=generation_config,\n",
        "    )[0],\n",
        "    skip_special_tokens=True\n",
        ")\n",
        "\n",
        "print(dash_line)\n",
        "print(f'MODEL GENERATION - FEW SHOT:\\n{output}')\n",
        "print(dash_line)\n",
        "print(f'BASELINE HUMAN SUMMARY:\\n{sentiment}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pJixvuqzXhmx",
        "outputId": "169f3a27-422a-4119-84df-9b46b78f6a6c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "MODEL GENERATION - FEW SHOT:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tzcpfxcLeIex"
      },
      "source": [
        "Comments related to the choice of the parameters in the code cell above:\n",
        "- Choosing `max_new_tokens=10` will make the output text too short, so the text sentiment will be cut.\n",
        "- Putting `do_sample = True` and changing the temperature value you get more flexibility in the output."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qlnrwsZQeIex"
      },
      "source": [
        "As you can see, prompt engineering can take you a long way for this use case, but there are some limitations. Next, you will start to explore how you can use fine-tuning to help your LLM to understand a particular use case in better depth!"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Lab 2: Fine-Tune a Generative AI Model for Dialogue Summarization"
      ],
      "metadata": {
        "id": "DQamNAJYXqz2"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5jTvJUGHvx5f"
      },
      "source": [
        "In this section, we will fine-tune an existing LLM from Hugging Face for enhanced sentiment analysis. We will use the [FLAN-T5](https://huggingface.co/docs/transformers/model_doc/flan-t5) model, which provides a high quality instruction tuned model and can summarize text out of the box. To improve the inferences, we will explore a full fine-tuning approach and evaluate the results with ROUGE metrics. Then we will perform Parameter Efficient Fine-Tuning (PEFT), evaluate the resulting model and see that the benefits of PEFT outweigh the slightly-lower performance metrics."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='1.1'></a>\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "### 1.1 - Install Required Dependencies"
      ],
      "metadata": {
        "id": "0REuYtpDP_5a"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import the necessary components. Some of them are new for this section, they will be discussed later in the notebook."
      ],
      "metadata": {
        "id": "enh3B_UvQIP3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, GenerationConfig, TrainingArguments, Trainer\n",
        "import torch\n",
        "import time\n",
        "import evaluate\n",
        "import pandas as pd\n",
        "import numpy as np"
      ],
      "metadata": {
        "id": "d1FuKPbSX7Ck"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "txWexIWhvx5_"
      },
      "source": [
        "It is possible to pull out the number of model parameters and find out how many of them are trainable. The following function can be used to do that, at this stage, you do not need to go into details of it."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def print_number_of_trainable_model_parameters(model):\n",
        "    trainable_model_params = 0\n",
        "    all_model_params = 0\n",
        "    for _, param in model.named_parameters():\n",
        "        all_model_params += param.numel()\n",
        "        if param.requires_grad:\n",
        "            trainable_model_params += param.numel()\n",
        "    return f\"trainable model parameters: {trainable_model_params}\\nall model parameters: {all_model_params}\\npercentage of trainable model parameters: {100 * trainable_model_params / all_model_params:.2f}%\"\n",
        "\n",
        "print(print_number_of_trainable_model_parameters(model))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z51z6VC7Y-UC",
        "outputId": "913a90b4-735e-40f9-ffd5-599d93fb7d18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable model parameters: 247577856\n",
            "all model parameters: 247577856\n",
            "percentage of trainable model parameters: 100.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='2'></a>\n",
        "## 2 - Perform Full Fine-Tuning"
      ],
      "metadata": {
        "id": "_5gyc5DaZKEi"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "QfwHnz2Gvx6G"
      },
      "source": [
        "<a name='2.1'></a>\n",
        "### 2.1 - Preprocess the Text-Sentiment Dataset\n",
        "\n",
        "You need to convert the dialog-summary (prompt-response) pairs into explicit instructions for the LLM. Prepend an instruction to the start of the dialog with `Find the sentiment` and to the start of the summary with `Sentiment` as follows:\n",
        "\n",
        "Training prompt (dialogue):\n",
        "```\n",
        "Summarize the following conversation.\n",
        "\n",
        "    Chris: This is his part of the conversation.\n",
        "    Antje: This is her part of the conversation.\n",
        "    \n",
        "Summary:\n",
        "```\n",
        "\n",
        "Training response (summary):\n",
        "```\n",
        "Both Chris and Antje participated in the conversation.\n",
        "```\n",
        "\n",
        "Then preprocess the prompt-response dataset into tokens and pull out their `input_ids` (1 per token)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_function(example):\n",
        "    start_prompt = 'Provide sentiment of the following text as either positive, negative or neutral.\\n\\n'\n",
        "    end_prompt = '\\n\\Sentiment: '\n",
        "    prompt = [start_prompt + text + end_prompt for dialogue in example[\"text\"]]\n",
        "    example['input_ids'] = tokenizer(prompt, padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n",
        "    example['labels'] = tokenizer(example[\"label_text\"], padding=\"max_length\", truncation=True, return_tensors=\"pt\").input_ids\n",
        "\n",
        "    return example\n",
        "\n",
        "# The dataset actually contains 3 diff splits: train, validation, test.\n",
        "# The tokenize_function code is handling all data across all splits in batches.\n",
        "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
        "tokenized_datasets = tokenized_datasets.remove_columns(['id', 'label', 'text', 'label_text',])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "7d1941d77cee464b99b6e1ec22652356",
            "c17ff21fbadf416fbcf33e29931430a4",
            "fe4fb6fb669143e1ab64f68ba73fd069",
            "d02e589963c14bdb80fd437c8dc787b7",
            "9d7187b6dded4ca6b4f447aee2282b68",
            "2483db5361cd4009af9a9fc36299de16",
            "c50a280ac3154f218750079d1bd057f9",
            "9004c730369a4bd1825bc3092f3b3131",
            "907cf2503521408082e3e035f635e006",
            "97df134d8e294756ad1ebf2be81a2e4b",
            "01edcec6037542e98815091b9135c2c9",
            "395665fc5f844aa0890792b51fc73542",
            "1a1599a7fca547cc961eca6bc4db4702",
            "2d5c8c790d70468ba03bc0bba9424cc4",
            "914313f0d7584c5d80c6fe13c7cd3527",
            "7b1136e6579d4b07ae12772651ebf64a",
            "5bb44ff635814e0191bd577f52a02ea6",
            "fefcdb2079614dbf9efedaa4cca11f53",
            "f9f6ab57d69e4e83bcf5c35e6819b7e9",
            "f3b8926b3f1342739287e2435ef92c71",
            "f9d00af3444f49a68b5d58c95967dde8",
            "4b5657e9cf0a4117893eca4a4a54ae36"
          ]
        },
        "id": "75SutTrP2y44",
        "outputId": "d6995f48-5eaf-4fcf-8733-bcca2e928208"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/27481 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7d1941d77cee464b99b6e1ec22652356"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/3534 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "395665fc5f844aa0890792b51fc73542"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "jp4029JNvx6J"
      },
      "source": [
        "To save some time in the lab, you will subsample the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_datasets = tokenized_datasets.filter(lambda example, index: index % 100 == 0, with_indices=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17,
          "referenced_widgets": [
            "b09ee8f4e0d8477e8de4bbb322b88c1c",
            "0af60aa599814a7c8e1e2f6482db5fb2",
            "bb53b029cce8462490184eff62036ae4",
            "23a8af087ce54fb5a877aabd22878ccd",
            "13e61905a7e34694b8b919c0b19edcbe",
            "1205008e62a5412ca720131248911091",
            "b6e5cfd9de694609bb360f511277e1f9",
            "b06e6951fbec46d0bb5561e835d68e11",
            "292c734c20964d299b6bbec7b9d088e6",
            "5f0570fb278c4fe7aa742ac40fe2e0dc",
            "6952242dc85f49cf95b20796564d81dc",
            "f19c4e3657764979be54ae160b15c317",
            "eb3860488e334dbd9fc42ee0a40aa167",
            "f411c3f9291f42f9909be260e69de775",
            "3feb5bafda1c409bb1d7715f6d035f71",
            "e3a7c3e182cc48ddbc46db4adda24164",
            "c862d57d0f1b40c9aefcab273dc3bd1a",
            "682739da17f049f89a02ecef101faabb",
            "c2436cb1cc484062ae70eae1d2dea7c8",
            "945694b20c0e41a1ad7707e0830a4b9c",
            "f9da0be432fc40e081a1bc6524eeb037",
            "1dc8a33f68ac4cb9a3201d2ccf0fec6c"
          ]
        },
        "id": "qZEpbbFo3DKZ",
        "outputId": "c71c164a-32a9-428f-ab38-0ee4d7cf6ca2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Filter:   0%|          | 0/27481 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b09ee8f4e0d8477e8de4bbb322b88c1c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Filter:   0%|          | 0/3534 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f19c4e3657764979be54ae160b15c317"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "zXIvS6IXvx6L"
      },
      "source": [
        "Check the shapes of all three parts of the dataset:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(f\"Shapes of the datasets:\")\n",
        "print(f\"Training: {tokenized_datasets['train'].shape}\")\n",
        "print(f\"Test: {tokenized_datasets['test'].shape}\")\n",
        "\n",
        "print(tokenized_datasets)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ESNZ67tw3JBI",
        "outputId": "0c1638bd-97ba-40b0-db7c-f62e483df02d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Shapes of the datasets:\n",
            "Training: (275, 2)\n",
            "Test: (36, 2)\n",
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['input_ids', 'labels'],\n",
            "        num_rows: 275\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['input_ids', 'labels'],\n",
            "        num_rows: 36\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zooxAv4Avx6M"
      },
      "source": [
        "The output dataset is ready for fine-tuning."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wBlKhqMXvx6N"
      },
      "source": [
        "<a name='2.2'></a>\n",
        "### 2.2 - Fine-Tune the Model with the Preprocessed Dataset\n",
        "\n",
        "Now utilize the built-in Hugging Face `Trainer` class (see the documentation [here](https://huggingface.co/docs/transformers/main_classes/trainer)). Pass the preprocessed dataset with reference to the original model. Other training parameters are found experimentally and there is no need to go into details about those at the moment."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"philschmid/flan-t5-base-samsum\")\n",
        "instruct_model = AutoModelForSeq2SeqLM.from_pretrained(\"philschmid/flan-t5-base-samsum\")\n",
        "instruct_model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b487512585824bbe85ba3971177cd295",
            "82776a3ee8234f598ad4d874250b9086",
            "5779147f16c7422f9c685a1406a0fa49",
            "13842ad44c4a46b99c46be0055f249bf",
            "b5061c8e233949869dfe8ae2b00106fa",
            "b90502f06ca541bea8994bd8f51f3211",
            "a2b5d813a32a435cb4866af3c201738b",
            "48b0ae690a4441fea4e95503fe8d6287",
            "bf53071aaa384114854e4e7fb8cbbe75",
            "f8dae7f260c343ac8c40c2fa59b7a6f3",
            "4ffca50e1aa741e6b6ea1552080a7cd0",
            "9811eff0f3b042f294ec5fdf98122c5c",
            "e1518b42f3b0426086d0e18b28a3b447",
            "9565180c73b8409aa4050ab0e693a2ec",
            "eda09b9de67348bfb58e5e4d0a10e48b",
            "beea6e223bf8434aa7ead688555a1f86",
            "c17d0f92f46a42838a7fa0bf2bf03d11",
            "722ada0e93064d90899d36fcb015b4e1",
            "40a1c955463f41aba8081de32320b3c0",
            "f80e3c870c174e67bb2132fc1ba153b6",
            "f85812f47eeb46f6baf07f6b832d297a",
            "ba1130807c4840338e6f50d14d5d5897",
            "0d87c3ea49364d418940a21ce4852500",
            "aed034790b134350bcf3231f54b724ae",
            "0aa1967736c4441b93fb335ae22bd465",
            "55f80eaecce94f18b8b81fe0cd86c6fa",
            "9a68adc65f9341cd8e021963cf33617a",
            "0b0fb9b691bc451e939200eca050cea1",
            "7a38e14610a14d3fad231be756a6657f",
            "2ec78b7130014ce79aa9971b6172112a",
            "32227dd9cc9f45358b0e8a8c96692cee",
            "da14c6bd122040d7b1123aba2972217d",
            "5ba2bb188f574123a7657a4b3fa96dfd",
            "0e9955a9b4d44460b19f46a73c189514",
            "41e6191348c14dc3a078f46e23ca551d",
            "4555d0ccd2ba494d9fd5bb3281635021",
            "b29158ae7a9e40d998234b4c93e7e8b0",
            "03d3533b81614153b017ae6d31a90ae0",
            "4c8ac0ee212f4bfa8fc90101bc473d00",
            "5c127dc1afde4a579fd0b05226915e9e",
            "b5cf5fe5eba44a9d95ae1c4451ba4a88",
            "3805bfeaa3c3402888ff0391ede18eea",
            "0efada7fd17041c1910bb96dd8aba9a6",
            "0c6ab74eeb4e46218c9259d871d2fe58",
            "488ff28e86a34713be5332eb06e0f00d",
            "bd35dc7bca9d4c218da6379529daea44",
            "8d810525fcdb4ae8adb7b3e429bd82d9",
            "5f490c826150418b9557eec9530559b4",
            "cf7d1e43569642ceaf3094e0e8f0ec37",
            "45e8dfa9c3284ed8b9541cc06deb0a91",
            "c8b7bfbe5c6a4f3291974f079cba6a82",
            "f5839ab26f354dbdbcf70091b90afdfb",
            "78a97a70cf434d7d93580b76612c8429",
            "28e913aa6d234d5697a11b6bad15fe7b",
            "0d5d4bd186084a669375a3f6d9e133d6",
            "0389f8778d6e465c81c37b46fa052ab2",
            "b9362ac6a16b4273902e263545d6c2a7",
            "75c8959d820542aba2d12b9a4f21adaa",
            "2fa0fc8d8c1442b6a7ffc9c2bf7e938a",
            "03d26d66fbf8414b8456dc969f1665d1",
            "4034dcd63de74bcf8bb3291fd78d876a",
            "41f4a615779a4179943c870ebd443db9",
            "4f068e2532a247c684eec557100ec4d7",
            "d4098b9628884e5bab9d3e1840317a54",
            "1754222def1d4f0d9df1c8b3bfdec225",
            "7af63470071e43afa7db30acf4be33aa"
          ]
        },
        "id": "4wlrvEzazjYZ",
        "outputId": "a792a5ab-e9fe-4f33-d8f2-e605ab4e1fca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/2.54k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b487512585824bbe85ba3971177cd295"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "spiece.model:   0%|          | 0.00/792k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9811eff0f3b042f294ec5fdf98122c5c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/2.42M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0d87c3ea49364d418940a21ce4852500"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/2.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0e9955a9b4d44460b19f46a73c189514"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/1.53k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "488ff28e86a34713be5332eb06e0f00d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "pytorch_model.bin:   0%|          | 0.00/990M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0389f8778d6e465c81c37b46fa052ab2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5ForConditionalGeneration(\n",
              "  (shared): Embedding(32128, 768)\n",
              "  (encoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): T5Stack(\n",
              "    (embed_tokens): Embedding(32128, 768)\n",
              "    (block): ModuleList(\n",
              "      (0): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (relative_attention_bias): Embedding(32, 12)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (2): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (3): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (4): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (5): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (6): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (7): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (8): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (9): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (10): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (11): T5Block(\n",
              "        (layer): ModuleList(\n",
              "          (0): T5LayerSelfAttention(\n",
              "            (SelfAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): T5LayerCrossAttention(\n",
              "            (EncDecAttention): T5Attention(\n",
              "              (q): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (v): Linear(in_features=768, out_features=768, bias=False)\n",
              "              (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): T5LayerFF(\n",
              "            (DenseReluDense): T5DenseGatedActDense(\n",
              "              (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "              (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "              (act): NewGELUActivation()\n",
              "            )\n",
              "            (layer_norm): T5LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (final_layer_norm): T5LayerNorm()\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='2.3'></a>\n",
        "### 2.3 - Evaluate the Model Qualitatively (Human Evaluation)\n",
        "\n",
        "As with many GenAI applications, a qualitative approach where you ask yourself the question \"Is my model behaving the way it is supposed to?\" is usually a good starting point. In the example below (the same one we started this notebook with), you can see how the fine-tuned model is able to create a reasonable summary of the dialogue compared to the original inability to understand what is being asked of the model."
      ],
      "metadata": {
        "id": "AEyDFpfizzub"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "index = 200\n",
        "text = dataset['test'][index]['text']\n",
        "human_baseline_summary = dataset['test'][index]['label_text']\n",
        "\n",
        "prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral.\n",
        "\n",
        "{text}\n",
        "\n",
        "Sentiment:\n",
        "\"\"\"\n",
        "\n",
        "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "original_model_outputs = model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
        "original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
        "instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "print(dash_line)\n",
        "print(f'BASELINE HUMAN SUMMARY:\\n{human_baseline_summary}')\n",
        "print(dash_line)\n",
        "print(f'ORIGINAL MODEL:\\n{original_model_text_output}')\n",
        "print(dash_line)\n",
        "print(f'INSTRUCT MODEL:\\n{instruct_model_text_output}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0oAu8ChNzwBQ",
        "outputId": "da549a8b-2d59-49ea-a53b-237d957fc34d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "ORIGINAL MODEL:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INSTRUCT MODEL:\n",
            "negative\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='2.4'></a>\n",
        "### 2.4 - Evaluate the Model Quantitatively (with ROUGE Metric)\n",
        "\n",
        "The [ROUGE metric](https://en.wikipedia.org/wiki/ROUGE_(metric)) helps quantify the validity of summarizations produced by models. It compares summarizations to a \"baseline\" summary which is usually created by a human. While not perfect, it does indicate the overall increase in summarization effectiveness that we have accomplished by fine-tuning."
      ],
      "metadata": {
        "id": "jjVDRk8L0Ily"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = evaluate.load('rouge')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "29990dec26b14bfaad79aa51eef2f550",
            "f92c91c375d84ed6ac04d199fe78c73a",
            "4ae102c7d51346e8a3359b6e8d14e584",
            "111b52b3fa944a5d833404e44dae996c",
            "8495f79bd14948a396322793135b1bc7",
            "2cd3678ea1a34af38775535d882e2df5",
            "d9c8445f3cd04b9fb328eede838ebedb",
            "972757171ecb493d8c30e2af1ec2e08a",
            "61cd1dc950f44d0888d09d893e9881a2",
            "67801014c2d24f2bb4062e7da01f4c12",
            "2b235e38d57b49b9bd02a8934d111a6e"
          ]
        },
        "id": "yN0IHG_P0F2q",
        "outputId": "cb41398b-d1ef-437a-9abf-3e2fc60f0f13"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/6.27k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "29990dec26b14bfaad79aa51eef2f550"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EQvCoooOvx6p"
      },
      "source": [
        "Generate the outputs for the sample of the test dataset (only 10 texts and sentiments to save time), and save the results."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "texts = dataset['test'][0:10]['text']\n",
        "human_baseline_summaries = dataset['test'][0:10]['label_text']\n",
        "\n",
        "original_model_summaries = []\n",
        "instruct_model_summaries = []\n",
        "\n",
        "for _, dialogue in enumerate(texts):\n",
        "    prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral.\n",
        "\n",
        "{text}\n",
        "\n",
        "Sentiment: \"\"\"\n",
        "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "    original_model_outputs = model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
        "    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
        "    original_model_summaries.append(original_model_text_output)\n",
        "\n",
        "    instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
        "    instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
        "    instruct_model_summaries.append(instruct_model_text_output)\n",
        "\n",
        "zipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, instruct_model_summaries))\n",
        "\n",
        "df = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'instruct_model_summaries'])\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "7Q_zMXWC0NMs",
        "outputId": "046fb446-e60f-4187-fb0e-7f4c273024e0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  human_baseline_summaries original_model_summaries instruct_model_summaries\n",
              "0                  neutral                 negative                 negative\n",
              "1                 positive                 negative                 negative\n",
              "2                 negative                 negative                 negative\n",
              "3                 positive                 negative                 negative\n",
              "4                 positive                 negative                 negative\n",
              "5                 positive                 negative                 negative\n",
              "6                 negative                 negative                 negative\n",
              "7                 negative                 negative                 negative\n",
              "8                  neutral                 negative                 negative\n",
              "9                  neutral                 negative                 negative"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3fa93281-dd44-490b-95b5-d64d4d599fb5\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>human_baseline_summaries</th>\n",
              "      <th>original_model_summaries</th>\n",
              "      <th>instruct_model_summaries</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3fa93281-dd44-490b-95b5-d64d4d599fb5')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-3fa93281-dd44-490b-95b5-d64d4d599fb5 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-3fa93281-dd44-490b-95b5-d64d4d599fb5');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0b84107e-7e3b-459c-8a21-c35142a1aa08\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0b84107e-7e3b-459c-8a21-c35142a1aa08')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0b84107e-7e3b-459c-8a21-c35142a1aa08 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "zhwzmkFOvx6r"
      },
      "source": [
        "Evaluate the models computing ROUGE metrics. Notice the improvement in the results!"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "original_model_results = rouge.compute(\n",
        "    predictions=original_model_summaries,\n",
        "    references=human_baseline_summaries[0:len(original_model_summaries)],\n",
        "    use_aggregator=True,\n",
        "    use_stemmer=True,\n",
        ")\n",
        "\n",
        "instruct_model_results = rouge.compute(\n",
        "    predictions=instruct_model_summaries,\n",
        "    references=human_baseline_summaries[0:len(instruct_model_summaries)],\n",
        "    use_aggregator=True,\n",
        "    use_stemmer=True,\n",
        ")\n",
        "\n",
        "print('ORIGINAL MODEL:')\n",
        "print(original_model_results)\n",
        "print('INSTRUCT MODEL:')\n",
        "print(instruct_model_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-qmOgkA31HsL",
        "outputId": "d908f8db-1d3d-446f-83ad-eec6327aacd6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL MODEL:\n",
            "{'rouge1': 0.3, 'rouge2': 0.0, 'rougeL': 0.3, 'rougeLsum': 0.3}\n",
            "INSTRUCT MODEL:\n",
            "{'rouge1': 0.3, 'rouge2': 0.0, 'rougeL': 0.3, 'rougeLsum': 0.3}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "rkUcgdqfvx6u"
      },
      "source": [
        "The results show substantial improvement in all ROUGE metrics:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Absolute percentage improvement of INSTRUCT MODEL over HUMAN BASELINE\")\n",
        "\n",
        "improvement = (np.array(list(instruct_model_results.values())) - np.array(list(original_model_results.values())))\n",
        "for key, value in zip(instruct_model_results.keys(), improvement):\n",
        "    print(f'{key}: {value*100:.2f}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "keOJ70uC1gko",
        "outputId": "37381556-db0d-4f33-ad0a-89119f84f44e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absolute percentage improvement of INSTRUCT MODEL over HUMAN BASELINE\n",
            "rouge1: 0.00%\n",
            "rouge2: 0.00%\n",
            "rougeL: 0.00%\n",
            "rougeLsum: 0.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3'></a>\n",
        "## 3 - Perform Parameter Efficient Fine-Tuning (PEFT)\n",
        "\n",
        "Now, let's perform **Parameter Efficient Fine-Tuning (PEFT)** fine-tuning as opposed to \"full fine-tuning\" as you did above. PEFT is a form of instruction fine-tuning that is much more efficient than full fine-tuning - with comparable evaluation results as you will see soon.\n",
        "\n",
        "PEFT is a generic term that includes **Low-Rank Adaptation (LoRA)** and prompt tuning (which is NOT THE SAME as prompt engineering!). In most cases, when someone says PEFT, they typically mean LoRA. LoRA, at a very high level, allows the user to fine-tune their model using fewer compute resources (in some cases, a single GPU). After fine-tuning for a specific task, use case, or tenant with LoRA, the result is that the original LLM remains unchanged and a newly-trained “LoRA adapter” emerges. This LoRA adapter is much, much smaller than the original LLM - on the order of a single-digit % of the original LLM size (MBs vs GBs).  \n",
        "\n",
        "That said, at inference time, the LoRA adapter needs to be reunited and combined with its original LLM to serve the inference request.  The benefit, however, is that many LoRA adapters can re-use the original LLM which reduces overall memory requirements when serving multiple tasks and use cases."
      ],
      "metadata": {
        "id": "zqZloqaI1j2n"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "prZC_pHAvx6y"
      },
      "source": [
        "<a name='3.1'></a>\n",
        "### 3.1 - Setup the PEFT/LoRA model for Fine-Tuning\n",
        "\n",
        "You need to set up the PEFT/LoRA model for fine-tuning with a new layer/parameter adapter. Using PEFT/LoRA, you are freezing the underlying LLM and only training the adapter. Have a look at the LoRA configuration below. Note the rank (`r`) hyper-parameter, which defines the rank/dimension of the adapter to be trained."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from peft import LoraConfig, get_peft_model, TaskType\n",
        "\n",
        "lora_config = LoraConfig(\n",
        "    r=32, # Rank\n",
        "    lora_alpha=32,\n",
        "    target_modules=[\"q\", \"v\"],\n",
        "    lora_dropout=0.05,\n",
        "    bias=\"none\",\n",
        "    task_type=TaskType.SEQ_2_SEQ_LM # FLAN-T5\n",
        ")"
      ],
      "metadata": {
        "id": "MFiPPzRW1k1v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "-gkxQGXQvx60"
      },
      "source": [
        "Add LoRA adapter layers/parameters to the original LLM to be trained."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "peft_model = get_peft_model(model, lora_config)\n",
        "print(print_number_of_trainable_model_parameters(peft_model))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MWPD-kGq1oWi",
        "outputId": "90ec3742-0087-41b1-afbe-ff1c353122a2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable model parameters: 3538944\n",
            "all model parameters: 251116800\n",
            "percentage of trainable model parameters: 1.41%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3.2'></a>\n",
        "### 3.2 - Train PEFT Adapter\n",
        "\n",
        "Define training arguments and create `Trainer` instance."
      ],
      "metadata": {
        "id": "anPX2FL310-M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output_dir = f'./peft-dialogue-summary-training-{str(int(time.time()))}'\n",
        "\n",
        "peft_training_args = TrainingArguments(\n",
        "    output_dir=output_dir,\n",
        "    auto_find_batch_size=True,\n",
        "    learning_rate=1e-3, # Higher learning rate than full fine-tuning.\n",
        "    num_train_epochs=1,\n",
        "    logging_steps=1,\n",
        "    max_steps=1\n",
        ")\n",
        "\n",
        "peft_trainer = Trainer(\n",
        "    model=peft_model,\n",
        "    args=peft_training_args,\n",
        "    train_dataset=tokenized_datasets[\"train\"],\n",
        ")"
      ],
      "metadata": {
        "id": "TP2VH4dg11vC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XnX-N5zOvx62"
      },
      "source": [
        "Now everything is ready to train the PEFT adapter and save the model.\n",
        "\n",
        "<img src=\"data:image/svg+xml;base64,Cjxzdmcgd2lkdGg9IjgwMCIgaGVpZ2h0PSIxMjUiIHZpZXdCb3g9IjAgMCA4MDAgMTI1IiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogICAgPGRlZnM+CiAgICAgICAgPGxpbmVhckdyYWRpZW50IGlkPSJmYWRlR3JhZGllbnQiIHgxPSIwIiB4Mj0iMSI+CiAgICAgICAgICAgIDxzdG9wIG9mZnNldD0iMCUiIHN0b3AtY29sb3I9IiNGMEYwRjAiLz4KICAgICAgICAgICAgPHN0b3Agb2Zmc2V0PSIxMDAlIiBzdG9wLWNvbG9yPSIjRjBGMEYwIiBzdG9wLW9wYWNpdHk9IjAiLz4KICAgICAgICA8L2xpbmVhckdyYWRpZW50PgogICAgICAgIDxtYXNrIGlkPSJmYWRlTWFzayI+CiAgICAgICAgICAgIDxyZWN0IHg9IjAiIHk9IjAiIHdpZHRoPSI3NTAiIGhlaWdodD0iMTI1IiBmaWxsPSJ3aGl0ZSIvPgogICAgICAgICAgICA8cmVjdCB4PSI3NTAiIHk9IjAiIHdpZHRoPSI1MCIgaGVpZ2h0PSIxMjUiIGZpbGw9InVybCgjZmFkZUdyYWRpZW50KSIvPgogICAgICAgIDwvbWFzaz4KICAgIDwvZGVmcz4KICAgIDxwYXRoIGQ9Ik0zLDUwIEE1MCw1MCAwIDAgMSA1MywzIEw3OTcsMyBMNzk3LDk3IEw5Nyw5NyBMNTAsMTE1IEwzLDk3IFoiIGZpbGw9IiNGMEYwRjAiIHN0cm9rZT0iI0UwRTBFMCIgc3Ryb2tlLXdpZHRoPSIxIiBtYXNrPSJ1cmwoI2ZhZGVNYXNrKSIvPgogICAgPGNpcmNsZSBjeD0iNTAiIGN5PSI1MCIgcj0iMzAiIGZpbGw9IiM1N2M0ZjgiIHN0cm9rZT0iIzU3YzRmOCIgc3Ryb2tlLXdpZHRoPSIxIi8+CiAgICA8Y2lyY2xlIGN4PSI1MCIgY3k9IjUwIiByPSIyNSIgZmlsbD0iI0YwRjBGMCIvPgogICAgPGxpbmUgeDE9IjUwIiB5MT0iNTAiIHgyPSI1MCIgeTI9IjMwIiBzdHJva2U9IiM1N2M0ZjgiIHN0cm9rZS13aWR0aD0iMyIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgICA8bGluZSB4MT0iNTAiIHkxPSI1MCIgeDI9IjY1IiB5Mj0iNTAiIHN0cm9rZT0iIzU3YzRmOCIgc3Ryb2tlLXdpZHRoPSIzIiBzdHJva2UtbGluZWNhcD0icm91bmQiLz4KICAgIDx0ZXh0IHg9IjEwMCIgeT0iMzQiIGZvbnQtZmFtaWx5PSJBcmlhbCwgc2Fucy1zZXJpZiIgZm9udC1zaXplPSIxNCIgZmlsbD0iIzMzMzMzMyI+VGhlIG5leHQgY2VsbCBtYXkgdGFrZSBhIGZldyBtaW51dGVzIHRvIHJ1bi48L3RleHQ+Cjwvc3ZnPgo=\" alt=\"Time alert open medium\"/>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "peft_trainer.train()\n",
        "\n",
        "peft_model_path=\"./peft-dialogue-summary-checkpoint-local\"\n",
        "\n",
        "peft_trainer.model.save_pretrained(peft_model_path)\n",
        "tokenizer.save_pretrained(peft_model_path)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        },
        "id": "8vt_Uk2Z3T-r",
        "outputId": "694e2f33-a361-4c76-bb11-21b7cc3fe3a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1/1 00:00, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>51.028000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./peft-dialogue-summary-checkpoint-local/tokenizer_config.json',\n",
              " './peft-dialogue-summary-checkpoint-local/special_tokens_map.json',\n",
              " './peft-dialogue-summary-checkpoint-local/tokenizer.json')"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "gB-IejPrvx66"
      },
      "source": [
        "Prepare this model by adding an adapter to the original FLAN-T5 model. You are setting `is_trainable=False` because the plan is only to perform inference with this PEFT model. If you were preparing the model for further training, you would set `is_trainable=True`."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from peft import PeftModel, PeftConfig\n",
        "\n",
        "peft_model_base = AutoModelForSeq2SeqLM.from_pretrained(\"google/flan-t5-base\", torch_dtype=torch.bfloat16)\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"google/flan-t5-base\")\n",
        "\n",
        "peft_model = PeftModel.from_pretrained(peft_model_base,\n",
        "                                       peft_model_path,\n",
        "                                       torch_dtype=torch.bfloat16,\n",
        "                                       is_trainable=False)\n",
        "\n",
        "peft_model.to(device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-NzmrMDQ3ts2",
        "outputId": "49dba296-18e0-471b-e240-98675a103ded"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PeftModelForSeq2SeqLM(\n",
              "  (base_model): LoraModel(\n",
              "    (model): T5ForConditionalGeneration(\n",
              "      (shared): Embedding(32128, 768)\n",
              "      (encoder): T5Stack(\n",
              "        (embed_tokens): Embedding(32128, 768)\n",
              "        (block): ModuleList(\n",
              "          (0): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (relative_attention_bias): Embedding(32, 12)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (1): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (2): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (3): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (4): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (5): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (6): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (7): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (8): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (9): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (10): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (11): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "        (final_layer_norm): T5LayerNorm()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (decoder): T5Stack(\n",
              "        (embed_tokens): Embedding(32128, 768)\n",
              "        (block): ModuleList(\n",
              "          (0): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (relative_attention_bias): Embedding(32, 12)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (1): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (2): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (3): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (4): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (5): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (6): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (7): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (8): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (9): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (10): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "          (11): T5Block(\n",
              "            (layer): ModuleList(\n",
              "              (0): T5LayerSelfAttention(\n",
              "                (SelfAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (1): T5LayerCrossAttention(\n",
              "                (EncDecAttention): T5Attention(\n",
              "                  (q): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (k): Linear(in_features=768, out_features=768, bias=False)\n",
              "                  (v): Linear(\n",
              "                    in_features=768, out_features=768, bias=False\n",
              "                    (lora_dropout): ModuleDict(\n",
              "                      (default): Dropout(p=0.05, inplace=False)\n",
              "                    )\n",
              "                    (lora_A): ModuleDict(\n",
              "                      (default): Linear(in_features=768, out_features=32, bias=False)\n",
              "                    )\n",
              "                    (lora_B): ModuleDict(\n",
              "                      (default): Linear(in_features=32, out_features=768, bias=False)\n",
              "                    )\n",
              "                    (lora_embedding_A): ParameterDict()\n",
              "                    (lora_embedding_B): ParameterDict()\n",
              "                  )\n",
              "                  (o): Linear(in_features=768, out_features=768, bias=False)\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "              (2): T5LayerFF(\n",
              "                (DenseReluDense): T5DenseGatedActDense(\n",
              "                  (wi_0): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wi_1): Linear(in_features=768, out_features=2048, bias=False)\n",
              "                  (wo): Linear(in_features=2048, out_features=768, bias=False)\n",
              "                  (dropout): Dropout(p=0.1, inplace=False)\n",
              "                  (act): NewGELUActivation()\n",
              "                )\n",
              "                (layer_norm): T5LayerNorm()\n",
              "                (dropout): Dropout(p=0.1, inplace=False)\n",
              "              )\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "        (final_layer_norm): T5LayerNorm()\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "Kq8Zf5UOvx67"
      },
      "source": [
        "The number of trainable parameters will be `0` due to `is_trainable=False` setting:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(print_number_of_trainable_model_parameters(peft_model))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZQhk6S9q3-5F",
        "outputId": "f882cbf0-a6a8-441e-c07f-648a12731473"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "trainable model parameters: 0\n",
            "all model parameters: 251116800\n",
            "percentage of trainable model parameters: 0.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3.3'></a>\n",
        "### 3.3 - Evaluate the Model Qualitatively (Human Evaluation)\n",
        "\n",
        "Make inferences for the same example as in sections [1.3](#1.3) and [2.3](#2.3), with the original model, fully fine-tuned and PEFT model."
      ],
      "metadata": {
        "id": "8zyEje1_4CHM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "index = 200\n",
        "dialogue = dataset['test'][index]['text']\n",
        "baseline_human_summary = dataset['test'][index]['label_text']\n",
        "\n",
        "prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral.\n",
        "\n",
        "{text}\n",
        "\n",
        "Sentiment: \"\"\"\n",
        "\n",
        "input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "original_model_outputs = model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
        "original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
        "instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200, num_beams=1))\n",
        "peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "print(dash_line)\n",
        "print(f'BASELINE HUMAN SUMMARY:\\n{human_baseline_summary}')\n",
        "print(dash_line)\n",
        "print(f'ORIGINAL MODEL:\\n{original_model_text_output}')\n",
        "print(dash_line)\n",
        "print(f'INSTRUCT MODEL:\\n{instruct_model_text_output}')\n",
        "print(dash_line)\n",
        "print(f'PEFT MODEL: {peft_model_text_output}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5i4ppciq4AMh",
        "outputId": "5c5a3d8d-490c-48dc-dc7c-5524991320fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---------------------------------------------------------------------------------------------------\n",
            "BASELINE HUMAN SUMMARY:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "ORIGINAL MODEL:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "INSTRUCT MODEL:\n",
            "negative\n",
            "---------------------------------------------------------------------------------------------------\n",
            "PEFT MODEL: negative\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3.4'></a>\n",
        "### 3.4 - Evaluate the Model Quantitatively (with ROUGE Metric)\n",
        "Perform inferences for the sample of the test dataset (only 10 dialogues and summaries to save time)."
      ],
      "metadata": {
        "id": "h0xyamwGATSl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dialogues = dataset['test'][0:10]['text']\n",
        "human_baseline_summaries = dataset['test'][0:10]['label_text']\n",
        "\n",
        "original_model_summaries = []\n",
        "instruct_model_summaries = []\n",
        "peft_model_summaries = []\n",
        "\n",
        "for idx, dialogue in enumerate(dialogues):\n",
        "    prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral.\n",
        "\n",
        "{text}\n",
        "\n",
        "Sentiment: \"\"\"\n",
        "\n",
        "    input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "    human_baseline_text_output = human_baseline_summaries[idx]\n",
        "\n",
        "    original_model_outputs = model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
        "    original_model_text_output = tokenizer.decode(original_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "    instruct_model_outputs = instruct_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
        "    instruct_model_text_output = tokenizer.decode(instruct_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "    peft_model_outputs = peft_model.generate(input_ids=input_ids, generation_config=GenerationConfig(max_new_tokens=200))\n",
        "    peft_model_text_output = tokenizer.decode(peft_model_outputs[0], skip_special_tokens=True)\n",
        "\n",
        "    original_model_summaries.append(original_model_text_output)\n",
        "    instruct_model_summaries.append(instruct_model_text_output)\n",
        "    peft_model_summaries.append(peft_model_text_output)\n",
        "\n",
        "zipped_summaries = list(zip(human_baseline_summaries, original_model_summaries, instruct_model_summaries, peft_model_summaries))\n",
        "\n",
        "df = pd.DataFrame(zipped_summaries, columns = ['human_baseline_summaries', 'original_model_summaries', 'instruct_model_summaries', 'peft_model_summaries'])\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "DWJ_imFdAULA",
        "outputId": "8c1f2d82-5aee-40db-c277-f5637f3a53be"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  human_baseline_summaries original_model_summaries instruct_model_summaries  \\\n",
              "0                  neutral                 negative                 negative   \n",
              "1                 positive                 negative                 negative   \n",
              "2                 negative                 negative                 negative   \n",
              "3                 positive                 negative                 negative   \n",
              "4                 positive                 negative                 negative   \n",
              "5                 positive                 negative                 negative   \n",
              "6                 negative                 negative                 negative   \n",
              "7                 negative                 negative                 negative   \n",
              "8                  neutral                 negative                 negative   \n",
              "9                  neutral                 negative                 negative   \n",
              "\n",
              "  peft_model_summaries  \n",
              "0             negative  \n",
              "1             negative  \n",
              "2             negative  \n",
              "3             negative  \n",
              "4             negative  \n",
              "5             negative  \n",
              "6             negative  \n",
              "7             negative  \n",
              "8             negative  \n",
              "9             negative  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2f57b599-215c-4c32-96ef-019b672e02d8\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>human_baseline_summaries</th>\n",
              "      <th>original_model_summaries</th>\n",
              "      <th>instruct_model_summaries</th>\n",
              "      <th>peft_model_summaries</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>positive</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>neutral</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "      <td>negative</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2f57b599-215c-4c32-96ef-019b672e02d8')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2f57b599-215c-4c32-96ef-019b672e02d8 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2f57b599-215c-4c32-96ef-019b672e02d8');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6f8a384f-c626-46c7-84db-90bb05f9b924\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6f8a384f-c626-46c7-84db-90bb05f9b924')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6f8a384f-c626-46c7-84db-90bb05f9b924 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rouge = evaluate.load('rouge')\n",
        "\n",
        "original_model_results = rouge.compute(\n",
        "    predictions=original_model_summaries,\n",
        "    references=human_baseline_summaries[0:len(original_model_summaries)],\n",
        "    use_aggregator=True,\n",
        "    use_stemmer=True,\n",
        ")\n",
        "\n",
        "instruct_model_results = rouge.compute(\n",
        "    predictions=instruct_model_summaries,\n",
        "    references=human_baseline_summaries[0:len(instruct_model_summaries)],\n",
        "    use_aggregator=True,\n",
        "    use_stemmer=True,\n",
        ")\n",
        "\n",
        "peft_model_results = rouge.compute(\n",
        "    predictions=peft_model_summaries,\n",
        "    references=human_baseline_summaries[0:len(peft_model_summaries)],\n",
        "    use_aggregator=True,\n",
        "    use_stemmer=True,\n",
        ")\n",
        "\n",
        "print('ORIGINAL MODEL:')\n",
        "print(original_model_results)\n",
        "print('INSTRUCT MODEL:')\n",
        "print(instruct_model_results)\n",
        "print('PEFT MODEL:')\n",
        "print(peft_model_results)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YP6HUVNvAk3h",
        "outputId": "60a3fc4b-6b76-4932-d5e8-feeea1ab769a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL MODEL:\n",
            "{'rouge1': 0.3, 'rouge2': 0.0, 'rougeL': 0.3, 'rougeLsum': 0.3}\n",
            "INSTRUCT MODEL:\n",
            "{'rouge1': 0.3, 'rouge2': 0.0, 'rougeL': 0.3, 'rougeLsum': 0.3}\n",
            "PEFT MODEL:\n",
            "{'rouge1': 0.3, 'rouge2': 0.0, 'rougeL': 0.3, 'rougeLsum': 0.3}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7oLym4Xuvx7A"
      },
      "source": [
        "You already computed ROUGE score on the full dataset, after loading the results from the `data/dialogue-summary-training-results.csv` file. Load the values for the PEFT model now and check its performance compared to other models."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FeB43kG4vx7B"
      },
      "source": [
        "The results show less of an improvement over full fine-tuning, but the benefits of PEFT typically outweigh the slightly-lower performance metrics.\n",
        "\n",
        "Calculate the improvement of PEFT over the original model:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Absolute percentage improvement of PEFT MODEL over HUMAN BASELINE\")\n",
        "\n",
        "improvement = (np.array(list(peft_model_results.values())) - np.array(list(original_model_results.values())))\n",
        "for key, value in zip(peft_model_results.keys(), improvement):\n",
        "    print(f'{key}: {value*100:.2f}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tl9Wic5zAsd-",
        "outputId": "798eac9b-2e7e-4b19-ede9-064822705386"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absolute percentage improvement of PEFT MODEL over HUMAN BASELINE\n",
            "rouge1: 0.00%\n",
            "rouge2: 0.00%\n",
            "rougeL: 0.00%\n",
            "rougeLsum: 0.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H3vqHhMFvx7C"
      },
      "source": [
        "Now calculate the improvement of PEFT over a full fine-tuned model:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Absolute percentage improvement of PEFT MODEL over INSTRUCT MODEL\")\n",
        "\n",
        "improvement = (np.array(list(peft_model_results.values())) - np.array(list(instruct_model_results.values())))\n",
        "for key, value in zip(peft_model_results.keys(), improvement):\n",
        "    print(f'{key}: {value*100:.2f}%')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KoJrUW5mAt4i",
        "outputId": "19322038-73c7-48ca-ff69-d223eb87a598"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Absolute percentage improvement of PEFT MODEL over INSTRUCT MODEL\n",
            "rouge1: 0.00%\n",
            "rouge2: 0.00%\n",
            "rougeL: 0.00%\n",
            "rougeLsum: 0.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9WSoPDLqvx7D"
      },
      "source": [
        "Here you see a small percentage decrease in the ROUGE metrics vs. full fine-tuned. However, the training requires much less computing and memory resources (often just a single GPU)."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Lab 3: Fine-Tune FLAN-T5 with Reinforcement Learning (PPO) and PEFT to Generate Less-Toxic Summaries"
      ],
      "metadata": {
        "id": "dzxCGSJlA4Ro"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "36ef668a-9c51-489b-be47-a07a09ef2289"
      },
      "source": [
        "In this notebook, you will fine-tune a FLAN-T5 model to generate less toxic content with Meta AI's hate speech reward model. The reward model is a binary classifier that predicts either \"not hate\" or \"hate\" for the given text. You will use Proximal Policy Optimization (PPO) to fine-tune and reduce the model's toxicity."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import the necessary components."
      ],
      "metadata": {
        "id": "L4Z3_i6NKaJw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForSeq2SeqLM, GenerationConfig\n",
        "from datasets import load_dataset\n",
        "from peft import PeftModel, PeftConfig, LoraConfig, TaskType\n",
        "\n",
        "# trl: Transformer Reinforcement Learning library\n",
        "from trl import PPOTrainer, PPOConfig, AutoModelForSeq2SeqLMWithValueHead\n",
        "from trl import create_reference_model\n",
        "from trl.core import LengthSampler\n",
        "\n",
        "import torch\n",
        "import evaluate\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# tqdm library makes the loops show a smart progress meter.\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()"
      ],
      "metadata": {
        "id": "_XXSFJ9UAwd6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b76eea84-8e3a-4487-9692-613977e6c8e3"
      },
      "source": [
        "<a name='2'></a>\n",
        "## 2 - Load FLAN-T5 Model, Prepare Reward Model and Toxicity Evaluator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "4a5f97d4-ea5f-4072-b5d6-785d1d833ed4"
      },
      "source": [
        "<a name='2.1'></a>\n",
        "### 2.1 - Load Data and FLAN-T5 Model Fine-Tuned with Summarization Instruction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "668d30d6-6f81-4e52-a81a-3057163ddb0e"
      },
      "source": [
        "The next step will be to preprocess the dataset. You will take only a part of it, then filter the dialogues of a particular length (just to make those examples long enough and, at the same time, easy to read). Then wrap each dialogue with the instruction and tokenize the prompts. Save the token ids in the field `input_ids` and decoded version of the prompts in the field `query`.\n",
        "\n",
        "You could do that all step by step in the cell below, but it is a good habit to organize that all in a function `build_dataset`:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def build_dataset(model_name,\n",
        "                  dataset_name,\n",
        "                  input_min_text_length,\n",
        "                  input_max_text_length):\n",
        "\n",
        "    \"\"\"\n",
        "    Preprocess the dataset and split it into train and test parts.\n",
        "\n",
        "    Parameters:\n",
        "    - model_name (str): Tokenizer model name.\n",
        "    - dataset_name (str): Name of the dataset to load.\n",
        "    - input_min_text_length (int): Minimum length of the dialogues.\n",
        "    - input_max_text_length (int): Maximum length of the dialogues.\n",
        "\n",
        "    Returns:\n",
        "    - dataset_splits (datasets.dataset_dict.DatasetDict): Preprocessed dataset containing train and test parts.\n",
        "    \"\"\"\n",
        "\n",
        "    # load dataset (only \"train\" part will be enough for this lab).\n",
        "    dataset = load_dataset(dataset_name, split=\"train\")\n",
        "\n",
        "    # Filter the dialogues of length between input_min_text_length and input_max_text_length characters.\n",
        "    dataset = dataset.filter(lambda x: len(x[\"text\"]) > input_min_text_length and len(x[\"text\"]) <= input_max_text_length, batched=False)\n",
        "\n",
        "    # Prepare tokenizer. Setting device_map=\"auto\" allows to switch between GPU and CPU automatically.\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_name, device_map=\"auto\")\n",
        "\n",
        "    def tokenize(sample):\n",
        "\n",
        "        # Wrap each dialogue with the instruction.\n",
        "        prompt = f\"\"\"\n",
        "Provide sentiment of the following text as either positive, negative or neutral.\n",
        "\n",
        "{sample[\"text\"]}\n",
        "\n",
        "Sentiment:\n",
        "\"\"\"\n",
        "        sample[\"input_ids\"] = tokenizer.encode(prompt)\n",
        "\n",
        "        # This must be called \"query\", which is a requirement of our PPO library.\n",
        "        sample[\"query\"] = tokenizer.decode(sample[\"input_ids\"])\n",
        "        return sample\n",
        "\n",
        "    # Tokenize each dialogue.\n",
        "    dataset = dataset.map(tokenize, batched=False)\n",
        "    dataset.set_format(type=\"torch\")\n",
        "\n",
        "    # Split the dataset into train and test parts.\n",
        "    dataset_splits = dataset.train_test_split(test_size=0.2, shuffle=False, seed=42)\n",
        "\n",
        "    return dataset_splits\n",
        "\n",
        "dataset = build_dataset(model_name=model_name,\n",
        "                        dataset_name=huggingface_dataset_name,\n",
        "                        input_min_text_length=200,\n",
        "                        input_max_text_length=1000)\n",
        "\n",
        "print(dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228,
          "referenced_widgets": [
            "5375818c120d48e0a595372254c38a5f",
            "a967b78a161d4382b0815d570dd1b14a",
            "cde4d86e76ad4b88863d8c1cd3ab06a4",
            "06d481e527504722803437906aa334aa",
            "22fc417159d74e37835241c8cc3031e3",
            "953686abe5374d81a5444f2d49a9dfa7",
            "f2d245b178e2401a89fef0759277b1f7",
            "b9d11bc9015a4bdca2ad25d55dbfe4c3",
            "c67af42a7eb740ee95b402693b746f7a",
            "30fe5bf925fc49a4b18f3e5323678a06",
            "f5bb05b04b8c4d9f84a64511129440b9"
          ]
        },
        "id": "t5s6cUodDvY8",
        "outputId": "b8bf4ca9-5a8a-41af-fb6b-59a26f75c989"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset json (/root/.cache/huggingface/datasets/mteb___json/mteb--tweet_sentiment_extraction-19a608df77d581ad/0.0.0/fe5dd6ea2639a6df622901539cb550cf8797e5a6b2dd7af1cf934bed8e233e6e)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Filter:   0%|          | 0/27481 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5375818c120d48e0a595372254c38a5f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "DatasetDict({\n",
            "    train: Dataset({\n",
            "        features: ['id', 'text', 'label', 'label_text'],\n",
            "        num_rows: 0\n",
            "    })\n",
            "    test: Dataset({\n",
            "        features: ['id', 'text', 'label', 'label_text'],\n",
            "        num_rows: 0\n",
            "    })\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "7d03155e-649b-45bb-a5a0-94edd682c069"
      },
      "source": [
        "In the previous lab, you fine-tuned the PEFT model with summarization instructions. The training in the notebook was done on a subset of data. Then you downloaded the checkpoint of the fully trained PEFT model from S3.\n",
        "\n",
        "Let's load the same model checkpoint here:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def print_number_of_trainable_model_parameters(model):\n",
        "    trainable_model_params = 0\n",
        "    all_model_params = 0\n",
        "    for _, param in model.named_parameters():\n",
        "        all_model_params += param.numel()\n",
        "        if param.requires_grad:\n",
        "            trainable_model_params += param.numel()\n",
        "    return f\"\\ntrainable model parameters: {trainable_model_params}\\nall model parameters: {all_model_params}\\npercentage of trainable model parameters: {100 * trainable_model_params / all_model_params:.2f}%\""
      ],
      "metadata": {
        "id": "YfD0YK4cEICQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "4n7wF0XbSbwO"
      },
      "source": [
        "In the previous lab, you fine-tuned the PEFT model with summarization instructions. The training in the notebook was done on a subset of data. Then you downloaded the checkpoint of the fully trained PEFT model from S3.\n",
        "\n",
        "Let's load the same model checkpoint here:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lora_config = LoraConfig(\n",
        "    r=32, # Rank\n",
        "    lora_alpha=32,\n",
        "    target_modules=[\"q\", \"v\"],\n",
        "    lora_dropout=0.05,\n",
        "    bias=\"none\",\n",
        "    task_type=TaskType.SEQ_2_SEQ_LM # FLAN-T5\n",
        ")\n",
        "\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_name,\n",
        "                                              torch_dtype=torch.bfloat16)\n",
        "\n",
        "peft_model = PeftModel.from_pretrained(model,\n",
        "                                       peft_model_path,\n",
        "                                       lora_config=lora_config,\n",
        "                                       torch_dtype=torch.bfloat16,\n",
        "                                       device_map=\"auto\",\n",
        "                                       is_trainable=True)\n",
        "\n",
        "\n",
        "print(f'PEFT model parameters to be updated:\\n{print_number_of_trainable_model_parameters(peft_model)}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GcvrgM1dEJwI",
        "outputId": "b97cc65c-7960-4e7a-e7f9-75d80f971ebd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PEFT model parameters to be updated:\n",
            "\n",
            "trainable model parameters: 3538944\n",
            "all model parameters: 251116800\n",
            "percentage of trainable model parameters: 1.41%\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "NMuveeLgSdpI"
      },
      "source": [
        "In the previous lab, you fine-tuned the PEFT model with summarization instructions. The training in the notebook was done on a subset of data. Then you downloaded the checkpoint of the fully trained PEFT model from S3.\n",
        "\n",
        "Let's load the same model checkpoint here:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ppo_model = AutoModelForSeq2SeqLMWithValueHead.from_pretrained(peft_model,\n",
        "                                                               torch_dtype=torch.bfloat16,\n",
        "                                                               is_trainable=True)\n",
        "\n",
        "print(f'PPO model parameters to be updated (ValueHead + 769 params):\\n{print_number_of_trainable_model_parameters(ppo_model)}\\n')\n",
        "print(ppo_model.v_head)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k06Ifw05FJ4J",
        "outputId": "1409d5a0-052c-4a48-ff99-81f27d65ef57"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "PPO model parameters to be updated (ValueHead + 769 params):\n",
            "\n",
            "trainable model parameters: 3539713\n",
            "all model parameters: 251117569\n",
            "percentage of trainable model parameters: 1.41%\n",
            "\n",
            "ValueHead(\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            "  (summary): Linear(in_features=768, out_features=1, bias=True)\n",
            "  (flatten): Flatten(start_dim=1, end_dim=-1)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2913ef05-737e-4cdf-9bac-467ee6cf9f76"
      },
      "source": [
        "During PPO, only a few parameters will be updated. Specifically, the parameters of the `ValueHead`. More information about this class of models can be found in the [documentation](https://huggingface.co/docs/trl/main/en/models#trl.create_reference_model). The number of trainable parameters can be computed as $(n+1)*m$, where $n$ is the number of input units (here $n=768$) and $m$ is the number of output units (you have $m=1$). The $+1$ term in the equation takes into account the bias term."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "76c7e2df-0c75-4bd0-bf8d-1f1545ef864e"
      },
      "source": [
        "Now create a frozen copy of the PPO which will not be fine-tuned - a reference model. The reference model will represent the LLM before detoxification. None of the parameters of the reference model will be updated during PPO training. This is on purpose."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ref_model = create_reference_model(ppo_model)\n",
        "\n",
        "print(f'Reference model parameters to be updated:\\n{print_number_of_trainable_model_parameters(ref_model)}\\n')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U7CMW9dMFNb8",
        "outputId": "8b1ef030-d6f6-4fcf-cb1e-ff0dbcb3d705"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reference model parameters to be updated:\n",
            "\n",
            "trainable model parameters: 0\n",
            "all model parameters: 251117569\n",
            "percentage of trainable model parameters: 0.00%\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "e3a14848-e83d-4fdd-bc68-eb770c5951d7"
      },
      "source": [
        "Everything is set. It is time to prepare the reward model!"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='2.2'></a>\n",
        "### 2.2 - Prepare Reward Model\n",
        "\n",
        "**Reinforcement Learning (RL)** is one type of machine learning where agents take actions in an environment aimed at maximizing their cumulative rewards. The agent's behavior is defined by the **policy**. And the goal of reinforcement learning is for the agent to learn an optimal, or nearly-optimal, policy that maximizes the **reward function**.\n",
        "\n",
        "In the [previous section](#2.1) the original policy is based on the instruct PEFT model - this is the LLM before detoxification. Then you could ask human labelers to give feedback on the outputs' toxicity. However, it can be expensive to use them for the entire fine-tuning process. A practical way to avoid that is to use a reward model encouraging the agent to detoxify the dialogue summaries. The intuitive approach would be to do some form of sentiment analysis across two classes (`nothate` and `hate`) and give a higher reward if there is higher a chance of getting class `nothate` as an output.\n",
        "\n",
        "For example, we can mention that having human labelers for the entire finetuning process can be expensive. A practical way to avoid that is to use a reward model.\n",
        "\n",
        "use feedback generated by a model\n",
        "\n",
        "You will use [Meta AI's RoBERTa-based hate speech model](https://huggingface.co/facebook/roberta-hate-speech-dynabench-r4-target) for the reward model. This model will output **logits** and then predict probabilities across two classes: `nothate` and `hate`. The logits of the output `nothate` will be taken as a positive reward. Then, the model will be fine-tuned with PPO using those reward values.\n",
        "\n",
        "Create the instance of the required model class for the RoBERTa model. You also need to load a tokenizer to test the model. Notice that the model label `0` will correspond to the class `nothate` and label `1` to the class `hate`."
      ],
      "metadata": {
        "id": "h_LIPcyJFQiA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "toxicity_model_name = \"facebook/roberta-hate-speech-dynabench-r4-target\"\n",
        "toxicity_tokenizer = AutoTokenizer.from_pretrained(toxicity_model_name, device_map=\"auto\")\n",
        "toxicity_model = AutoModelForSequenceClassification.from_pretrained(toxicity_model_name, device_map=\"auto\")\n",
        "print(toxicity_model.config.id2label)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226,
          "referenced_widgets": [
            "215e4ff9678948648ef8d30362315020",
            "f6e2a356684a4649ac14f03bc3c5263c",
            "71b3a8504810408c9524e6f78af3bb8d",
            "9096e9620a324cec8bd3acfdc4ec14c2",
            "5f958662a19648e79b0a41ee5d005483",
            "eb0569828b0248ebbfa7d87fe464c274",
            "ac653d35316341879f1dff4be7601d4c",
            "690dfd6fd5df4041989f63b6df9f68b4",
            "d2341aca405c41f4a420c4bbe8fa0420",
            "72142749796149c1bfe96446a590f592",
            "d49aeac061d946a1968711636c38847d",
            "d067de736c604e529df81241a3061e7f",
            "b7df1d0926fc43469c4431e131b11074",
            "dcbfd87d665f420a8505c44ea05a17f2",
            "bf3d9e4b09b540038c586f8b17eaedf3",
            "797891b7e649429e8c0c50bc9e0571ba",
            "cfece662a0454a81988ee61971646cb6",
            "bdd655e2b6f24470ab7d25baeecd18bf",
            "4527f9a9122c459cab5e6d6b43d8960c",
            "5417e9962e0c439f88f41264cd18972a",
            "f68b6020dbae4ea9b944aa8c0ff54729",
            "016c65063e6640069210fab18c529e2a",
            "681d1db7a53945139586bd3bf6ec858f",
            "5a4a0a6521c844a0b536430884aa4bd9",
            "03a08c52cc804e769bcdb83be7f21668",
            "554bd6fe1bad4405bde8dc1acc6bbab4",
            "7926f30128394b26bd833d3193b999d3",
            "26251873aea648abb0a0f65c3043d04a",
            "fc9fde2733e04346a6d2df2a343407b8",
            "d312eae439e848119f2447563af97fa1",
            "55c2f7f4289040f4bbb3ac8c6dbffe37",
            "a25d598baa6a4708940c308fea786c11",
            "5e2fb42ef2394ae888e3f392dafd9c14",
            "53efb35d0aa644e49bcccc721095b253",
            "3d6774fc272f4cd994999b09124817a0",
            "2d2084fe21224e3fa9c5892e3824929c",
            "92676ee648904f929fd913ae123a4cec",
            "bee79abe1a7847cdb71258417cc0f729",
            "9f203076c09f4243aeb3fdabcdcb2b09",
            "e75e4467ca464898a1f6c7bf65d02a76",
            "f76613090a0543e6b42ee02956771248",
            "d2b63578ce204f5189eae5f87fa93f28",
            "b4c1167611f84f3583a90d963a178a2d",
            "dcb8e682b79241a4932c5486d8d2b825",
            "6d7053f780b24c2d9bc27a6015a7bdfb",
            "a2371df166474311b6af7a36df59ef08",
            "c3eb7f089fec4dbfb28b712c114f5ac0",
            "48d0f640414f417f9617227f1195e14f",
            "ef32d82d387a403c8b646c3243796374",
            "3680bda4106b404b999777927a7d1cae",
            "0dbc66caa907436c9d2635a44235446f",
            "f82464171012489886db18c721b0a80c",
            "10bb8d045d9f47c19c35d7f3b2dc65d6",
            "8b9478f7e8fe470d84f4c889c8aa57e1",
            "d829703081444e2fb27ba654f5d723e7",
            "74862d731eb94de084b62658ecad8e72",
            "5880d5ac9c884dd291cf43f5da74b02a",
            "1a349f9a05ab4765ba9cf6d90d5dd1d8",
            "83d99a3c1b794a93a8a210d327a2f1a3",
            "07cbe58b06ce439f916c8efedb389204",
            "67895bec89cd4e588b9499543df87b50",
            "afeac9c8a73c4db5aba3f1509aab6af1",
            "f4557123618e4639baf18b805cfa9682",
            "c4985daf6059440eb95d2af90cca3a88",
            "2610ee92b9204b09ae548cf7e34c62f8",
            "c61a627edba04f8fbe60ca41d1314803"
          ]
        },
        "id": "oHChiB-_FRhl",
        "outputId": "93ce7ab0-2ac6-4e5e-bd4d-0d53b5c76976"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.11k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "215e4ff9678948648ef8d30362315020"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d067de736c604e529df81241a3061e7f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "681d1db7a53945139586bd3bf6ec858f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/239 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "53efb35d0aa644e49bcccc721095b253"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/816 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6d7053f780b24c2d9bc27a6015a7bdfb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/499M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "74862d731eb94de084b62658ecad8e72"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{0: 'nothate', 1: 'hate'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "79d68799-a6e8-42d7-8d61-002e47210c18"
      },
      "source": [
        "Take some non-toxic text, tokenize it, and pass it to the model. Print the output logits, probabilities, and the corresponding reward that will be used for fine-tuning."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "non_toxic_text = \"I really hate cookies.\"\n",
        "\n",
        "toxicity_input_ids = toxicity_tokenizer(non_toxic_text, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "logits = toxicity_model(input_ids=toxicity_input_ids).logits\n",
        "print(f'logits [not hate, hate]: {logits.tolist()[0]}')\n",
        "\n",
        "# Print the probabilities for [not hate, hate]\n",
        "probabilities = logits.softmax(dim=-1).tolist()[0]\n",
        "print(f'probabilities [not hate, hate]: {probabilities}')\n",
        "\n",
        "# get the logits for \"not hate\" - this is the reward!\n",
        "not_hate_index = 0\n",
        "nothate_reward = (logits[:, not_hate_index]).tolist()\n",
        "print(f'reward (high): {nothate_reward}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "StjY6reWFW7T",
        "outputId": "abd62b31-e4cb-4b7b-bafa-24210a451ffa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "logits [not hate, hate]: [4.685029983520508, -4.2312469482421875]\n",
            "probabilities [not hate, hate]: [0.9998657703399658, 0.00013416889123618603]\n",
            "reward (high): [4.685029983520508]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "63f729c5-98c3-4745-96e8-3484670215db"
      },
      "source": [
        "Let's show a toxic comment.  This will have a low reward because it is more toxic."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "toxic_text = \"I think bananas are horrible and dreadful.\"\n",
        "\n",
        "toxicity_input_ids = toxicity_tokenizer(toxic_text, return_tensors=\"pt\").input_ids.to(device)\n",
        "\n",
        "logits = toxicity_model(toxicity_input_ids).logits\n",
        "print(f'logits [not hate, hate]: {logits.tolist()[0]}')\n",
        "\n",
        "# Print the probabilities for [not hate, hate]\n",
        "probabilities = logits.softmax(dim=-1).tolist()[0]\n",
        "print(f'probabilities [not hate, hate]: {probabilities}')\n",
        "\n",
        "# Get the logits for \"not hate\" - this is the reward!\n",
        "nothate_reward = (logits[:, not_hate_index]).tolist()\n",
        "print(f'reward (low): {nothate_reward}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DxvTPOVfFmhi",
        "outputId": "c81b67f9-ce53-48d5-b35b-80f4688460b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "logits [not hate, hate]: [4.600322723388672, -4.2066216468811035]\n",
            "probabilities [not hate, hate]: [0.9998502731323242, 0.00014966761227697134]\n",
            "reward (low): [4.600322723388672]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='2.3'></a>\n",
        "### 2.3 - Evaluate Toxicity\n",
        "\n",
        "To evaluate the model before and after fine-tuning/detoxification you need to set up the [toxicity evaluation metric](https://huggingface.co/spaces/evaluate-measurement/toxicity). The **toxicity score** is a decimal value between 0 and 1 where 1 is the highest toxicity."
      ],
      "metadata": {
        "id": "4nMBGU_cK63b"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "840fbc47-c5c2-469a-b5f2-6407e8f0bfde"
      },
      "source": [
        "Try to calculate toxicity for the same sentences as in section [2.2](#2.2). It's no surprise that the toxicity scores are the probabilities of `hate` class returned directly from the reward model."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_toxicity(model,\n",
        "                      toxicity_evaluator,\n",
        "                      tokenizer,\n",
        "                      dataset,\n",
        "                      num_samples):\n",
        "\n",
        "    \"\"\"\n",
        "    Preprocess the dataset and split it into train and test parts.\n",
        "\n",
        "    Parameters:\n",
        "    - model (trl model): Model to be evaluated.\n",
        "    - toxicity_evaluator (evaluate_modules toxicity metrics): Toxicity evaluator.\n",
        "    - tokenizer (transformers tokenizer): Tokenizer to be used.\n",
        "    - dataset (dataset): Input dataset for the evaluation.\n",
        "    - num_samples (int): Maximum number of samples for the evaluation.\n",
        "\n",
        "    Returns:\n",
        "    tuple: A tuple containing two numpy.float64 values:\n",
        "    - mean (numpy.float64): Mean of the samples toxicity.\n",
        "    - std (numpy.float64): Standard deviation of the samples toxicity.\n",
        "    \"\"\"\n",
        "\n",
        "    max_new_tokens=100\n",
        "\n",
        "    toxicities = []\n",
        "    input_texts = []\n",
        "    for i, sample in tqdm(enumerate(dataset)):\n",
        "        input_text = sample[\"query\"]\n",
        "\n",
        "        if i > num_samples:\n",
        "            break\n",
        "\n",
        "        input_ids = tokenizer(input_text, return_tensors=\"pt\", padding=True).input_ids\n",
        "\n",
        "        generation_config = GenerationConfig(max_new_tokens=max_new_tokens,\n",
        "                                             tok_k=0.0,\n",
        "                                             top_p=1.0,\n",
        "                                             do_sample=True)\n",
        "\n",
        "        response_token_ids = model.generate(input_ids=input_ids,\n",
        "                                            generation_config=generation_config)\n",
        "\n",
        "        generated_text = tokenizer.decode(response_token_ids[0], skip_special_tokens=True)\n",
        "\n",
        "        toxicity_score = toxicity_evaluator.compute(predictions=[(input_text + \" \" + generated_text)])\n",
        "\n",
        "        toxicities.extend(toxicity_score[\"toxicity\"])\n",
        "\n",
        "    # Compute mean & std using np.\n",
        "    mean = np.mean(toxicities)\n",
        "    std = np.std(toxicities)\n",
        "\n",
        "    return mean, std"
      ],
      "metadata": {
        "id": "OM4HRNJ_K4Gr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "7d3e835b-14b9-4646-b1c9-c975ef3ea944"
      },
      "source": [
        "This evaluator can be used to compute the toxicity of the dialogues prepared in section [2.1](#2.1). You will need to pass the test dataset (`dataset[\"test\"]`), the same tokenizer which was used in that section, the frozen PEFT model prepared in section [2.2](#2.2), and the toxicity evaluator. It is convenient to wrap the required steps in the function `evaluate_toxicity`."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<a name='3'></a>\n",
        "## 3 - Perform Fine-Tuning to Detoxify the Summaries\n",
        "Optimize a RL policy against the reward model using Proximal Policy Optimization (PPO)."
      ],
      "metadata": {
        "id": "8pTl18NQLXBw"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5516e318-8fce-4ca7-bf19-b7baf5255480"
      },
      "source": [
        "<a name='3.1'></a>\n",
        "### 3.1 - Initialize `PPOTrainer`\n",
        "\n",
        "For the `PPOTrainer` initialization, you will need a collator. Here it will be a function transforming the dictionaries in a particular way. You can define and test it:"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def collator(data):\n",
        "    return dict((key, [d[key] for d in data]) for key in data[0])\n",
        "\n",
        "test_data = [{\"key1\": \"value1\", \"key2\": \"value2\", \"key3\": \"value3\"}]\n",
        "print(f'Collator input: {test_data}')\n",
        "print(f'Collator output: {collator(test_data)}')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nYdBSkVaLYLP",
        "outputId": "f85d3848-b9e7-4f4c-cb22-21483ab0853f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collator input: [{'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}]\n",
            "Collator output: {'key1': ['value1'], 'key2': ['value2'], 'key3': ['value3']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Navigating the Difficulties in the Process\n",
        "\n",
        "## Optimizing Hyperparameters\n",
        "\n",
        "### Exploring Hyperparameter Combinations\n",
        "- **Utilize techniques** such as grid search or random search to test various hyperparameter settings (e.g., learning rate, batch size).\n",
        "\n",
        "### Leveraging Automated Optimization Tools\n",
        "- **Employ tools** like Hyperopt or Bayesian Optimization for a more streamlined hyperparameter tuning process.\n",
        "\n",
        "## Balancing Model Capacity in PEFT\n",
        "\n",
        "### Defining Model Capacity\n",
        "- **This is the model's ability** to recognize intricate patterns in data. In PEFT, it's crucial to find the right balance to prevent both underfitting (insufficient capacity) and overfitting (excessive capacity).\n",
        "\n",
        "### Fine-Tuning Strategy\n",
        "- **Modify the degree of fine-tuning** in PEFT, including choosing which layers to adjust and to what extent. Sometimes, only fine-tuning the top layers or a subset of parameters can be adequate.\n",
        "\n",
        "## Strategies to Counter Overfitting\n",
        "\n",
        "### Implementing Regularization Methods\n",
        "- **Apply techniques** such as dropout, weight decay, or early stopping to combat overfitting during training.\n",
        "\n",
        "### Data Partitioning Approach\n",
        "- **Ensure a proper distribution** of data across training, validation, and test sets to accurately assess the model's ability to generalize."
      ],
      "metadata": {
        "id": "Q6w4HnyXNsO7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MIT License\n",
        "\n",
        "Copyright (c) 2023\n",
        "\n",
        "Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n",
        "\n",
        "The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n",
        "\n",
        "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE."
      ],
      "metadata": {
        "id": "2siZceUpNv7l"
      }
    }
  ]
}