nmt_full_version.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "machine_translation_v2_yt.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPTAHO0cuGe01ORH+qm3qmI",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d547c94dd3a64665bba324c697febb00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f42bb32b70cd48129cd47f103d4e7833",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b968d0ec5ec84cda9932ef2e5d810f94",
              "IPY_MODEL_16b6d1c4359448628d03f2c3ba26db34"
            ]
          }
        },
        "f42bb32b70cd48129cd47f103d4e7833": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b968d0ec5ec84cda9932ef2e5d810f94": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e687f593436f42ed8ed07b45babb6cd2",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_327d12730f91427c9346e4c34cddc97a"
          }
        },
        "16b6d1c4359448628d03f2c3ba26db34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8feff991e3654109a593b68c09c71989",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:44&lt;00:00,  3.29it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f2c6721e0db45c0a14106f34724ef81"
          }
        },
        "e687f593436f42ed8ed07b45babb6cd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "327d12730f91427c9346e4c34cddc97a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8feff991e3654109a593b68c09c71989": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f2c6721e0db45c0a14106f34724ef81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dbfb67ca7cd0466c976488cca4a36c6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7046edd83446491aaab2256e6bcc52e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0aed11e47cec49c3ac6fe4c63000769e",
              "IPY_MODEL_eba82bf9861342c49b78b1e61408aecd"
            ]
          }
        },
        "7046edd83446491aaab2256e6bcc52e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0aed11e47cec49c3ac6fe4c63000769e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69ae01a816004cbebe52c0f8d6f98f25",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1a720adf34b34713b9ac59d24d1404c2"
          }
        },
        "eba82bf9861342c49b78b1e61408aecd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b2cd47dd3eb54d19be042d5dd5f6de25",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:39&lt;00:00,  3.33it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_382b6251119d4fb881c8ae0538efcd10"
          }
        },
        "69ae01a816004cbebe52c0f8d6f98f25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1a720adf34b34713b9ac59d24d1404c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2cd47dd3eb54d19be042d5dd5f6de25": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "382b6251119d4fb881c8ae0538efcd10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c5f2cbab1044d6fa7e63e0c19a3db3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b9456d09ce5d46c6b17019845ed8ee11",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9cdbf499f0564e4695c7cb1c02abd8ff",
              "IPY_MODEL_16873be391fe4ae186022a9a81775d4d"
            ]
          }
        },
        "b9456d09ce5d46c6b17019845ed8ee11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9cdbf499f0564e4695c7cb1c02abd8ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_83e308daead14376a22df02643df80fe",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6f7f7266d6a472a959068b9ae8d40fe"
          }
        },
        "16873be391fe4ae186022a9a81775d4d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_3854068540044f3e99bc6378a83ffc26",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:39&lt;00:00,  3.33it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f642e2a22b84bfeb8fd9067e84b4db8"
          }
        },
        "83e308daead14376a22df02643df80fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6f7f7266d6a472a959068b9ae8d40fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3854068540044f3e99bc6378a83ffc26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f642e2a22b84bfeb8fd9067e84b4db8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "03325133b7554a76840d4d2367bcd0c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9655332d70af4c0fa96fdea7d3dad056",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6e04c5ab370046018bffa827e93c5173",
              "IPY_MODEL_d619d55e2cd14d8d8c2fa135c3b8ea6a"
            ]
          }
        },
        "9655332d70af4c0fa96fdea7d3dad056": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6e04c5ab370046018bffa827e93c5173": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_fd4e775d53c84798aa610ac283d7b629",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39bf3356ce2e4ec28d52954a7e1eaae6"
          }
        },
        "d619d55e2cd14d8d8c2fa135c3b8ea6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d83db5b082cc4cf496693a5256395468",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:38&lt;00:00,  3.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e6773ea09854295b135401ea5db7c41"
          }
        },
        "fd4e775d53c84798aa610ac283d7b629": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39bf3356ce2e4ec28d52954a7e1eaae6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d83db5b082cc4cf496693a5256395468": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e6773ea09854295b135401ea5db7c41": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9122956a67ee4e26a21f7d2ac3ce4357": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fad9ca4eb1064a43a6e0a6f06fd418f7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0aa829059d8a4fd7a698513304f434bb",
              "IPY_MODEL_8dbf73941f5e4e7d9df959239a093742"
            ]
          }
        },
        "fad9ca4eb1064a43a6e0a6f06fd418f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0aa829059d8a4fd7a698513304f434bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_44d97dcd66794416b178510faaba22a8",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_16114ff04cf84746a880f846bb57f24f"
          }
        },
        "8dbf73941f5e4e7d9df959239a093742": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_be5b064142a64a2686e2e469c5e647ab",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:40&lt;00:00,  3.32it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2a5cb70b7e0c46bc9b5ae563319f4467"
          }
        },
        "44d97dcd66794416b178510faaba22a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "16114ff04cf84746a880f846bb57f24f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "be5b064142a64a2686e2e469c5e647ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2a5cb70b7e0c46bc9b5ae563319f4467": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "10542fa0178b4ab1aa166ece33c9d92f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0fe64c14a4ca46a9b83de6c7cf1b1feb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b2e727825c9a477fb5ee8c4ee281963d",
              "IPY_MODEL_364a164f8e114f2e8e9d0c287050b33e"
            ]
          }
        },
        "0fe64c14a4ca46a9b83de6c7cf1b1feb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b2e727825c9a477fb5ee8c4ee281963d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_62b02344033d485397b881cea48b3bbd",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1131,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_53edd8b36f134b4a9aa62984866f3893"
          }
        },
        "364a164f8e114f2e8e9d0c287050b33e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2d999834e4e4488aae3b033b7d099323",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1131/1131 [05:42&lt;00:00,  3.30it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d52c1e0ff627430985aac092e57836ea"
          }
        },
        "62b02344033d485397b881cea48b3bbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "53edd8b36f134b4a9aa62984866f3893": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2d999834e4e4488aae3b033b7d099323": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d52c1e0ff627430985aac092e57836ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "912d78120eae4d768cae2c5d6ecb25dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_01ef8dacfaec47369c59e43d28e0129b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3c2357cfe1534f019146a1c08848532e",
              "IPY_MODEL_c69c5625b54e4923bd509668534583a4"
            ]
          }
        },
        "01ef8dacfaec47369c59e43d28e0129b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3c2357cfe1534f019146a1c08848532e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dfed8ec945b0431c9d81e2ad40dd48ad",
            "_dom_classes": [],
            "description": " 82%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 1131,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 927,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80b7604fe4d44700bf37d892993712cb"
          }
        },
        "c69c5625b54e4923bd509668534583a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_63ff944b5fb04840b9a44ce4c965eeb0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 927/1131 [04:25&lt;00:58,  3.49it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a2da175f276d4d819cfd9bf8114187d0"
          }
        },
        "dfed8ec945b0431c9d81e2ad40dd48ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80b7604fe4d44700bf37d892993712cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63ff944b5fb04840b9a44ce4c965eeb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a2da175f276d4d819cfd9bf8114187d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ejmejm/multilingual-nmt-mt5/blob/main/nmt_full_version.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XAXQfZ1sWNwV"
      },
      "source": [
        "## Part 1\n",
        "- Overview\n",
        "- Demo\n",
        "- Model overview\n",
        "- Preloaded model & tokenizer\n",
        "- Quick test\n",
        "\n",
        "## Part 2\n",
        "- Update model\n",
        "- Load dataset\n",
        "- Data formatting loader\n",
        "\n",
        "## Part 3\n",
        "- Training\n",
        "- Testing\n",
        "- Recreate demo"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f3m7l1YS7XXF"
      },
      "source": [
        "# Download Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2a609pb6dKqA",
        "outputId": "892a259a-de28-4c54-e183-551e73e075cd"
      },
      "source": [
        "!pip install transformers sentencepiece datasets"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.5.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.95)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.2)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.44)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.0.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: pyarrow>=0.17.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.7/dist-packages (from datasets) (0.9.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5m4W5h7cmDn"
      },
      "source": [
        "from datasets import load_dataset\n",
        "from google.colab import drive\n",
        "from IPython.display import display\n",
        "from IPython.html import widgets\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import torch\n",
        "from torch import optim\n",
        "from torch.nn import functional as F\n",
        "from transformers import AdamW, AutoModelForSeq2SeqLM, AutoTokenizer\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from tqdm import tqdm_notebook\n",
        "\n",
        "sns.set()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y8IM3iH4SmzI",
        "outputId": "91cbb293-6949-4b8a-dda7-48fde1132196"
      },
      "source": [
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmkmzYNOQ9xC"
      },
      "source": [
        "model_repo = 'google/mt5-base'\n",
        "model_path = '/content/gdrive/My Drive/mt5_translation.pt'\n",
        "max_seq_len = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fdx0IvJW7dDz"
      },
      "source": [
        "# Load Tokenizer & Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAXKisA8FuAR"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_repo)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "En-NjwOrUgJN"
      },
      "source": [
        "# Model description: https://huggingface.co/google/mt5-base\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(model_repo)\n",
        "model = model.cuda()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hprdz6wlMy8f"
      },
      "source": [
        "# Overview and Quick Test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QsO9nJEyMAPq",
        "outputId": "e01dd7e3-4fd7-4b07-926c-efbda08628aa"
      },
      "source": [
        "token_ids = tokenizer.encode(\n",
        "    '<jp> This will be translated to Japanese! (hopefully)',\n",
        "    return_tensors='pt').cuda()\n",
        "print(token_ids)\n",
        "\n",
        "model_out = model.generate(token_ids)\n",
        "print(model_out)\n",
        "\n",
        "output_text = tokenizer.convert_tokens_to_string(\n",
        "    tokenizer.convert_ids_to_tokens(model_out[0]))\n",
        "print(output_text)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[  1042,   3889,    669,   1494,    898,    390,  37194,    285,    288,\n",
            "          30865,    309,    274, 116024,  11994,    271,      1]],\n",
            "       device='cuda:0')\n",
            "tensor([[     0, 250099,    259,    264,    259, 228700,      1]],\n",
            "       device='cuda:0')\n",
            "<pad> <extra_id_0> - issuu</s>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qRXrD3IkODsw"
      },
      "source": [
        "# Steps\n",
        "1. Load the pretrained model and tokenizer\n",
        "2. Load dataset\n",
        "3. Transform dataset into input (entails a minor model change)\n",
        "4. Train/finetune the model on our dataset\n",
        "5. Test the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wHNfsuI57iGg"
      },
      "source": [
        "# Test Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a8JmwhL1d6g7",
        "outputId": "0ded8556-1e89-4110-eaf7-085dba4519c7"
      },
      "source": [
        "example_input_str = '<jp> This is just a test nbuig.'\n",
        "# example_input_str = 'これは普通のテスト'\n",
        "input_ids = tokenizer.encode(example_input_str, return_tensors='pt')\n",
        "print('Input IDs:', input_ids)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(input_ids[0])\n",
        "print('Tokens:', tokens)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input IDs: tensor([[1042, 3889,  669, 1494,  339, 1627,  259,  262, 2978,  259,  272, 1982,\n",
            "         1315,  260,    1]])\n",
            "Tokens: ['▁<', 'jp', '>', '▁This', '▁is', '▁just', '▁', 'a', '▁test', '▁', 'n', 'bu', 'ig', '.', '</s>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mwd4V8Krjga5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0910b905-5130-4590-89c5-ff5de0505955"
      },
      "source": [
        "sorted(tokenizer.vocab.items(), key=lambda x: x[1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('<pad>', 0),\n",
              " ('</s>', 1),\n",
              " ('<unk>', 2),\n",
              " ('<0x00>', 3),\n",
              " ('<0x01>', 4),\n",
              " ('<0x02>', 5),\n",
              " ('<0x03>', 6),\n",
              " ('<0x04>', 7),\n",
              " ('<0x05>', 8),\n",
              " ('<0x06>', 9),\n",
              " ('<0x07>', 10),\n",
              " ('<0x08>', 11),\n",
              " ('<0x09>', 12),\n",
              " ('<0x0A>', 13),\n",
              " ('<0x0B>', 14),\n",
              " ('<0x0C>', 15),\n",
              " ('<0x0D>', 16),\n",
              " ('<0x0E>', 17),\n",
              " ('<0x0F>', 18),\n",
              " ('<0x10>', 19),\n",
              " ('<0x11>', 20),\n",
              " ('<0x12>', 21),\n",
              " ('<0x13>', 22),\n",
              " ('<0x14>', 23),\n",
              " ('<0x15>', 24),\n",
              " ('<0x16>', 25),\n",
              " ('<0x17>', 26),\n",
              " ('<0x18>', 27),\n",
              " ('<0x19>', 28),\n",
              " ('<0x1A>', 29),\n",
              " ('<0x1B>', 30),\n",
              " ('<0x1C>', 31),\n",
              " ('<0x1D>', 32),\n",
              " ('<0x1E>', 33),\n",
              " ('<0x1F>', 34),\n",
              " ('<0x20>', 35),\n",
              " ('<0x21>', 36),\n",
              " ('<0x22>', 37),\n",
              " ('<0x23>', 38),\n",
              " ('<0x24>', 39),\n",
              " ('<0x25>', 40),\n",
              " ('<0x26>', 41),\n",
              " ('<0x27>', 42),\n",
              " ('<0x28>', 43),\n",
              " ('<0x29>', 44),\n",
              " ('<0x2A>', 45),\n",
              " ('<0x2B>', 46),\n",
              " ('<0x2C>', 47),\n",
              " ('<0x2D>', 48),\n",
              " ('<0x2E>', 49),\n",
              " ('<0x2F>', 50),\n",
              " ('<0x30>', 51),\n",
              " ('<0x31>', 52),\n",
              " ('<0x32>', 53),\n",
              " ('<0x33>', 54),\n",
              " ('<0x34>', 55),\n",
              " ('<0x35>', 56),\n",
              " ('<0x36>', 57),\n",
              " ('<0x37>', 58),\n",
              " ('<0x38>', 59),\n",
              " ('<0x39>', 60),\n",
              " ('<0x3A>', 61),\n",
              " ('<0x3B>', 62),\n",
              " ('<0x3C>', 63),\n",
              " ('<0x3D>', 64),\n",
              " ('<0x3E>', 65),\n",
              " ('<0x3F>', 66),\n",
              " ('<0x40>', 67),\n",
              " ('<0x41>', 68),\n",
              " ('<0x42>', 69),\n",
              " ('<0x43>', 70),\n",
              " ('<0x44>', 71),\n",
              " ('<0x45>', 72),\n",
              " ('<0x46>', 73),\n",
              " ('<0x47>', 74),\n",
              " ('<0x48>', 75),\n",
              " ('<0x49>', 76),\n",
              " ('<0x4A>', 77),\n",
              " ('<0x4B>', 78),\n",
              " ('<0x4C>', 79),\n",
              " ('<0x4D>', 80),\n",
              " ('<0x4E>', 81),\n",
              " ('<0x4F>', 82),\n",
              " ('<0x50>', 83),\n",
              " ('<0x51>', 84),\n",
              " ('<0x52>', 85),\n",
              " ('<0x53>', 86),\n",
              " ('<0x54>', 87),\n",
              " ('<0x55>', 88),\n",
              " ('<0x56>', 89),\n",
              " ('<0x57>', 90),\n",
              " ('<0x58>', 91),\n",
              " ('<0x59>', 92),\n",
              " ('<0x5A>', 93),\n",
              " ('<0x5B>', 94),\n",
              " ('<0x5C>', 95),\n",
              " ('<0x5D>', 96),\n",
              " ('<0x5E>', 97),\n",
              " ('<0x5F>', 98),\n",
              " ('<0x60>', 99),\n",
              " ('<0x61>', 100),\n",
              " ('<0x62>', 101),\n",
              " ('<0x63>', 102),\n",
              " ('<0x64>', 103),\n",
              " ('<0x65>', 104),\n",
              " ('<0x66>', 105),\n",
              " ('<0x67>', 106),\n",
              " ('<0x68>', 107),\n",
              " ('<0x69>', 108),\n",
              " ('<0x6A>', 109),\n",
              " ('<0x6B>', 110),\n",
              " ('<0x6C>', 111),\n",
              " ('<0x6D>', 112),\n",
              " ('<0x6E>', 113),\n",
              " ('<0x6F>', 114),\n",
              " ('<0x70>', 115),\n",
              " ('<0x71>', 116),\n",
              " ('<0x72>', 117),\n",
              " ('<0x73>', 118),\n",
              " ('<0x74>', 119),\n",
              " ('<0x75>', 120),\n",
              " ('<0x76>', 121),\n",
              " ('<0x77>', 122),\n",
              " ('<0x78>', 123),\n",
              " ('<0x79>', 124),\n",
              " ('<0x7A>', 125),\n",
              " ('<0x7B>', 126),\n",
              " ('<0x7C>', 127),\n",
              " ('<0x7D>', 128),\n",
              " ('<0x7E>', 129),\n",
              " ('<0x7F>', 130),\n",
              " ('<0x80>', 131),\n",
              " ('<0x81>', 132),\n",
              " ('<0x82>', 133),\n",
              " ('<0x83>', 134),\n",
              " ('<0x84>', 135),\n",
              " ('<0x85>', 136),\n",
              " ('<0x86>', 137),\n",
              " ('<0x87>', 138),\n",
              " ('<0x88>', 139),\n",
              " ('<0x89>', 140),\n",
              " ('<0x8A>', 141),\n",
              " ('<0x8B>', 142),\n",
              " ('<0x8C>', 143),\n",
              " ('<0x8D>', 144),\n",
              " ('<0x8E>', 145),\n",
              " ('<0x8F>', 146),\n",
              " ('<0x90>', 147),\n",
              " ('<0x91>', 148),\n",
              " ('<0x92>', 149),\n",
              " ('<0x93>', 150),\n",
              " ('<0x94>', 151),\n",
              " ('<0x95>', 152),\n",
              " ('<0x96>', 153),\n",
              " ('<0x97>', 154),\n",
              " ('<0x98>', 155),\n",
              " ('<0x99>', 156),\n",
              " ('<0x9A>', 157),\n",
              " ('<0x9B>', 158),\n",
              " ('<0x9C>', 159),\n",
              " ('<0x9D>', 160),\n",
              " ('<0x9E>', 161),\n",
              " ('<0x9F>', 162),\n",
              " ('<0xA0>', 163),\n",
              " ('<0xA1>', 164),\n",
              " ('<0xA2>', 165),\n",
              " ('<0xA3>', 166),\n",
              " ('<0xA4>', 167),\n",
              " ('<0xA5>', 168),\n",
              " ('<0xA6>', 169),\n",
              " ('<0xA7>', 170),\n",
              " ('<0xA8>', 171),\n",
              " ('<0xA9>', 172),\n",
              " ('<0xAA>', 173),\n",
              " ('<0xAB>', 174),\n",
              " ('<0xAC>', 175),\n",
              " ('<0xAD>', 176),\n",
              " ('<0xAE>', 177),\n",
              " ('<0xAF>', 178),\n",
              " ('<0xB0>', 179),\n",
              " ('<0xB1>', 180),\n",
              " ('<0xB2>', 181),\n",
              " ('<0xB3>', 182),\n",
              " ('<0xB4>', 183),\n",
              " ('<0xB5>', 184),\n",
              " ('<0xB6>', 185),\n",
              " ('<0xB7>', 186),\n",
              " ('<0xB8>', 187),\n",
              " ('<0xB9>', 188),\n",
              " ('<0xBA>', 189),\n",
              " ('<0xBB>', 190),\n",
              " ('<0xBC>', 191),\n",
              " ('<0xBD>', 192),\n",
              " ('<0xBE>', 193),\n",
              " ('<0xBF>', 194),\n",
              " ('<0xC0>', 195),\n",
              " ('<0xC1>', 196),\n",
              " ('<0xC2>', 197),\n",
              " ('<0xC3>', 198),\n",
              " ('<0xC4>', 199),\n",
              " ('<0xC5>', 200),\n",
              " ('<0xC6>', 201),\n",
              " ('<0xC7>', 202),\n",
              " ('<0xC8>', 203),\n",
              " ('<0xC9>', 204),\n",
              " ('<0xCA>', 205),\n",
              " ('<0xCB>', 206),\n",
              " ('<0xCC>', 207),\n",
              " ('<0xCD>', 208),\n",
              " ('<0xCE>', 209),\n",
              " ('<0xCF>', 210),\n",
              " ('<0xD0>', 211),\n",
              " ('<0xD1>', 212),\n",
              " ('<0xD2>', 213),\n",
              " ('<0xD3>', 214),\n",
              " ('<0xD4>', 215),\n",
              " ('<0xD5>', 216),\n",
              " ('<0xD6>', 217),\n",
              " ('<0xD7>', 218),\n",
              " ('<0xD8>', 219),\n",
              " ('<0xD9>', 220),\n",
              " ('<0xDA>', 221),\n",
              " ('<0xDB>', 222),\n",
              " ('<0xDC>', 223),\n",
              " ('<0xDD>', 224),\n",
              " ('<0xDE>', 225),\n",
              " ('<0xDF>', 226),\n",
              " ('<0xE0>', 227),\n",
              " ('<0xE1>', 228),\n",
              " ('<0xE2>', 229),\n",
              " ('<0xE3>', 230),\n",
              " ('<0xE4>', 231),\n",
              " ('<0xE5>', 232),\n",
              " ('<0xE6>', 233),\n",
              " ('<0xE7>', 234),\n",
              " ('<0xE8>', 235),\n",
              " ('<0xE9>', 236),\n",
              " ('<0xEA>', 237),\n",
              " ('<0xEB>', 238),\n",
              " ('<0xEC>', 239),\n",
              " ('<0xED>', 240),\n",
              " ('<0xEE>', 241),\n",
              " ('<0xEF>', 242),\n",
              " ('<0xF0>', 243),\n",
              " ('<0xF1>', 244),\n",
              " ('<0xF2>', 245),\n",
              " ('<0xF3>', 246),\n",
              " ('<0xF4>', 247),\n",
              " ('<0xF5>', 248),\n",
              " ('<0xF6>', 249),\n",
              " ('<0xF7>', 250),\n",
              " ('<0xF8>', 251),\n",
              " ('<0xF9>', 252),\n",
              " ('<0xFA>', 253),\n",
              " ('<0xFB>', 254),\n",
              " ('<0xFC>', 255),\n",
              " ('<0xFD>', 256),\n",
              " ('<0xFE>', 257),\n",
              " ('<0xFF>', 258),\n",
              " ('▁', 259),\n",
              " ('.', 260),\n",
              " (',', 261),\n",
              " ('a', 262),\n",
              " ('s', 263),\n",
              " ('-', 264),\n",
              " ('e', 265),\n",
              " ('i', 266),\n",
              " (':', 267),\n",
              " ('o', 268),\n",
              " ('▁de', 269),\n",
              " ('t', 270),\n",
              " (')', 271),\n",
              " ('n', 272),\n",
              " ('u', 273),\n",
              " ('▁(', 274),\n",
              " ('/', 275),\n",
              " ('y', 276),\n",
              " (\"'\", 277),\n",
              " ('en', 278),\n",
              " ('и', 279),\n",
              " ('l', 280),\n",
              " ('▁in', 281),\n",
              " ('m', 282),\n",
              " ('▁la', 283),\n",
              " ('com', 284),\n",
              " ('d', 285),\n",
              " ('r', 286),\n",
              " ('▁the', 287),\n",
              " ('▁to', 288),\n",
              " ('▁en', 289),\n",
              " ('_', 290),\n",
              " ('?', 291),\n",
              " ('、', 292),\n",
              " ('’', 293),\n",
              " ('▁na', 294),\n",
              " ('er', 295),\n",
              " (';', 296),\n",
              " ('c', 297),\n",
              " ('▁A', 298),\n",
              " ('es', 299),\n",
              " ('▁v', 300),\n",
              " ('▁di', 301),\n",
              " ('...', 302),\n",
              " ('▁se', 303),\n",
              " ('▁of', 304),\n",
              " ('▁and', 305),\n",
              " ('。', 306),\n",
              " ('▁|', 307),\n",
              " ('а', 308),\n",
              " ('!', 309),\n",
              " ('▁на', 310),\n",
              " ('\"', 311),\n",
              " ('(', 312),\n",
              " ('▁\"', 313),\n",
              " ('k', 314),\n",
              " ('▁в', 315),\n",
              " ('b', 316),\n",
              " ('▁c', 317),\n",
              " ('g', 318),\n",
              " ('▁que', 319),\n",
              " ('▁S', 320),\n",
              " ('an', 321),\n",
              " ('▁–', 322),\n",
              " ('▁www', 323),\n",
              " ('е', 324),\n",
              " ('p', 325),\n",
              " ('▁m', 326),\n",
              " ('▁sa', 327),\n",
              " ('3', 328),\n",
              " ('x', 329),\n",
              " ('▁b', 330),\n",
              " ('▁d', 331),\n",
              " ('▁for', 332),\n",
              " ('▁1', 333),\n",
              " ('h', 334),\n",
              " ('▁un', 335),\n",
              " ('▁I', 336),\n",
              " ('os', 337),\n",
              " ('2', 338),\n",
              " ('▁is', 339),\n",
              " ('▁le', 340),\n",
              " ('▁و', 341),\n",
              " ('▁do', 342),\n",
              " ('،', 343),\n",
              " ('▁at', 344),\n",
              " ('ed', 345),\n",
              " ('te', 346),\n",
              " ('ing', 347),\n",
              " ('in', 348),\n",
              " ('=', 349),\n",
              " ('▁da', 350),\n",
              " ('▁on', 351),\n",
              " ('▁M', 352),\n",
              " ('1', 353),\n",
              " ('у', 354),\n",
              " ('▁đ', 355),\n",
              " ('▁2', 356),\n",
              " ('A', 357),\n",
              " ('as', 358),\n",
              " ('▁“', 359),\n",
              " ('z', 360),\n",
              " ('é', 361),\n",
              " ('▁el', 362),\n",
              " ('▁P', 363),\n",
              " ('▁B', 364),\n",
              " ('”', 365),\n",
              " ('▁T', 366),\n",
              " ('f', 367),\n",
              " ('de', 368),\n",
              " ('à', 369),\n",
              " ('ng', 370),\n",
              " ('▁C', 371),\n",
              " ('ar', 372),\n",
              " ('▁og', 373),\n",
              " ('▁за', 374),\n",
              " ('▁no', 375),\n",
              " ('ه', 376),\n",
              " ('na', 377),\n",
              " ('।', 378),\n",
              " ('v', 379),\n",
              " ('re', 380),\n",
              " ('▁3', 381),\n",
              " ('▁h', 382),\n",
              " ('▁et', 383),\n",
              " ('▁je', 384),\n",
              " ('j', 385),\n",
              " ('▁il', 386),\n",
              " ('▁#', 387),\n",
              " ('▁с', 388),\n",
              " ('і', 389),\n",
              " ('▁be', 390),\n",
              " ('://', 391),\n",
              " ('▁2018', 392),\n",
              " ('▁per', 393),\n",
              " ('▁th', 394),\n",
              " ('▁si', 395),\n",
              " ('я', 396),\n",
              " ('▁z', 397),\n",
              " ('▁die', 398),\n",
              " ('S', 399),\n",
              " ('▁te', 400),\n",
              " ('▁не', 401),\n",
              " ('▁ال', 402),\n",
              " ('D', 403),\n",
              " ('▁«', 404),\n",
              " ('ne', 405),\n",
              " ('ی', 406),\n",
              " ('da', 407),\n",
              " ('▁k', 408),\n",
              " ('|', 409),\n",
              " ('4', 410),\n",
              " ('о', 411),\n",
              " ('▁K', 412),\n",
              " ('▁du', 413),\n",
              " ('▁w', 414),\n",
              " ('▁E', 415),\n",
              " ('▁me', 416),\n",
              " ('is', 417),\n",
              " ('▁are', 418),\n",
              " ('▁4', 419),\n",
              " ('í', 420),\n",
              " ('▁p', 421),\n",
              " ('ta', 422),\n",
              " ('の', 423),\n",
              " ('C', 424),\n",
              " ('▁по', 425),\n",
              " ('▁del', 426),\n",
              " ('▁ka', 427),\n",
              " ('5', 428),\n",
              " ('et', 429),\n",
              " ('▁5', 430),\n",
              " ('▁D', 431),\n",
              " ('▁ja', 432),\n",
              " ('ы', 433),\n",
              " ('▁V', 434),\n",
              " ('▁para', 435),\n",
              " ('»', 436),\n",
              " ('\",\"', 437),\n",
              " ('us', 438),\n",
              " (']', 439),\n",
              " ('▁al', 440),\n",
              " ('▁N', 441),\n",
              " ('▁der', 442),\n",
              " ('▁O', 443),\n",
              " ('on', 444),\n",
              " ('ة', 445),\n",
              " ('▁да', 446),\n",
              " ('▁H', 447),\n",
              " ('▁ne', 448),\n",
              " ('8', 449),\n",
              " ('▁con', 450),\n",
              " ('6', 451),\n",
              " ('B', 452),\n",
              " ('▁er', 453),\n",
              " ('ul', 454),\n",
              " ('▁by', 455),\n",
              " ('▁у', 456),\n",
              " ('▁yang', 457),\n",
              " ('▁L', 458),\n",
              " ('▁De', 459),\n",
              " ('0', 460),\n",
              " ('▁an', 461),\n",
              " ('ja', 462),\n",
              " ('\\xad', 463),\n",
              " ('▁van', 464),\n",
              " ('▁ה', 465),\n",
              " ('▁za', 466),\n",
              " ('】【', 467),\n",
              " ('le', 468),\n",
              " ('▁dan', 469),\n",
              " ('em', 470),\n",
              " ('á', 471),\n",
              " ('▁und', 472),\n",
              " ('al', 473),\n",
              " ('è', 474),\n",
              " ('▁10', 475),\n",
              " ('to', 476),\n",
              " ('ي', 477),\n",
              " ('E', 478),\n",
              " ('ka', 479),\n",
              " ('▁...', 480),\n",
              " ('w', 481),\n",
              " ('▁på', 482),\n",
              " (').', 483),\n",
              " ('ly', 484),\n",
              " ('▁po', 485),\n",
              " ('▁The', 486),\n",
              " ('7', 487),\n",
              " ('\":\"', 488),\n",
              " ('▁G', 489),\n",
              " ('T', 490),\n",
              " ('▁[', 491),\n",
              " ('la', 492),\n",
              " ('的', 493),\n",
              " ('li', 494),\n",
              " ('9', 495),\n",
              " ('▁ma', 496),\n",
              " ('▁0', 497),\n",
              " ('▁des', 498),\n",
              " ('▁med', 499),\n",
              " ('▁til', 500),\n",
              " ('▁La', 501),\n",
              " ('kan', 502),\n",
              " ('it', 503),\n",
              " ('▁ki', 504),\n",
              " ('no', 505),\n",
              " ('),', 506),\n",
              " ('м', 507),\n",
              " ('َ', 508),\n",
              " ('▁در', 509),\n",
              " ('▁so', 510),\n",
              " ('M', 511),\n",
              " ('▁som', 512),\n",
              " ('▁ke', 513),\n",
              " ('▁with', 514),\n",
              " ('▁F', 515),\n",
              " ('ni', 516),\n",
              " ('▁su', 517),\n",
              " ('▁και', 518),\n",
              " ('▁por', 519),\n",
              " ('▁les', 520),\n",
              " ('▁you', 521),\n",
              " ('si', 522),\n",
              " ('at', 523),\n",
              " ('ti', 524),\n",
              " ('id', 525),\n",
              " ('▁av', 526),\n",
              " ('▁as', 527),\n",
              " ('▁ya', 528),\n",
              " ('▁ve', 529),\n",
              " ('▁den', 530),\n",
              " ('▁R', 531),\n",
              " ('▁ב', 532),\n",
              " ('▁that', 533),\n",
              " ('▁tr', 534),\n",
              " ('は', 535),\n",
              " ('が', 536),\n",
              " ('do', 537),\n",
              " ('N', 538),\n",
              " ('ia', 539),\n",
              " ('\\\\', 540),\n",
              " ('ce', 541),\n",
              " ('▁om', 542),\n",
              " ('й', 543),\n",
              " ('▁се', 544),\n",
              " ('F', 545),\n",
              " ('&', 546),\n",
              " ('L', 547),\n",
              " ('▁م', 548),\n",
              " ('▁&', 549),\n",
              " ('▁د', 550),\n",
              " ('▁det', 551),\n",
              " ('▁от', 552),\n",
              " ('ó', 553),\n",
              " ('▁به', 554),\n",
              " ('▁pa', 555),\n",
              " ('▁من', 556),\n",
              " ('K', 557),\n",
              " ('на', 558),\n",
              " ('P', 559),\n",
              " ('▁ha', 560),\n",
              " ('V', 561),\n",
              " ('▁ch', 562),\n",
              " ('▁In', 563),\n",
              " ('▁W', 564),\n",
              " ('▁„', 565),\n",
              " ('I', 566),\n",
              " ('▁var', 567),\n",
              " ('▁ni', 568),\n",
              " ('se', 569),\n",
              " ('▁6', 570),\n",
              " ('ra', 571),\n",
              " ('ل', 572),\n",
              " ('▁una', 573),\n",
              " ('を', 574),\n",
              " ('▁في', 575),\n",
              " ('▁ta', 576),\n",
              " ('▁http', 577),\n",
              " ('COM', 578),\n",
              " ('am', 579),\n",
              " ('ה', 580),\n",
              " ('▁U', 581),\n",
              " ('R', 582),\n",
              " ('▁з', 583),\n",
              " ('▁re', 584),\n",
              " ('▁op', 585),\n",
              " ('ن', 586),\n",
              " ('т', 587),\n",
              " ('▁har', 588),\n",
              " ('ο', 589),\n",
              " ('H', 590),\n",
              " ('“', 591),\n",
              " ('ek', 592),\n",
              " ('▁ag', 593),\n",
              " ('▁ng', 594),\n",
              " ('▁los', 595),\n",
              " ('{', 596),\n",
              " ('▁och', 597),\n",
              " ('▁2017', 598),\n",
              " ('▁WWW', 599),\n",
              " ('に', 600),\n",
              " ('▁ku', 601),\n",
              " ('ir', 602),\n",
              " ('▁pe', 603),\n",
              " ('un', 604),\n",
              " ('х', 605),\n",
              " ('um', 606),\n",
              " ('▁2019', 607),\n",
              " ('je', 608),\n",
              " ('▁it', 609),\n",
              " ('▁до', 610),\n",
              " ('을', 611),\n",
              " ('ʻ', 612),\n",
              " ('www', 613),\n",
              " ('▁ب', 614),\n",
              " ('▁li', 615),\n",
              " ('но', 616),\n",
              " ('▁7', 617),\n",
              " ('▁»', 618),\n",
              " ('▁ir', 619),\n",
              " ('▁kan', 620),\n",
              " ('G', 621),\n",
              " ('▁het', 622),\n",
              " ('▁ho', 623),\n",
              " ('▁par', 624),\n",
              " ('▁vi', 625),\n",
              " ('・', 626),\n",
              " ('で', 627),\n",
              " ('▁20', 628),\n",
              " ('▁të', 629),\n",
              " ('▁8', 630),\n",
              " ('▁or', 631),\n",
              " ('ا', 632),\n",
              " ('م', 633),\n",
              " ('ie', 634),\n",
              " ('▁В', 635),\n",
              " ('ت', 636),\n",
              " ('ом', 637),\n",
              " ('W', 638),\n",
              " ('▁was', 639),\n",
              " ('την', 640),\n",
              " ('▁के', 641),\n",
              " ('▁En', 642),\n",
              " ('▁af', 643),\n",
              " ('▁12', 644),\n",
              " ('me', 645),\n",
              " ('O', 646),\n",
              " ('nya', 647),\n",
              " ('ma', 648),\n",
              " ('의', 649),\n",
              " ('ki', 650),\n",
              " ('▁cu', 651),\n",
              " ('μ', 652),\n",
              " ('▁No', 653),\n",
              " ('▁2016', 654),\n",
              " ('▁es', 655),\n",
              " ('▁een', 656),\n",
              " ('ки', 657),\n",
              " ('▁mi', 658),\n",
              " ('Ð', 659),\n",
              " ('10', 660),\n",
              " ('▁—', 661),\n",
              " ('ku', 662),\n",
              " ('\":', 663),\n",
              " ('▁J', 664),\n",
              " ('px', 665),\n",
              " ('일', 666),\n",
              " ('▁ל', 667),\n",
              " ('ни', 668),\n",
              " ('>', 669),\n",
              " ('▁15', 670),\n",
              " ('▁‘', 671),\n",
              " ('▁ver', 672),\n",
              " ('▁um', 673),\n",
              " ('▁man', 674),\n",
              " ('▁ko', 675),\n",
              " ('+', 676),\n",
              " ('▁nh', 677),\n",
              " ('η', 678),\n",
              " ('ка', 679),\n",
              " ('ny', 680),\n",
              " ('α', 681),\n",
              " ('▁od', 682),\n",
              " ('▁wa', 683),\n",
              " ('▁ge', 684),\n",
              " ('ов', 685),\n",
              " ('н', 686),\n",
              " ('ten', 687),\n",
              " ('▁С', 688),\n",
              " ('▁מ', 689),\n",
              " ('▁ph', 690),\n",
              " ('▁>', 691),\n",
              " ('▁men', 692),\n",
              " ('▁ber', 693),\n",
              " ('▁του', 694),\n",
              " ('▁از', 695),\n",
              " ('il', 696),\n",
              " ('ch', 697),\n",
              " ('▁bir', 698),\n",
              " ('▁το', 699),\n",
              " ('▁να', 700),\n",
              " ('el', 701),\n",
              " ('▁from', 702),\n",
              " ('▁nu', 703),\n",
              " ('ko', 704),\n",
              " ('st', 705),\n",
              " ('ë', 706),\n",
              " ('▁lo', 707),\n",
              " ('ủ', 708),\n",
              " ('▁az', 709),\n",
              " ('▁dem', 710),\n",
              " ('mi', 711),\n",
              " ('▁va', 712),\n",
              " ('▁att', 713),\n",
              " ('▁this', 714),\n",
              " ('ur', 715),\n",
              " ('▁nie', 716),\n",
              " ('#', 717),\n",
              " ('▁gi', 718),\n",
              " ('▁tu', 719),\n",
              " ('di', 720),\n",
              " ('å', 721),\n",
              " ('ات', 722),\n",
              " ('or', 723),\n",
              " ('▁em', 724),\n",
              " ('と', 725),\n",
              " ('ת', 726),\n",
              " ('▁Na', 727),\n",
              " ('▁am', 728),\n",
              " ('▁из', 729),\n",
              " ('▁11', 730),\n",
              " ('▁pro', 731),\n",
              " ('▁în', 732),\n",
              " ('▁30', 733),\n",
              " ('▁che', 734),\n",
              " ('для', 735),\n",
              " ('▁Z', 736),\n",
              " ('ru', 737),\n",
              " ('▁can', 738),\n",
              " ('ya', 739),\n",
              " ('▁ang', 740),\n",
              " ('ai', 741),\n",
              " ('▁f', 742),\n",
              " ('ga', 743),\n",
              " ('▁+', 744),\n",
              " ('za', 745),\n",
              " ('▁Se', 746),\n",
              " ('이', 747),\n",
              " ('ю', 748),\n",
              " ('▁mit', 749),\n",
              " ('ca', 750),\n",
              " ('▁all', 751),\n",
              " ('▁של', 752),\n",
              " ('ke', 753),\n",
              " ('\",', 754),\n",
              " ('°', 755),\n",
              " ('▁tak', 756),\n",
              " ('ने', 757),\n",
              " ('▁bu', 758),\n",
              " ('▁bo', 759),\n",
              " ('▁zu', 760),\n",
              " ('ą', 761),\n",
              " ('ή', 762),\n",
              " ('▁pour', 763),\n",
              " ('▁Le', 764),\n",
              " ('[', 765),\n",
              " ('▁ت', 766),\n",
              " ('▁ter', 767),\n",
              " ('▁با', 768),\n",
              " ('ci', 769),\n",
              " ('▁és', 770),\n",
              " ('co', 771),\n",
              " ('▁your', 772),\n",
              " ('om', 773),\n",
              " ('▁9', 774),\n",
              " ('▁کے', 775),\n",
              " ('▁not', 776),\n",
              " ('их', 777),\n",
              " ('▁к', 778),\n",
              " ('▁din', 779),\n",
              " ('im', 780),\n",
              " ('q', 781),\n",
              " ('ă', 782),\n",
              " ('▁have', 783),\n",
              " ('▁mai', 784),\n",
              " ('▁{', 785),\n",
              " ('▁pre', 786),\n",
              " ('▁we', 787),\n",
              " ('▁Re', 788),\n",
              " ('▁El', 789),\n",
              " ('▁he', 790),\n",
              " ('ς', 791),\n",
              " ('▁•', 792),\n",
              " ('và', 793),\n",
              " ('Y', 794),\n",
              " ('▁von', 795),\n",
              " ('▁là', 796),\n",
              " ('ې', 797),\n",
              " ('▁ar', 798),\n",
              " ('▁16', 799),\n",
              " ('▁las', 800),\n",
              " ('ú', 801),\n",
              " ('app', 802),\n",
              " ('▁کی', 803),\n",
              " ('▁au', 804),\n",
              " ('▁при', 805),\n",
              " ('U', 806),\n",
              " ('th', 807),\n",
              " ('▁}', 808),\n",
              " ('▁2014', 809),\n",
              " ('▁ba', 810),\n",
              " ('be', 811),\n",
              " ('▁18', 812),\n",
              " ('X', 813),\n",
              " ('▁2015', 814),\n",
              " ('▁2013', 815),\n",
              " ('▁(1)', 816),\n",
              " ('ой', 817),\n",
              " ('▁14', 818),\n",
              " ('▁qu', 819),\n",
              " ('ِ', 820),\n",
              " ('ha', 821),\n",
              " ('▁می', 822),\n",
              " ('man', 823),\n",
              " ('▁met', 824),\n",
              " ('are', 825),\n",
              " ('▁nga', 826),\n",
              " ('▁das', 827),\n",
              " ('▁της', 828),\n",
              " ('‘', 829),\n",
              " ('▁है', 830),\n",
              " ('ية', 831),\n",
              " ('то', 832),\n",
              " ('ь', 833),\n",
              " ('va', 834),\n",
              " ('ba', 835),\n",
              " ('】', 836),\n",
              " ('▁bi', 837),\n",
              " ('日', 838),\n",
              " ('한', 839),\n",
              " ('▁24', 840),\n",
              " ('ر', 841),\n",
              " ('ى', 842),\n",
              " ('▁est', 843),\n",
              " ('▁में', 844),\n",
              " ('lar', 845),\n",
              " ('▁2012', 846),\n",
              " ('▁dengan', 847),\n",
              " ('年', 848),\n",
              " ('▁13', 849),\n",
              " ('▁με', 850),\n",
              " ('▁untuk', 851),\n",
              " ('▁Y', 852),\n",
              " (');', 853),\n",
              " ('▁ini', 854),\n",
              " ('▁ש', 855),\n",
              " ('▁ist', 856),\n",
              " ('ve', 857),\n",
              " ('▁ا', 858),\n",
              " ('▁im', 859),\n",
              " ('this', 860),\n",
              " ('est', 861),\n",
              " ('▁online', 862),\n",
              " ('न', 863),\n",
              " ('▁А', 864),\n",
              " ('▁sur', 865),\n",
              " ('J', 866),\n",
              " ('▁У', 867),\n",
              " ('ך', 868),\n",
              " ('은', 869),\n",
              " ('ado', 870),\n",
              " ('▁ti', 871),\n",
              " ('ہ', 872),\n",
              " ('에', 873),\n",
              " ('ri', 874),\n",
              " ('▁för', 875),\n",
              " ('tu', 876),\n",
              " ('▁25', 877),\n",
              " ('lo', 878),\n",
              " ('」', 879),\n",
              " ('den', 880),\n",
              " ('%', 881),\n",
              " ('▁א', 882),\n",
              " ('د', 883),\n",
              " ('▁את', 884),\n",
              " ('▁có', 885),\n",
              " ('▁pas', 886),\n",
              " ('=\"', 887),\n",
              " ('▁ein', 888),\n",
              " ('ou', 889),\n",
              " ('▁mu', 890),\n",
              " ('月', 891),\n",
              " ('▁что', 892),\n",
              " ('ого', 893),\n",
              " ('*', 894),\n",
              " ('ի', 895),\n",
              " ('ים', 896),\n",
              " ('р', 897),\n",
              " ('▁will', 898),\n",
              " ('▁fa', 899),\n",
              " ('net', 900),\n",
              " ('▁για', 901),\n",
              " ('д', 902),\n",
              " ('ê', 903),\n",
              " ('▁*', 904),\n",
              " ('ُ', 905),\n",
              " ('ada', 906),\n",
              " ('▁qui', 907),\n",
              " ('ới', 908),\n",
              " ('г', 909),\n",
              " ('▁over', 910),\n",
              " ('▁17', 911),\n",
              " ('▁από', 912),\n",
              " ('ها', 913),\n",
              " (',\"', 914),\n",
              " ('ā', 915),\n",
              " ('▁را', 916),\n",
              " ('▁со', 917),\n",
              " ('та', 918),\n",
              " ('▁ser', 919),\n",
              " ('л', 920),\n",
              " ('que', 921),\n",
              " ('▁так', 922),\n",
              " ('▁про', 923),\n",
              " ('ể', 924),\n",
              " ('ok', 925),\n",
              " ('▁To', 926),\n",
              " ('▁σ', 927),\n",
              " ('▁და', 928),\n",
              " ('가', 929),\n",
              " ('ό', 930),\n",
              " ('ción', 931),\n",
              " ('ak', 932),\n",
              " ('ị', 933),\n",
              " ('▁که', 934),\n",
              " ('▁non', 935),\n",
              " ('ן', 936),\n",
              " ('▁је', 937),\n",
              " ('ro', 938),\n",
              " ('「', 939),\n",
              " ('ag', 940),\n",
              " ('ان', 941),\n",
              " ('على', 942),\n",
              " ('▁आ', 943),\n",
              " ('ите', 944),\n",
              " ('да', 945),\n",
              " ('с', 946),\n",
              " ('▁się', 947),\n",
              " ('▁€', 948),\n",
              " ('▁mo', 949),\n",
              " ('▁است', 950),\n",
              " ('▁·', 951),\n",
              " ('ý', 952),\n",
              " ('▁این', 953),\n",
              " ('Р', 954),\n",
              " ('▁if', 955),\n",
              " ('▁für', 956),\n",
              " ('не', 957),\n",
              " ('▁como', 958),\n",
              " ('▁X', 959),\n",
              " ('▁ca', 960),\n",
              " ('▁är', 961),\n",
              " ('ní', 962),\n",
              " ('▁19', 963),\n",
              " ('▁co', 964),\n",
              " ('▁כ', 965),\n",
              " ('▁100', 966),\n",
              " ('ere', 967),\n",
              " ('▁að', 968),\n",
              " ('wa', 969),\n",
              " ('▁cho', 970),\n",
              " ('▁voor', 971),\n",
              " ('▁2020', 972),\n",
              " ('▁میں', 973),\n",
              " ('و', 974),\n",
              " ('▁की', 975),\n",
              " ('ji', 976),\n",
              " ('▁Đ', 977),\n",
              " ('も', 978),\n",
              " ('▁pri', 979),\n",
              " ('▁este', 980),\n",
              " ('▁2011', 981),\n",
              " ('▁ce', 982),\n",
              " ('▁О', 983),\n",
              " ('▁է', 984),\n",
              " ('ik', 985),\n",
              " ('ት', 986),\n",
              " ('▁21', 987),\n",
              " ('는', 988),\n",
              " ('ку', 989),\n",
              " ('ж', 990),\n",
              " ('ے', 991),\n",
              " ('▁во', 992),\n",
              " ('ç', 993),\n",
              " ('ে', 994),\n",
              " ('п', 995),\n",
              " ('र', 996),\n",
              " ('Z', 997),\n",
              " ('▁од', 998),\n",
              " ('▁ob', 999),\n",
              " ...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08P0YFnc3aK4"
      },
      "source": [
        "# Prepare Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xIFdSmFImJLE",
        "outputId": "eddf35dc-3f2d-4cd3-d4e2-5e7ebd44cbe6"
      },
      "source": [
        "# Source: https://huggingface.co/datasets/alt\n",
        "dataset = load_dataset('alt')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No config specified, defaulting to: alt/alt-parallel\n",
            "Reusing dataset alt (/root/.cache/huggingface/datasets/alt/alt-parallel/1.0.0/70124cadb11e926eb17904d6edf5f79041aee772bab75a7ef37cb25633fed98f)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V2T0CF-Nmvgc"
      },
      "source": [
        "train_dataset = dataset['train']\n",
        "test_dataset = dataset['test']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pvow98j0kxME",
        "outputId": "164f72e8-ccaa-4dcf-ca73-c1cf691ec435"
      },
      "source": [
        "train_dataset[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'SNT.URLID': '80188',\n",
              " 'SNT.URLID.SNTID': '1',\n",
              " 'translation': {'bg': 'ফ্রান্সের প্যারিসের পার্ক দি প্রিন্সেস-এ হওয়া ২০০৭-এর রাগবি বিশ্বকাপের পুল সি-তে ইটালি পর্তুগালকে ৩১-৫ গোলে হারিয়েছে।',\n",
              "  'en': 'Italy have defeated Portugal 31-5 in Pool C of the 2007 Rugby World Cup at Parc des Princes, Paris, France.',\n",
              "  'en_tok': 'Italy have defeated Portugal 31-5 in Pool C of the 2007 Rugby World Cup at Parc des Princes , Paris , France .',\n",
              "  'fil': 'Natalo ng Italya ang Portugal sa puntos na 31-5 sa Grupong C noong 2007 sa Pandaigdigang laro ng Ragbi sa Parc des Princes, Paris, France.',\n",
              "  'hi': '2007 में फ़्रांस, पेरिस के पार्क डेस प्रिंसेस में हुए रग्बी विश्व कप के पूल C में इटली ने पुर्तगाल को 31-5 से हराया।',\n",
              "  'id': 'Italia berhasil mengalahkan Portugal 31-5 di grup C dalam Piala Dunia Rugby 2007 di Parc des Princes, Paris, Perancis.',\n",
              "  'ja': 'フランスのパリ、パルク・デ・プランスで行われた2007年ラグビーワールドカップのプールCで、イタリアは31対5でポルトガルを下した。',\n",
              "  'khm': 'អ៊ីតាលីបានឈ្នះលើព័រទុយហ្គាល់ 31-5 ក្នុងប៉ូលCនៃពីធីប្រកួតពានរង្វាន់ពិភពលោកនៃកីឡាបាល់ឱបឆ្នាំ2007ដែលប្រព្រឹត្តនៅប៉ាសឌេសប្រីន ក្រុងប៉ារីស បារាំង។',\n",
              "  'lo': 'ອິຕາລີໄດ້ເສຍໃຫ້ປ໊ອກຕຸຍການ 31 ຕໍ່ 5 ໃນພູລ C ຂອງ ການແຂ່ງຂັນຣັກບີ້ລະດັບໂລກປີ 2007 ທີ່ ປາກເດແພຣັງ ປາຣີ ປະເທດຝຣັ່ງ.',\n",
              "  'ms': 'Itali telah mengalahkan Portugal 31-5 dalam Pool C pada Piala Dunia Ragbi 2007 di Parc des Princes, Paris, Perancis.',\n",
              "  'my': 'ပြင်သစ်နိုင်ငံ ပါရီမြို့ ပါ့ဒက်စ် ပရင့်စက် ၌ ၂၀၀၇ခုနှစ် ရပ်ဘီ ကမ္ဘာ့ ဖလား တွင် အီတလီ သည် ပေါ်တူဂီ ကို ၃၁-၅ ဂိုး ဖြင့် ရေကူးကန် စီ တွင် ရှုံးနိမ့်သွားပါသည် ။',\n",
              "  'th': 'อิตาลีได้เอาชนะโปรตุเกสด้วยคะแนน31ต่อ5 ในกลุ่มc ของการแข่งขันรักบี้เวิลด์คัพปี2007 ที่สนามปาร์กเดแพร็งส์ ที่กรุงปารีส ประเทศฝรั่งเศส',\n",
              "  'vi': 'Ý đã đánh bại Bồ Đào Nha với tỉ số 31-5 ở Bảng C Giải vô địch Rugby thế giới 2007 tại Parc des Princes, Pari, Pháp.',\n",
              "  'zh': '意大利在法国巴黎王子公园体育场举办的2007年橄榄球世界杯C组以31-5击败葡萄牙。'},\n",
              " 'url': 'http://en.wikinews.org/wiki/2007_Rugby_World_Cup:_Italy_31_-_5_Portugal'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ydBXAHaqPGeF"
      },
      "source": [
        "LANG_TOKEN_MAPPING = {\n",
        "    'en': '<en>',\n",
        "    'ja': '<jp>',\n",
        "    'zh': '<zh>'\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p_a6ur1FTkD_",
        "outputId": "35b98b9f-7f0d-4451-d441-1989510df1b3"
      },
      "source": [
        "special_tokens_dict = {'additional_special_tokens': list(LANG_TOKEN_MAPPING.values())}\n",
        "tokenizer.add_special_tokens(special_tokens_dict)\n",
        "model.resize_token_embeddings(len(tokenizer))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Embedding(250103, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pMw-kMxyKiGC",
        "outputId": "c9bf7811-e899-4166-f185-95f2e4a2602d"
      },
      "source": [
        "token_ids = tokenizer.encode(\n",
        "    example_input_str, return_tensors='pt', padding='max_length',\n",
        "    truncation=True, max_length=max_seq_len)\n",
        "print(token_ids)\n",
        "\n",
        "tokens = tokenizer.convert_ids_to_tokens(token_ids[0])\n",
        "print(tokens)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[250101,   1494,    339,   1627,    259,    262,   2978,    259,    272,\n",
            "           1982,   1315,    260,      1,      0,      0,      0,      0,      0,\n",
            "              0,      0]])\n",
            "['<jp>', '▁This', '▁is', '▁just', '▁', 'a', '▁test', '▁', 'n', 'bu', 'ig', '.', '</s>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJNx4Mw7nlRr"
      },
      "source": [
        "def encode_input_str(text, target_lang, tokenizer, seq_len,\n",
        "                     lang_token_map=LANG_TOKEN_MAPPING):\n",
        "  target_lang_token = lang_token_map[target_lang]\n",
        "\n",
        "  # Tokenize and add special tokens\n",
        "  input_ids = tokenizer.encode(\n",
        "      text = target_lang_token + text,\n",
        "      return_tensors = 'pt',\n",
        "      padding = 'max_length',\n",
        "      truncation = True,\n",
        "      max_length = seq_len)\n",
        "\n",
        "  return input_ids[0]\n",
        "  \n",
        "def encode_target_str(text, tokenizer, seq_len,\n",
        "                      lang_token_map=LANG_TOKEN_MAPPING):\n",
        "  token_ids = tokenizer.encode(\n",
        "      text = text,\n",
        "      return_tensors = 'pt',\n",
        "      padding = 'max_length',\n",
        "      truncation = True,\n",
        "      max_length = seq_len)\n",
        "  \n",
        "  return token_ids[0]\n",
        "\n",
        "def format_translation_data(translations, lang_token_map,\n",
        "                            tokenizer, seq_len=128):\n",
        "  # Choose a random 2 languages for in i/o\n",
        "  langs = list(lang_token_map.keys())\n",
        "  input_lang, target_lang = np.random.choice(langs, size=2, replace=False)\n",
        "\n",
        "  # Get the translations for the batch\n",
        "  input_text = translations[input_lang]\n",
        "  target_text = translations[target_lang]\n",
        "\n",
        "  if input_text is None or target_text is None:\n",
        "    return None\n",
        "\n",
        "  input_token_ids = encode_input_str(\n",
        "      input_text, target_lang, tokenizer, seq_len, lang_token_map)\n",
        "  \n",
        "  target_token_ids = encode_target_str(\n",
        "      target_text, tokenizer, seq_len, lang_token_map)\n",
        "\n",
        "  return input_token_ids, target_token_ids\n",
        "\n",
        "def transform_batch(batch, lang_token_map, tokenizer):\n",
        "  inputs = []\n",
        "  targets = []\n",
        "  for translation_set in batch['translation']:\n",
        "    formatted_data = format_translation_data(\n",
        "        translation_set, lang_token_map, tokenizer, max_seq_len)\n",
        "    \n",
        "    if formatted_data is None:\n",
        "      continue\n",
        "    \n",
        "    input_ids, target_ids = formatted_data\n",
        "    inputs.append(input_ids.unsqueeze(0))\n",
        "    targets.append(target_ids.unsqueeze(0))\n",
        "    \n",
        "  batch_input_ids = torch.cat(inputs).cuda()\n",
        "  batch_target_ids = torch.cat(targets).cuda()\n",
        "\n",
        "  return batch_input_ids, batch_target_ids\n",
        "\n",
        "def get_data_generator(dataset, lang_token_map, tokenizer, batch_size=32):\n",
        "  dataset = dataset.shuffle()\n",
        "  for i in range(0, len(dataset), batch_size):\n",
        "    raw_batch = dataset[i:i+batch_size]\n",
        "    yield transform_batch(raw_batch, lang_token_map, tokenizer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmdUeqObF_j_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "51a9c893-ba45-4048-a176-c8972454c504"
      },
      "source": [
        "# Testing `data_transform`\n",
        "in_ids, out_ids = format_translation_data(\n",
        "    train_dataset[0]['translation'], LANG_TOKEN_MAPPING, tokenizer)\n",
        "\n",
        "print(' '.join(tokenizer.convert_ids_to_tokens(in_ids)))\n",
        "print(' '.join(tokenizer.convert_ids_to_tokens(out_ids)))\n",
        "\n",
        "# Testing data generator\n",
        "data_gen = get_data_generator(train_dataset, LANG_TOKEN_MAPPING, tokenizer, 8)\n",
        "data_batch = next(data_gen)\n",
        "print('Input shape:', data_batch[0].shape)\n",
        "print('Output shape:', data_batch[1].shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<zh> ▁Italy ▁have ▁de feat ed ▁Portugal ▁3 1-5 ▁in ▁Pool ▁C ▁of ▁the ▁2007 ▁ Rugby ▁World ▁Cup ▁at ▁Parc ▁des ▁Princes , ▁Paris , ▁France . </s> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n",
            "▁ 意大利 在 法国 巴黎 王子 公园 体育 场 举办 的 2007 年 橄 榄 球 世界杯 C 组 以 3 1-5 击 败 葡萄 牙 。 </s> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad> <pad>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Loading cached shuffled indices for dataset at /root/.cache/huggingface/datasets/alt/alt-parallel/1.0.0/70124cadb11e926eb17904d6edf5f79041aee772bab75a7ef37cb25633fed98f/cache-3f2f82a4f39ef8f1.arrow\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Input shape: torch.Size([8, 20])\n",
            "Output shape: torch.Size([8, 20])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xKLEqcU4m9M0"
      },
      "source": [
        "# Train/Finetune BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQ8R4WCHpfuf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37633608-d23b-4f49-db53-bdbbae146b5d"
      },
      "source": [
        "model.load_state_dict(torch.load(model_path))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4uv5u_FnE2F"
      },
      "source": [
        "# Constants\n",
        "n_epochs = 8\n",
        "batch_size = 16\n",
        "print_freq = 50\n",
        "checkpoint_freq = 1000\n",
        "lr = 5e-4\n",
        "n_batches = int(np.ceil(len(train_dataset) / batch_size))\n",
        "total_steps = n_epochs * n_batches\n",
        "n_warmup_steps = int(total_steps * 0.01)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yv-85lPx-Oo7"
      },
      "source": [
        "# Optimizer\n",
        "optimizer = AdamW(model.parameters(), lr=lr)\n",
        "scheduler = get_linear_schedule_with_warmup(\n",
        "    optimizer, n_warmup_steps, total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OLN37ltk__ws"
      },
      "source": [
        "losses = []"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKU9rtbHWkeB"
      },
      "source": [
        "def eval_model(model, gdataset, max_iters=8):\n",
        "  test_generator = get_data_generator(gdataset, LANG_TOKEN_MAPPING,\n",
        "                                      tokenizer, batch_size)\n",
        "  eval_losses = []\n",
        "  for i, (input_batch, label_batch) in enumerate(test_generator):\n",
        "    if i >= max_iters:\n",
        "      break\n",
        "\n",
        "    model_out = model.forward(\n",
        "        input_ids = input_batch,\n",
        "        labels = label_batch)\n",
        "    eval_losses.append(model_out.loss.item())\n",
        "\n",
        "  return np.mean(eval_losses)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "d547c94dd3a64665bba324c697febb00",
            "f42bb32b70cd48129cd47f103d4e7833",
            "b968d0ec5ec84cda9932ef2e5d810f94",
            "16b6d1c4359448628d03f2c3ba26db34",
            "e687f593436f42ed8ed07b45babb6cd2",
            "327d12730f91427c9346e4c34cddc97a",
            "8feff991e3654109a593b68c09c71989",
            "3f2c6721e0db45c0a14106f34724ef81",
            "dbfb67ca7cd0466c976488cca4a36c6e",
            "7046edd83446491aaab2256e6bcc52e7",
            "0aed11e47cec49c3ac6fe4c63000769e",
            "eba82bf9861342c49b78b1e61408aecd",
            "69ae01a816004cbebe52c0f8d6f98f25",
            "1a720adf34b34713b9ac59d24d1404c2",
            "b2cd47dd3eb54d19be042d5dd5f6de25",
            "382b6251119d4fb881c8ae0538efcd10",
            "1c5f2cbab1044d6fa7e63e0c19a3db3c",
            "b9456d09ce5d46c6b17019845ed8ee11",
            "9cdbf499f0564e4695c7cb1c02abd8ff",
            "16873be391fe4ae186022a9a81775d4d",
            "83e308daead14376a22df02643df80fe",
            "f6f7f7266d6a472a959068b9ae8d40fe",
            "3854068540044f3e99bc6378a83ffc26",
            "2f642e2a22b84bfeb8fd9067e84b4db8",
            "03325133b7554a76840d4d2367bcd0c3",
            "9655332d70af4c0fa96fdea7d3dad056",
            "6e04c5ab370046018bffa827e93c5173",
            "d619d55e2cd14d8d8c2fa135c3b8ea6a",
            "fd4e775d53c84798aa610ac283d7b629",
            "39bf3356ce2e4ec28d52954a7e1eaae6",
            "d83db5b082cc4cf496693a5256395468",
            "6e6773ea09854295b135401ea5db7c41",
            "9122956a67ee4e26a21f7d2ac3ce4357",
            "fad9ca4eb1064a43a6e0a6f06fd418f7",
            "0aa829059d8a4fd7a698513304f434bb",
            "8dbf73941f5e4e7d9df959239a093742",
            "44d97dcd66794416b178510faaba22a8",
            "16114ff04cf84746a880f846bb57f24f",
            "be5b064142a64a2686e2e469c5e647ab",
            "2a5cb70b7e0c46bc9b5ae563319f4467",
            "10542fa0178b4ab1aa166ece33c9d92f",
            "0fe64c14a4ca46a9b83de6c7cf1b1feb",
            "b2e727825c9a477fb5ee8c4ee281963d",
            "364a164f8e114f2e8e9d0c287050b33e",
            "62b02344033d485397b881cea48b3bbd",
            "53edd8b36f134b4a9aa62984866f3893",
            "2d999834e4e4488aae3b033b7d099323",
            "d52c1e0ff627430985aac092e57836ea",
            "912d78120eae4d768cae2c5d6ecb25dc",
            "01ef8dacfaec47369c59e43d28e0129b",
            "3c2357cfe1534f019146a1c08848532e",
            "c69c5625b54e4923bd509668534583a4",
            "dfed8ec945b0431c9d81e2ad40dd48ad",
            "80b7604fe4d44700bf37d892993712cb",
            "63ff944b5fb04840b9a44ce4c965eeb0",
            "a2da175f276d4d819cfd9bf8114187d0"
          ]
        },
        "id": "Kv8a0jwDnEzK",
        "outputId": "5c70baf4-6bea-4e25-d355-4f365e2b4826"
      },
      "source": [
        "for epoch_idx in range(n_epochs):\n",
        "  # Randomize data order\n",
        "  data_generator = get_data_generator(train_dataset, LANG_TOKEN_MAPPING,\n",
        "                                      tokenizer, batch_size)\n",
        "                \n",
        "  for batch_idx, (input_batch, label_batch) \\\n",
        "      in tqdm_notebook(enumerate(data_generator), total=n_batches):\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # Forward pass\n",
        "    model_out = model.forward(\n",
        "        input_ids = input_batch,\n",
        "        labels = label_batch)\n",
        "\n",
        "    # Calculate loss and update weights\n",
        "    loss = model_out.loss\n",
        "    losses.append(loss.item())\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    scheduler.step()\n",
        "\n",
        "    # Print training update info\n",
        "    if (batch_idx + 1) % print_freq == 0:\n",
        "      avg_loss = np.mean(losses[-print_freq:])\n",
        "      print('Epoch: {} | Step: {} | Avg. loss: {:.3f} | lr: {}'.format(\n",
        "          epoch_idx+1, batch_idx+1, avg_loss, scheduler.get_last_lr()[0]))\n",
        "      \n",
        "    if (batch_idx + 1) % checkpoint_freq == 0:\n",
        "      test_loss = eval_model(model, test_dataset)\n",
        "      print('Saving model with test loss of {:.3f}'.format(test_loss))\n",
        "      torch.save(model.state_dict(), model_path)\n",
        "\n",
        "torch.save(model.state_dict(), model_path)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:6: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
            "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d547c94dd3a64665bba324c697febb00",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 | Step: 50 | Avg. loss: 2.017 | lr: 0.0003060392944853762\n",
            "Epoch: 1 | Step: 100 | Avg. loss: 2.022 | lr: 0.00030324849296718015\n",
            "Epoch: 1 | Step: 150 | Avg. loss: 2.073 | lr: 0.0003004576914489842\n",
            "Epoch: 1 | Step: 200 | Avg. loss: 2.025 | lr: 0.0002976668899307881\n",
            "Epoch: 1 | Step: 250 | Avg. loss: 2.012 | lr: 0.0002948760884125921\n",
            "Epoch: 1 | Step: 300 | Avg. loss: 2.066 | lr: 0.0002920852868943961\n",
            "Epoch: 1 | Step: 350 | Avg. loss: 2.069 | lr: 0.00028929448537620006\n",
            "Epoch: 1 | Step: 400 | Avg. loss: 2.087 | lr: 0.00028650368385800404\n",
            "Epoch: 1 | Step: 450 | Avg. loss: 2.055 | lr: 0.000283712882339808\n",
            "Epoch: 1 | Step: 500 | Avg. loss: 2.060 | lr: 0.00028092208082161195\n",
            "Epoch: 1 | Step: 550 | Avg. loss: 2.088 | lr: 0.000278131279303416\n",
            "Epoch: 1 | Step: 600 | Avg. loss: 2.095 | lr: 0.0002753404777852199\n",
            "Epoch: 1 | Step: 650 | Avg. loss: 2.046 | lr: 0.00027254967626702385\n",
            "Epoch: 1 | Step: 700 | Avg. loss: 2.117 | lr: 0.0002697588747488279\n",
            "Epoch: 1 | Step: 750 | Avg. loss: 2.067 | lr: 0.0002669680732306318\n",
            "Epoch: 1 | Step: 800 | Avg. loss: 2.082 | lr: 0.00026417727171243585\n",
            "Epoch: 1 | Step: 850 | Avg. loss: 2.078 | lr: 0.0002613864701942398\n",
            "Epoch: 1 | Step: 900 | Avg. loss: 2.086 | lr: 0.00025859566867604376\n",
            "Epoch: 1 | Step: 950 | Avg. loss: 2.036 | lr: 0.00025580486715784774\n",
            "Epoch: 1 | Step: 1000 | Avg. loss: 2.061 | lr: 0.00025301406563965173\n",
            "Saving model with test loss of 2.530\n",
            "Epoch: 1 | Step: 1050 | Avg. loss: 2.066 | lr: 0.0002502232641214557\n",
            "Epoch: 1 | Step: 1100 | Avg. loss: 2.024 | lr: 0.00024743246260325964\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dbfb67ca7cd0466c976488cca4a36c6e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 2 | Step: 50 | Avg. loss: 1.874 | lr: 0.00024291136414378208\n",
            "Epoch: 2 | Step: 100 | Avg. loss: 1.892 | lr: 0.00024012056262558607\n",
            "Epoch: 2 | Step: 150 | Avg. loss: 1.883 | lr: 0.00023732976110739005\n",
            "Epoch: 2 | Step: 200 | Avg. loss: 1.863 | lr: 0.00023453895958919403\n",
            "Epoch: 2 | Step: 250 | Avg. loss: 1.917 | lr: 0.000231748158070998\n",
            "Epoch: 2 | Step: 300 | Avg. loss: 1.878 | lr: 0.00022895735655280197\n",
            "Epoch: 2 | Step: 350 | Avg. loss: 1.908 | lr: 0.00022616655503460595\n",
            "Epoch: 2 | Step: 400 | Avg. loss: 1.882 | lr: 0.00022337575351640993\n",
            "Epoch: 2 | Step: 450 | Avg. loss: 1.813 | lr: 0.0002205849519982139\n",
            "Epoch: 2 | Step: 500 | Avg. loss: 1.961 | lr: 0.00021779415048001785\n",
            "Epoch: 2 | Step: 550 | Avg. loss: 1.819 | lr: 0.00021500334896182183\n",
            "Epoch: 2 | Step: 600 | Avg. loss: 1.885 | lr: 0.0002122125474436258\n",
            "Epoch: 2 | Step: 650 | Avg. loss: 1.813 | lr: 0.0002094217459254298\n",
            "Epoch: 2 | Step: 700 | Avg. loss: 1.815 | lr: 0.00020663094440723375\n",
            "Epoch: 2 | Step: 750 | Avg. loss: 1.861 | lr: 0.00020384014288903773\n",
            "Epoch: 2 | Step: 800 | Avg. loss: 1.838 | lr: 0.00020104934137084172\n",
            "Epoch: 2 | Step: 850 | Avg. loss: 1.862 | lr: 0.0001982585398526457\n",
            "Epoch: 2 | Step: 900 | Avg. loss: 1.900 | lr: 0.00019546773833444968\n",
            "Epoch: 2 | Step: 950 | Avg. loss: 1.881 | lr: 0.00019267693681625364\n",
            "Epoch: 2 | Step: 1000 | Avg. loss: 1.884 | lr: 0.0001898861352980576\n",
            "Saving model with test loss of 2.394\n",
            "Epoch: 2 | Step: 1050 | Avg. loss: 1.857 | lr: 0.00018709533377986158\n",
            "Epoch: 2 | Step: 1100 | Avg. loss: 1.915 | lr: 0.00018430453226166556\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1c5f2cbab1044d6fa7e63e0c19a3db3c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 3 | Step: 50 | Avg. loss: 1.627 | lr: 0.00017978343380218797\n",
            "Epoch: 3 | Step: 100 | Avg. loss: 1.748 | lr: 0.00017699263228399196\n",
            "Epoch: 3 | Step: 150 | Avg. loss: 1.715 | lr: 0.00017420183076579594\n",
            "Epoch: 3 | Step: 200 | Avg. loss: 1.739 | lr: 0.0001714110292475999\n",
            "Epoch: 3 | Step: 250 | Avg. loss: 1.648 | lr: 0.00016862022772940388\n",
            "Epoch: 3 | Step: 300 | Avg. loss: 1.733 | lr: 0.00016582942621120786\n",
            "Epoch: 3 | Step: 350 | Avg. loss: 1.702 | lr: 0.00016303862469301184\n",
            "Epoch: 3 | Step: 400 | Avg. loss: 1.734 | lr: 0.00016024782317481583\n",
            "Epoch: 3 | Step: 450 | Avg. loss: 1.691 | lr: 0.00015745702165661978\n",
            "Epoch: 3 | Step: 500 | Avg. loss: 1.766 | lr: 0.00015466622013842377\n",
            "Epoch: 3 | Step: 550 | Avg. loss: 1.719 | lr: 0.00015187541862022775\n",
            "Epoch: 3 | Step: 600 | Avg. loss: 1.705 | lr: 0.0001490846171020317\n",
            "Epoch: 3 | Step: 650 | Avg. loss: 1.681 | lr: 0.00014629381558383566\n",
            "Epoch: 3 | Step: 700 | Avg. loss: 1.756 | lr: 0.00014350301406563964\n",
            "Epoch: 3 | Step: 750 | Avg. loss: 1.747 | lr: 0.00014071221254744363\n",
            "Epoch: 3 | Step: 800 | Avg. loss: 1.721 | lr: 0.0001379214110292476\n",
            "Epoch: 3 | Step: 850 | Avg. loss: 1.703 | lr: 0.0001351306095110516\n",
            "Epoch: 3 | Step: 900 | Avg. loss: 1.743 | lr: 0.00013233980799285555\n",
            "Epoch: 3 | Step: 950 | Avg. loss: 1.765 | lr: 0.00012954900647465953\n",
            "Epoch: 3 | Step: 1000 | Avg. loss: 1.690 | lr: 0.0001267582049564635\n",
            "Saving model with test loss of 2.397\n",
            "Epoch: 3 | Step: 1050 | Avg. loss: 1.686 | lr: 0.00012396740343826747\n",
            "Epoch: 3 | Step: 1100 | Avg. loss: 1.725 | lr: 0.00012117660192007145\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "03325133b7554a76840d4d2367bcd0c3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 4 | Step: 50 | Avg. loss: 1.526 | lr: 0.00011665550346059388\n",
            "Epoch: 4 | Step: 100 | Avg. loss: 1.610 | lr: 0.00011386470194239785\n",
            "Epoch: 4 | Step: 150 | Avg. loss: 1.575 | lr: 0.00011107390042420183\n",
            "Epoch: 4 | Step: 200 | Avg. loss: 1.557 | lr: 0.00010828309890600582\n",
            "Epoch: 4 | Step: 250 | Avg. loss: 1.622 | lr: 0.00010549229738780978\n",
            "Epoch: 4 | Step: 300 | Avg. loss: 1.555 | lr: 0.00010270149586961377\n",
            "Epoch: 4 | Step: 350 | Avg. loss: 1.618 | lr: 9.991069435141772e-05\n",
            "Epoch: 4 | Step: 400 | Avg. loss: 1.586 | lr: 9.71198928332217e-05\n",
            "Epoch: 4 | Step: 450 | Avg. loss: 1.589 | lr: 9.432909131502567e-05\n",
            "Epoch: 4 | Step: 500 | Avg. loss: 1.558 | lr: 9.153828979682966e-05\n",
            "Epoch: 4 | Step: 550 | Avg. loss: 1.579 | lr: 8.874748827863363e-05\n",
            "Epoch: 4 | Step: 600 | Avg. loss: 1.559 | lr: 8.59566867604376e-05\n",
            "Epoch: 4 | Step: 650 | Avg. loss: 1.529 | lr: 8.316588524224157e-05\n",
            "Epoch: 4 | Step: 700 | Avg. loss: 1.520 | lr: 8.037508372404555e-05\n",
            "Epoch: 4 | Step: 750 | Avg. loss: 1.595 | lr: 7.758428220584953e-05\n",
            "Epoch: 4 | Step: 800 | Avg. loss: 1.557 | lr: 7.47934806876535e-05\n",
            "Epoch: 4 | Step: 850 | Avg. loss: 1.548 | lr: 7.200267916945747e-05\n",
            "Epoch: 4 | Step: 900 | Avg. loss: 1.529 | lr: 6.921187765126144e-05\n",
            "Epoch: 4 | Step: 950 | Avg. loss: 1.573 | lr: 6.642107613306542e-05\n",
            "Epoch: 4 | Step: 1000 | Avg. loss: 1.566 | lr: 6.363027461486939e-05\n",
            "Saving model with test loss of 2.346\n",
            "Epoch: 4 | Step: 1050 | Avg. loss: 1.543 | lr: 6.0839473096673366e-05\n",
            "Epoch: 4 | Step: 1100 | Avg. loss: 1.605 | lr: 5.804867157847734e-05\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9122956a67ee4e26a21f7d2ac3ce4357",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 5 | Step: 50 | Avg. loss: 1.459 | lr: 5.352757311899978e-05\n",
            "Epoch: 5 | Step: 100 | Avg. loss: 1.402 | lr: 5.0736771600803755e-05\n",
            "Epoch: 5 | Step: 150 | Avg. loss: 1.434 | lr: 4.7945970082607724e-05\n",
            "Epoch: 5 | Step: 200 | Avg. loss: 1.468 | lr: 4.51551685644117e-05\n",
            "Epoch: 5 | Step: 250 | Avg. loss: 1.414 | lr: 4.236436704621567e-05\n",
            "Epoch: 5 | Step: 300 | Avg. loss: 1.457 | lr: 3.9573565528019646e-05\n",
            "Epoch: 5 | Step: 350 | Avg. loss: 1.426 | lr: 3.678276400982362e-05\n",
            "Epoch: 5 | Step: 400 | Avg. loss: 1.465 | lr: 3.399196249162759e-05\n",
            "Epoch: 5 | Step: 450 | Avg. loss: 1.476 | lr: 3.1201160973431574e-05\n",
            "Epoch: 5 | Step: 500 | Avg. loss: 1.484 | lr: 2.8410359455235543e-05\n",
            "Epoch: 5 | Step: 550 | Avg. loss: 1.507 | lr: 2.5619557937039515e-05\n",
            "Epoch: 5 | Step: 600 | Avg. loss: 1.466 | lr: 2.2828756418843495e-05\n",
            "Epoch: 5 | Step: 650 | Avg. loss: 1.465 | lr: 2.0037954900647467e-05\n",
            "Epoch: 5 | Step: 700 | Avg. loss: 1.449 | lr: 1.724715338245144e-05\n",
            "Epoch: 5 | Step: 750 | Avg. loss: 1.415 | lr: 1.4456351864255414e-05\n",
            "Epoch: 5 | Step: 800 | Avg. loss: 1.489 | lr: 1.1665550346059389e-05\n",
            "Epoch: 5 | Step: 850 | Avg. loss: 1.460 | lr: 8.874748827863363e-06\n",
            "Epoch: 5 | Step: 900 | Avg. loss: 1.424 | lr: 6.0839473096673365e-06\n",
            "Epoch: 5 | Step: 950 | Avg. loss: 1.375 | lr: 3.2931457914713108e-06\n",
            "Epoch: 5 | Step: 1000 | Avg. loss: 1.468 | lr: 5.023442732752847e-07\n",
            "Saving model with test loss of 2.608\n",
            "Epoch: 5 | Step: 1050 | Avg. loss: 1.418 | lr: 0.0\n",
            "Epoch: 5 | Step: 1100 | Avg. loss: 1.497 | lr: 0.0\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "10542fa0178b4ab1aa166ece33c9d92f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 6 | Step: 50 | Avg. loss: 1.382 | lr: 0.0\n",
            "Epoch: 6 | Step: 100 | Avg. loss: 1.382 | lr: 0.0\n",
            "Epoch: 6 | Step: 150 | Avg. loss: 1.391 | lr: 0.0\n",
            "Epoch: 6 | Step: 200 | Avg. loss: 1.332 | lr: 0.0\n",
            "Epoch: 6 | Step: 250 | Avg. loss: 1.435 | lr: 0.0\n",
            "Epoch: 6 | Step: 300 | Avg. loss: 1.448 | lr: 0.0\n",
            "Epoch: 6 | Step: 350 | Avg. loss: 1.401 | lr: 0.0\n",
            "Epoch: 6 | Step: 400 | Avg. loss: 1.385 | lr: 0.0\n",
            "Epoch: 6 | Step: 450 | Avg. loss: 1.375 | lr: 0.0\n",
            "Epoch: 6 | Step: 500 | Avg. loss: 1.348 | lr: 0.0\n",
            "Epoch: 6 | Step: 550 | Avg. loss: 1.368 | lr: 0.0\n",
            "Epoch: 6 | Step: 600 | Avg. loss: 1.412 | lr: 0.0\n",
            "Epoch: 6 | Step: 650 | Avg. loss: 1.448 | lr: 0.0\n",
            "Epoch: 6 | Step: 700 | Avg. loss: 1.411 | lr: 0.0\n",
            "Epoch: 6 | Step: 750 | Avg. loss: 1.393 | lr: 0.0\n",
            "Epoch: 6 | Step: 800 | Avg. loss: 1.418 | lr: 0.0\n",
            "Epoch: 6 | Step: 850 | Avg. loss: 1.446 | lr: 0.0\n",
            "Epoch: 6 | Step: 900 | Avg. loss: 1.408 | lr: 0.0\n",
            "Epoch: 6 | Step: 950 | Avg. loss: 1.434 | lr: 0.0\n",
            "Epoch: 6 | Step: 1000 | Avg. loss: 1.381 | lr: 0.0\n",
            "Saving model with test loss of 2.566\n",
            "Epoch: 6 | Step: 1050 | Avg. loss: 1.400 | lr: 0.0\n",
            "Epoch: 6 | Step: 1100 | Avg. loss: 1.411 | lr: 0.0\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "912d78120eae4d768cae2c5d6ecb25dc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1131.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: 7 | Step: 50 | Avg. loss: 1.431 | lr: 0.0\n",
            "Epoch: 7 | Step: 100 | Avg. loss: 1.414 | lr: 0.0\n",
            "Epoch: 7 | Step: 150 | Avg. loss: 1.420 | lr: 0.0\n",
            "Epoch: 7 | Step: 200 | Avg. loss: 1.376 | lr: 0.0\n",
            "Epoch: 7 | Step: 250 | Avg. loss: 1.377 | lr: 0.0\n",
            "Epoch: 7 | Step: 300 | Avg. loss: 1.422 | lr: 0.0\n",
            "Epoch: 7 | Step: 350 | Avg. loss: 1.369 | lr: 0.0\n",
            "Epoch: 7 | Step: 400 | Avg. loss: 1.354 | lr: 0.0\n",
            "Epoch: 7 | Step: 450 | Avg. loss: 1.338 | lr: 0.0\n",
            "Epoch: 7 | Step: 500 | Avg. loss: 1.390 | lr: 0.0\n",
            "Epoch: 7 | Step: 550 | Avg. loss: 1.405 | lr: 0.0\n",
            "Epoch: 7 | Step: 600 | Avg. loss: 1.429 | lr: 0.0\n",
            "Epoch: 7 | Step: 650 | Avg. loss: 1.377 | lr: 0.0\n",
            "Epoch: 7 | Step: 700 | Avg. loss: 1.402 | lr: 0.0\n",
            "Epoch: 7 | Step: 750 | Avg. loss: 1.447 | lr: 0.0\n",
            "Epoch: 7 | Step: 800 | Avg. loss: 1.429 | lr: 0.0\n",
            "Epoch: 7 | Step: 850 | Avg. loss: 1.381 | lr: 0.0\n",
            "Epoch: 7 | Step: 900 | Avg. loss: 1.411 | lr: 0.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-32-92a74ed0c718>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_out\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mlosses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mscheduler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    243\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 245\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    145\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m    146\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 147\u001b[0;31m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tgaCaATQNr9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "outputId": "73ee3948-39ad-4db6-d9e8-a0541ee4f001"
      },
      "source": [
        "# Graph the loss\n",
        "\n",
        "window_size = 50\n",
        "smoothed_losses = []\n",
        "for i in range(len(losses)-window_size):\n",
        "  smoothed_losses.append(np.mean(losses[i:i+window_size]))\n",
        "\n",
        "plt.plot(smoothed_losses[100:])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fc5d2affad0>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD7CAYAAABgzo9kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deUBU5f4/8PfMsI4sAzjsKKZJuCfmviSmUleFe6ublVZXFAvXut70ZmZp3cTMNDVRs/p2f2XdbgWZVuZFyz1cE1NRFBd2hk12Yc7vj4GBcQaYgYGZOb5ff82c85wzn0fwMw/PeRaJIAgCiIhIdKSWDoCIiNoHEzwRkUgxwRMRiRQTPBGRSDHBExGJFBM8EZFIMcETEYmUnaUDaKywsAxqtenD8r28XKBSlbZDRJbHutkm1s022VrdpFIJPDw6NXneqhK8Wi20KsHXXytWrJttYt1sk5jqxi4aIiKRYoInIhIpJngiIpFigiciEikmeCIikWKCJyISKZtP8Gcu52P+u/tQq1ZbOhQiIqti8wk+p7ACVzNLUFlda+lQiIisis0neAc7TRVu17AFT0TUmM0neDsZEzwRkSEiSPASAEBNLRM8EVFjIkjwmirU1opn/QgiInMQTYKv4SgaIiIdIkjw9V00bMETETVm8wlepu2iYQueiKgxm0/wbMETERkmggRf1wfPFjwRkQ6bT/AyKVvwRESGmJTgN27ciJCQEKSmpuqdW7JkCUaPHo3IyEhERkZi8+bNZguyOWzBExEZZvSerOfOncPp06cREBDQZJmYmBhMmzbNLIEZy86OCZ6IyBCjWvDV1dVYsWIFXn/99XYOx3T2XKqAiMggoxL8+vXrMWXKFAQGBjZb7uOPP8bkyZMRGxuLtLQ0swTYEkd7TRWqb3M1SSKixlrsojl16hRSUlKwaNGiZsu9+OKLUCqVkEqlSEhIwMyZM7F3717IZDKjg/HycjG6bL36rhk7Bzsola4mX28LxFovgHWzVaybbWgxwScnJyMtLQ3jxo0DAGRnZyM6Ohpvv/02Ro4cqS3n4+OjfR0VFYW3334b2dnZzfbZ30mlKoVabfpoGJlUgsLiCuTl3TL5WmunVLqKsl4A62arWDfrIZVKmm0Yt5jgY2JiEBMTo30fHh6O+Ph49OzZU6dcTk6ONskfOHAAUqlUJ+m3J2dHO5RV1nTIZxER2QqjR9EYEhkZia1bt8LHxweLFy+GSqWCRCKBi4sLNm/eDDu7Nt3eaK5yB1RWMcETETVmcgZOSkrSvk5MTNS+/uSTT8wSUGs4O9mhggmeiEiHzc9kBQA5EzwRkR5RJHgXZ3uUM8ETEekQSYJ34ENWIqI7iCPBy+1RVnHb0mEQEVkVUSR4V7kDqmvUuF3D2axERPVEkuDtAYDdNEREjYgiwbvIHQCA3TRERI2IIsGzBU9EpE8UCb6+BX/heqGFIyEish6iSPCudQk+4cBVC0dCRGQ9RJLg7S0dAhGR1RFFgpc7McETEd1JFAkeAEK7egAAatXcuo+ICBBRgndy0OwclVtYYeFIiIisg2gS/Oj+/gCAsgoOlSQiAkSU4D3dnAAARaVVFo6EiMg6iCbBK1w0QyULmeCJiACIKMG7ONtDJpWwBU9EVEc0CV4ikcCtkwNulXE9GiIiQEQJHgCcHe1QUc2HrEREgIkJfuPGjQgJCUFqaqreuYqKCixcuBDjx49HREQE9u3bZ7YgjeXsIEMlt+4jIgIA2Blb8Ny5czh9+jQCAgIMnt++fTtcXFzw888/Iz09HU8//TT27NmDTp06mS3Yljg72nFvViKiOka14Kurq7FixQq8/vrrTZb54Ycf8MQTTwAAgoOD0adPH/z6669mCdJYTo52qGCCJyICYGSCX79+PaZMmYLAwMAmy2RmZuq07v38/JCdnd32CE0gd5SxBU9EVKfFLppTp04hJSUFixYtavdgvLxcWn2tUukKT4UcVdW5UCpdzRiV5YmtPo2xbraJdbMNLSb45ORkpKWlYdy4cQCA7OxsREdH4+2338bIkSO15fz9/ZGRkQFPT08AQFZWFoYMGWJSMCpVKdRqwaRrAM0PJC/vFlCrRmV1LbJziiGTimOAkLZuIsS62SbWzXpIpZJmG8YtZsGYmBgcPHgQSUlJSEpKgq+vL7Zv366T3AEgIiICX375JQAgPT0dZ8+exahRo9oYvmmcHTXfVxVVtR36uURE1qhNzdzIyEjk5OQAAKKjo1FSUoLx48dj9uzZWLFiBVxcWt/l0hoNCZ798ERERg+TrJeUlKR9nZiYqH0tl8vx/vvvmyeqVnJ21CwZzARPRCTCmawAEzwRESDaBM8+eCIikSZ4tuCJiESZ4DnZiYhIZAleXveQtZIrShIRiSvB29vJYCeTsAVPRASRJXgAcHKw40NWIiKIMMHLuaIkEREAESZ4ZyZ4IiIAokzwMiZ4IiKIMsGzBU9EBDDBExGJligTfDlH0RARiTHBy1BZVQNBMH3jECIiMRFdggcAAUBlNVvxRHR3E12C/9+JmwCAHXsvWTgSIiLLEl2CnzW5NwDg4NksC0dCRGRZokvwIUEKAIC3wtnCkRARWZboEryzox28Fc64x9/N0qEQEVmU6BI8AOQWVeDoHzmWDoOIyKKM2nQ7NjYWN2/ehFQqhVwux7JlyxAaGqpTZsOGDfj888/h7e0NABg4cCCWL19u/ohNoBYESCUSi8ZARGQpRiX4uLg4uLq6AgD27t2LV155Bd9++61euaioKCxevNi8EbZBafltuHVysHQYREQWYVQXTX1yB4DS0lJIrLxVPHl4MADg0s0iywZCRGRBRrXgAWDp0qU4dOgQBEHAhx9+aLDMrl27cPDgQSiVSsybNw/333+/ScF4ebmYVL4xpbLhS2js4C7YeTgd7u7OOsdtlRjq0BTWzTaxbrZBIpg4pz8hIQG7du3Ctm3bdI7n5eVBoVDA3t4ehw4dwqJFi7B79254eHgYfW+VqhRqtelLDCiVrsjLu6V9X3irCn/fdAhPj++JcWGBJt/PmtxZNzFh3WwT62Y9pFJJsw1jk0fRREVF4dixYygsLNQ5rlQqYW9vDwAYMWIE/Pz8cOmSZWaTurs4QCqRoKi0yiKfT0RkDVpM8GVlZcjKapgVmpSUBHd3dygUCp1yOTkNwxLPnz+PjIwMdOvWzYyhGk8qkUAtCNh15JpFPp+IyBq02AdfUVGBBQsWoKKiAlKpFO7u7oiPj4dEIsGsWbMwf/589O3bF2vXrsW5c+cglUphb2+P1atXQ6lUdkQdmnXsjxwM6eVj6TCIiDpciwm+c+fO+M9//mPwXON++Li4OPNFZUZbvjvHBE9EdyVRzmQFgKXPhFk6BCIiixJtgu/u7659XVOrtmAkRESWIdoEDwAzJ2mWU8gtrLBwJEREHU/UCd7PqxMAIEtVZuFIiIg6nqgTvK+nHABw7HyuhSMhIup4ok7wTg4yAMDxC7koKq2CIAj48dj1Vs2WJSKyNaJO8I0XRXtp4yHsPXET/9l3Gas+O2nBqIiIOoaoEzwATJvQU/v6y/9dBgBczii2VDhERB1G9Ak+fGDDYmNq09ZVIyKyaaJP8IbcG+jeciEiIht3VyT4ldGDdd6XV9ZYKBIioo5zVyR4dxdHnfdllbctFAkRUce5KxK8i7O9zvsytuCJ6C5wVyR4AFgxYzCG9PLB42O743aNGqUVbMUTkbjdNQk+0NsFs6f01s5uvZ5jO9tyERG1xl2T4Osp6vrj2YInIrG7axM8++GJSOzuugTfyUmzidWtsmoLR0JE1L7uugTvYK9ZgCzh4FXMWJVk4WiIiNpPi3uyAkBsbCxu3rwJqVQKuVyOZcuWITQ0VKdMbW0t3nzzTRw4cAASiQQxMTF4/PHH2yVoc1KrBUilkpYLEhHZGKMSfFxcHFxdXQEAe/fuxSuvvIJvv/1Wp8zOnTtx/fp17NmzB0VFRYiKisKwYcMQGBho6JYWdV8XBS5cLwIA/Hz8BiYO7mLhiIiIzM+oLpr65A4ApaWlOsvw1tu9ezcef/xxSKVSeHp64qGHHsKPP/5ovkjN6OWnBmpf38wrtWAkRETtx+g++KVLl+LBBx/Ee++9h7i4OL3zWVlZ8Pf317738/NDdna2eaJsBxsXjgYAHDprvTESEbWFUV00APDWW28BABISErB69Wps27bN7MF4ebm0+lql0rXlQoY+092p1dd2FGuPry1YN9vEutkGoxN8vaioKLz22msoLCyEh4eH9rifnx8yMzPRr18/APotemOoVKWt2k5PqXRFXl7rZqaqiitx+Wq+3oJk1qItdbN2rJttYt2sh1QqabZh3GIXTVlZGbKysrTvk5KS4O7uDoVCoVMuIiICX331FdRqNQoKCrB3715MnDixDaF3nBc3HuLMViISnRZb8BUVFViwYAEqKioglUrh7u6O+Ph4SCQSzJo1C/Pnz0ffvn0RGRmJM2fOYMKECQCAOXPmICgoqN0r0BaxUX3wQUIKACCvqEJv1UkiIlsmEQTr2ceuo7toBEFAdNw+AJolhd9fMMrke7Q3W/uT0RSsm21i3axHm7toxEwikaBfdy8AmsXHuGcrEYnJXZ3gAWDOn/tqX8+sa83Xyy+uQH5xRUeHRERkFnd9gre3k6JHQMMm3Bu+/h0VVZqVJl/efAQvbz5iqdCIiNrkrk/wADBlRLD29alL+Zjz3q+4fLPYcgEREZkBEzyA3t089Y6dScvXvrai59BEREZjgofmYes/pg7QObbryDXt66/2p3V0SEREbcYEXyc02BPPTAwxeK6gpLKDoyEiajsm+EYevD8AS58J0zuekV9mgWiIiNqGCf4O3f0bRtT0rxsjn5FXhg1f/46aWrWlwiIiMhkTvAEO9pp/los3irTHTl3KR8w7+3Hw96ymLiMisiomryZ5N9j80hjsPXETo/v741r2Laz67KT23Ee7z6OrryuCvFu/tDERUUdgC94AiUSC8YOC4GgvQ88gBUb31132OOHAFQtFRkRkPCZ4Izz38H0Y0cdX+/7eQEUzpYmIrAMTvJGiJ/XCphc12/xV19RaOBoiopYxwZvA2VHzyGLnoXTLBkJEZAQm+FaobcWa9UREHY0JvpW4Pg0RWTsmeBNNGt4VAHC7Rn/SU0l5NdZ9dQazVu9D9W320xORZTHBm+jCNc3kp6tZJXrnNn59Fr+nqVCrFrB15x8dHRoRkQ4meBNNGh4MALhdt2xBeWUNKqpqIAgCLmc0rCF/MjXPEuEREWm1OJO1sLAQL7/8Mq5fvw4HBwd07doVK1asgKen7hrqS5YsweHDh+Hh4QEAiIiIwAsvvNA+UVuQr5ccALD2yzOYGt4DXyRdBgCsfmGYJcMiItLTYgteIpFg5syZ+Omnn7Bz504EBQVhzZo1BsvGxMQgMTERiYmJokzuAKDo5KB9XZ/cAWi39uvm56o9dvpSPh/GEpHFtJjgFQoFhgwZon0/YMAAZGZmtmtQ1szBXtbs+bl/6YeR/fwAAO9//Tui4/ZhxqokfHfwqk659//7O2asSkJ5ZU27xUpEdzeT+uDVajV27NiB8PBwg+c//vhjTJ48GbGxsUhLE+8uSH0MbPFXT+HigDF3rF0DAAkHryKvqAKL4w9jxqoknL6s2RLwcApXpySi9iERTOhDeOONN5CTk4ONGzdCKtX9bsjJyYFSqYRUKkVCQgLWr1+PvXv3QiZrvsVri27k3ELs6iS94y8+ORDhg4JQXnkbTyzdbfT9XnxyIA6dycTiZwbBwV6GyuoaCALgYCeFTMbn4ETUOkYn+Li4OFy8eBHx8fFwcHBosfyQIUPwzTffICAgwOhgVKpSqFsxS1SpdEVe3i2Tr2uL/KIKvPXvE5j7aF842smQfCEXfx59j/Z86o0inWWGjfXRknC88O4vqGo0jv7Dl8dCKpWYJW5rYomfW0dh3WyTrdVNKpXAy6vppcuNah6uXbsWKSkp2LRpU5PJPScnR/v6wIEDkEql8PHxMTFc29FZ4Yz35o1Ed393BHq76CR3AHXLDGv64us3EGksNqoPOjnpD2I6n16gk9wBYObqfWaMnIjuFi0Ok7x06RK2bNmC4OBgTJ06FQAQGBiITZs2ITIyElu3boWPjw8WL14MlUoFiUQCFxcXbN68GXZ2d/d+Is89HIrR/QMQ7OeKLFU5ln14DICmlQ4AocEemLfugM4173xxusPjJCJxMqkPvr3ZUhdNa2z4+ncoFc6YOu5e7bGq6lpcy7nVYnfO85G9cTI1D89H9mnvMDuMrfzcWoN1s022VjezdNGQecx7tJ9OcgcARwfNrlH1a83X2/TiaGxfPFb7Pj7xHH47n4vbXIueiIx0d/ehWBFnRzt8+PJYnL9eiF5dPSCRGH6oWlxWjc7uzh0cHRHZIrbgrYhUKkHvYE+d5H5fVw+dMtdzSjs6LCKyUUzwVm7V3FE67zd+c9ZCkRCRrWGCt3IyqQTTJ/TEvL/0tXQoRGRj2AdvA8YODAQABHm74EYuu2iIyDhswduQ+uReU6u/m1Rr5RZVIEtVZrb7EZH1YIK3QSVl1drX5ZW3tUMn1YKAlCuqFpcozi0sxwcJKcgrqsCS+CNYuu1Yk38ZbPr2LGasSkJBSaX5KkBEHYJdNDYk+k+h2L7rPApvVUEikeDSzSLEJ55DsK8rXnvuAexNvoEvki5j+oSeGDswEGu+OIU/0guxZdEYzF7zi979jl/I1b7+8dh1lFbchiAI+CO9EKtfGAZPNyecuKjZmeq17b9h4x1j9YnIujHB25AuPprNRL5IuoS0jIY9YdOzb6FWrdZuQPLvPam4cL0If6QXAgB2HbnW4r2PnMvWeb/og8N4ZGhX7ftxYYFtjp+IOha7aGyIh6sjAOgk93qzVu/XeZ/cqHX+3aH0Vn3e7qMNXww7D7fuHkRkOUzwNsTF2d6in59fXGHRzyci0zDB27DVLwzTW8Nm3fyRkBlYO97RXoYti8Zo37/+twfQI8AdkSO7aY9NHByEj5aEY9vLD+pc+/jY7gA4i5bI1rAP3oYZWpPGTe6ANXNGYOX/JeO5h+/DzkPpuHSzGKtmD4W9nQwLHusHZ0c7dPFxxSvTwwBAJ8kDgEwqxYaFozBv3QE8MrQrHrjPG1/tS8MnP1zAwJ7KDqkbEbUdE7yNWRk9GG98kowNCxpa7v+KGYpXth7FypmazdHdOzlgTewIAECfbl461/fv0dmoz+nkZI/3F4xCJyc71NYt4VxacdscVdBRU6vG2Ssq3BuosHgXFJHYMMHbmAClC7b+Y6zOMV9PuXYTEXOqT7h2MgmkEgnUgoDq27VwsDdun93dR6/hv/vTsHHhaMgN7F4FADHv7AcABCg7oZOjHbILyrFu/iiDZYnINOyDJ6MM7+sLADh9OV/v3KWbRVAbmFz13/1pAIBrOQ0bKNzILcWN3FL8cPQafms0NDMjrwypN4tRUn4bWaoyvPFxcosTtoioeWzBk1FG9PHFwd+zEJ94DiFBCri7aIZsJl/IxeaEFDz50L0YPyhIW76iqkb7+p0dpxD/9zFIvpCL7bvON9y07gvgTku3abY2PHtFhX7dO6O4tAoSqQRu8pY3eyeiBkzwZJSQLg3r0r+48RCWP/cAgnxcsDkhBQCwY+8lPHCfN+xkUsxff0Dv+uff1Z9J25J1X/2u8/7tmKHw8ZSbfB+iuxUTPLXKG58kI2JIF51jL2081K6f+c+tR7H1Hw/CTsaeRSJjtPg/pbCwELNmzcLEiRMxefJkzJ07FwUFBXrlKioqsHDhQowfPx4RERHYt29fuwRMljP3jjXpfzx23eR7+HnJ8eHisYiZ0kt7LPSOXauaE/POfmz//g+TP5fobtRigpdIJJg5cyZ++ukn7Ny5E0FBQVizZo1eue3bt8PFxQU///wz4uPj8eqrr6KsjMvQisnAnkp0rVsPpzFHh6ZH1UT/KVTn/VuzhkIqkSCs0Xj6xuPwP1w8Fn26eep0xQQqO+nc41CK7ro5RGRYi100CoUCQ4YM0b4fMGAAduzYoVfuhx9+wKpVqwAAwcHB6NOnD3799Vc8/PDDZgyXLG353x6AIAiIjmv4C23WpF46Wwm6ye0RFuKNKSOC4e7iCHcXB1zNLMHAEG9tGXs7GXa+G4m8vFsQBAESCTD3z30hlUjw0hMDcOpSHjZ8fRaDQ73xfGQflJRVY+GGgx1aVyJbZ1IfvFqtxo4dOxAerj/mOjMzEwEBAdr3fn5+yM42raXl5eViUvnGlEr9lqVYWHvd+vT0RsI7UxD1j+8AALOi+uLBsIYRNWOVrhjbxLX1dftuTaTO8QlKV4wf1k27AblSCSSsnoyol3fqXGfNbCHG1mLdbINJCX7lypWQy+WYNm1auwSjUpVCrTZ97LNS6Yq8vFstF7RB1lq3sfcHYN+pDACAvaBGgaoUm14cjYy8MvQIdDcq5tbU7dEx9+DrX67gZmYRHI2ccAVodq5SujtpvzDam7X+3MyBdbMeUqmk2Yax0cMR4uLicO3aNaxbtw5Sqf5l/v7+yMjI0L7PysqCr6+vieGSrZg0PFj7uj5pOjvaoUege7t+rqJu/P35a4VGX3M1qwRL4o/glzOZ7RUWkVUyKsGvXbsWKSkp2LRpExwcDE82iYiIwJdffgkASE9Px9mzZzFqFKeci5WHqyP+9vB9mPPnvi0XNqPO7k4AgPf/qztGXhAEVFY3TK767OdU/JGuGe2VX6zZbvDTHy+26i9EALhdU6tzfyJb0GKCv3TpErZs2YLc3FxMnToVkZGRmDNnDgAgMjISOTk5AIDo6GiUlJRg/PjxmD17NlasWAEXl9b3qZP1G9XfH2EhHbu65L1BCgCAp5sjcgvLtXvFRsftQ+zaX7Hg/QNQFVfifyduYs0Xp1GrVussn1xeZXySrqlV49DZLBSVVmH2ml8Qu/ZX81aGqJ1JBCta8IN98PpYN30zViWZVP7ZiBD8348XAQBTRgSjm5+bwVU1a2rV2sXPmmLsom78udkmW6ub2frgiWxVlqpc+/q7Q+lY/9/fUVOrBqBJ6teyNf+hEw9ebfFe9dcR2QIuVUCiYW8nxe0a/QScW6i/1WDy+VwM6+OLuM9OIi1Tf4/bpiSdzMCEB4JaLkhkBdiCJ9F4f/4ogzNtDS1xvK1uuQNDyf0vo+9BaFcPDOjRGYFK3T9/T6bmmSlaovbHFjzZtDdmDIZS4QQnB82v8qvPhiE9+xa6+7vj61/SsOvItSavfevfx/WO9e7miUnDg3WGgV68XoggbxfMXXcAqTeKzF4HovbCFjzZnJjJmoXKXojqgyBvF21yBzT7yXb314zFf3RMd+3xh4fqrnwJAGkZmtZ7z0Zj91/6a3+9ciFdPCB34naCZHvYgiebM7S3L/r36Axnx5Z/fSc8EIQDv2fi8Qd7YPB9PnB2lGHJlqM6ZYb18cWSaWEt3iuspxI38kpbHfed0jKLIXe0g59Xp5YLE7UCW/Bkk4xJ7gAwddy92PTiGABAV19XeHvIsW7eSHSq2yPW0V6G0f39jbqXm4sDysy48fhbn57Q7l4FANW3a6EqrkR+cQXOp+svyU1kKrbg6a7j1skBGxaONvm6Tk52KK+qQeGtKlzOKEbfezxhJ5Mip7ACAZ0Nt8LPXyvEOztOAQCWPhOm7T5a/flJbRm1IEAqkejterV0ehi6B2jK1w/P5GYnZAomeCIjyR3tIQjA3zc17FzVM9AdqTeL8e6cEaipVWNx/BEEdO4Ef28XPDQwQJvcAU2LffqEnhja2xcXrjc8rJ0Ztw9vzhyCO7317xOY/2g/vP91w7IMxk60IgKY4ImM1slZ/79L6s1iAEBJWTXe+CQZAJCRX4aM/DIk/5GjV/7fe1Lx7z2pesdf/fCY3jEA+OFY06OAiFrCv/eIjOQmN7zQHgBsTkwx+X7d/Fped/xS3RcIUWswwRMZyd2lIcFHDNYddmlotmxjS54eqHfslelheP1vD+gcG9XPD7Mm9dIrW6+1q2HS3YkJnshIjVvwocEeWBE9GAsf1x03v+zZQXrX3ddFgZ5BCrz6zCCdYzKpFF3umHn7t0dCMayP/j4KQ3r5AADyipr/IiFqjAmeyEguzg2TndzkDghUuqBfdy8sfup+7fFufm74aEk4dr4biQ8Xj8UzE0PwfFQfAMA9/m7acgMbbTq+dLpmDP7o/n7aYytmDNa+XhM7HCP7ac4VlVbpxCQIAo5fyDW4Bg8RH7ISGcmh0RaBbp0aWvP1QxnH3h+gU14qkeDBO459tCQcFVU1cHKQ6Vx/5+iYQG8XvP63B1Bwqwqebk6orK4FABTe0k3wqz8/hYs3ijC6vx+eezgUxaVV+Gp/GqaOu1fnC4nuTmzBE7VC4+RpJ5Pi/QWj8NT4e4261tnRzqi9Ybv4uGJA3br19VsVFpVW47tDV3E9R7PE8cW6tXF+PZOFgpJKvLjxEA6nZOOrfZdNqo8hn/50ETNWJWk3VSHbwxY8kQk2vTga2QXlsLfTbRu1d2vZ2VEGR3sZTqTmIi2jBAkHruq1+hd9cFj7+sDvWRjW2xf7T2fgt/O52tgbzwDOUpXB11OOyxnFUKsFhHTx0Lnf/rpN1Rd9cJjj720UEzyRCZwd7dDNz63lgmYmkUigcHHQLpAGAK9sPdrMFcDqRpOsAGBzQgpeemIAACD1RhFWfXYSjwztit1HNWPtP1oSjss3i/Gv/3dC714/H7+B8YO4Dr6tYRcNkY2oqOuHr5ddUN5EScNSrhYg8eBVCIKAVZ9plkqoT+4A8M2vVwwmdwDYsfcS1NazuycZyagEHxcXh/DwcISEhCA1VX8WHgBs2LABw4YNQ2RkJCIjI/HGG2+YNVCiu13jzcMbmxreQ/s6YkgXhPVseiP0xINXER23z+C57w+nN/v5xy/kthwkWRWjEvy4cePw2WefISAgoNlyUVFRSExMRGJiIpYvX26WAIlIo34EzUtP9IeXm+ahq6+nHOFhgdoyj43pjmkTeupcN3WccQ9/W1JcVm2W+1DHMaoPftAg/ckbRNSxpob3wBdJl+Hh6oR3YkcYLCOVSuDu4oiPloSjtOI2zl5RYVhvX/Tq6oHXPvpNp+zY+wOw75Rmj9k9yTcM3pYtqkUAAAzeSURBVG/J0wPh6ynHwg0HsWPvJaP74f/1/07g8s1i7WdsXDhKb9OUwltVyC4oR2hXD4P3yCksx4avz2L5cw/oPdQm45j1IeuuXbtw8OBBKJVKzJs3D/fff3/LFxGRUSYM7oKhvX11xuDXW/zU/XpDL12c7TGst2ZWbKC3Cz5aEo4PElK0XS3TJ4Zg2oSekEgkmDruXlzNKkF+cSW6+7vhalYJnBzs0DNIYVKMN3JLcfRcNi7XraGzr24kztx1BwBoRvLUzwGoX5XzL6Pv0dkisd6yD39DTa0as9fs5yieVpIIgvFPTsLDwxEfH4+ePXvqncvLy4NCoYC9vT0OHTqERYsWYffu3fDwMPztTES2470dJ3HqYi4+fT1C71x+UQVulVejm787Yv61F1mqsibv42AnRbWBWbefvDYBXu7OuJpZjJUfHcNbz4/Avz75DelZmlFDO9+NNF9l2uB2jeZBt72drIWS1sFsLXilsuHBzogRI+Dn54dLly5h8ODBzVylS6UqbdViSkqlK/Lybpl8nS1g3WyT2OpWWXkbhbeq8PkPf+Cph3vhnU+T8euZTJ0yy597oNnkDsBgcgeA51bsQfzfx2B+3aYnMW/v1TmfmVVssJumsroGuYUVemv6GKugpBLuLg6YtXo/AM0XSXM/txmrkgA0rMu/60g6lApnDA71MerzslRlOHouB5GjukFqxGS3lkilEnh5uTR53mwJPicnBz4+mkqeP38eGRkZ6Natm7luT0QWJK+bILVj7yUcOZejbVk3Vr8efj0/LzmyVMYP5cwvbnrGbH5xhd7etSlXVVj75RkAwOa/j4GjfdOt6tWfn8SF60WYNbkXQoIU+PiHC/D2cMa+kxk65VKvF8Kj0br/uUUV8HR1ROLBqxjTaGvHkvJquMkd8PUvVwBo1ha6c7et1LpZxn5ecrjKHZByRYW1/9HEu/NwOt6dMwKZ+WV498vTeG/uCLjXzVY2J6O6aN58803s2bMH+fn58PDwgEKhwK5duzBr1izMnz8fffv2xeLFi3Hu3DlIpVLY29tj/vz5GDNmjEnBsAWvj3WzTWKrmyAITQ6vNMTPS463Zg1FeeVt2Mmk+CLpMlKuqPSSeL/uXvg9TdXkfeofAMukEtzXRYG/T9U817uSWYI3Pz2uU3ZQiBKxf+6rE/M3v17BriO6m6Z4uDrqrelTz9FBhndjh2PuugNwldvjVrnhPXiXPTsIggBtDO/NGwn3Rs9G1IKAmY3+vbYvHtvsv9+fhnXFo2O6N3m+KWZpwb/66qt49dVX9Y5v27ZN+zouLs7k4IjINhhaO2dQiBLHL+bpHX979lD4eMgBQDty5pmJIUjPLsGKT3ST8oLH+jWb+Eb29cOe5BuoVQs4l16IjLxSODvaYd+pm3plj1/Mw+GULAzvo1l581b5bb3kDugv2NZYoLeL9oFwU8kdACqra3W2YyytuK1N8EdSsrHt+z90yu80MMeg8RdN/+6dm/ystuBSBURklLAQJU40SujTJoZAEIATqXlYN38kLt8sRk2tWpvc76Qw0AUhkUjw0ZJwbd82oGkN//Tbddwb6A5Xue7QymXbf7vzFjo+/P48egS4w9tDrl2IzVjeHs5IM3IHrTOX83Xel1U0fBncmdwBIOHAVe3r+vrWJ/fuAW7oEehuUqzGYoInIqM8Oe5enLiYhzmP9ce9/q5wkztgzl8aukQGNjODFtAk+NCuHogc2Q2rPjupsyuWt4ezdlesTk52+OtYzezc+lErTVny9EDteP9fTmse+i7ZchTPRITg0x8vAtAs7bxu3kgUlVbhpY2aoZkDenTG2IEB6O7vhmPnc9Er2ANrdpw2+Bmd3Z3g4+GMc+mF2mN3zhtY9dlJbFn0oN61b8wYjOWN5h9MNjActPH6QubGBE9ERvF0c8L2xWPh7e3W6ucL/3hS04d+57h2Hw+5NsE3fljZ3HDEXsEe2nH6A3sqkZZRgpt5pQCgTe4A8O6c4QA0XzAPD+mC7IJyzHu0n/Z8/Tr+qkbLIn+0JBzHL+TiZl4pokbdg5paNf5IL0Dvbp7aETd3mr2m4fgjQ7visQc1feqhXT1w/prmy+HPo++pq68zcurq26+7V5N1bCtODyMioxmzjn1rnL3S9IPWNbHD8WxECFbNHoo3Gu109UejFjUArIgeDF9P3e4hH085ZNKGNPf42B46yb2x+iUd5tX9VTLoPm9EjdIkZDuZFP26d9a5FwDMeCTU4L363uOpfb1oqmYFz9GNRuGsnDlE+7qlv3zagi14IrK43t08ce5qAeKeH6Z3ztPNCWMGNL8OVr1XnxmEQ2ezsON/lwAAy54JMzqG8YMC8dj4EJQUNT+0s/HIn5H9/ODsaIdN357VKRPs27CkdP1zhsbsZFL848n7kZ5VopP4zY0Jnogs7oXI3ki9UQylwrnFsuvnj8S7X57Gy0/qL4Uid7LD+AeC0K+HF1KuFOitf9MciUTS7Fj6egsf74/SitvaJRf69/DCoPu84esp167I6ejQ8n1Cu3o0uQ6PuZi0VEF74zh4faybbWLdbFNb61ZTq0ZtrWBUgjeHDpvJSkR0t7OTSWFNy9TwISsRkUgxwRMRiRQTPBGRSDHBExGJFBM8EZFIMcETEYmUVQ2TlEpbPw26LddaO9bNNrFutsmW6tZSrFY10YmIiMyHXTRERCLFBE9EJFJM8EREIsUET0QkUkzwREQixQRPRCRSTPBERCLFBE9EJFJM8EREImXzCf7q1at44oknMHHiRDzxxBNIT0+3dEhNKiwsxKxZszBx4kRMnjwZc+fORUFBAQDg9OnTmDJlCiZOnIgZM2ZApWrYZb615yxl48aNCAkJQWpqKgBx1K2qqgrLly/HhAkTMHnyZCxbtgxA879/rT3X0fbt24eoqChERkZiypQp2LNnT4sxWmvd4uLiEB4ervP71151saafYZMEGzd9+nQhISFBEARBSEhIEKZPn27hiJpWWFgoHD16VPt+1apVwj//+U+htrZWeOihh4Tk5GRBEARh06ZNwpIlSwRBEFp9zlJSUlKE6OhoYezYscLFixdFU7eVK1cKb731lqBWqwVBEIS8vDxBEJr//WvtuY6kVquFQYMGCRcvXhQEQRDOnz8vDBgwQKitrbXJuiUnJwuZmZna37+2xmut9TSWTSf4/Px8ISwsTKipqREEQRBqamqEsLAwQaVSWTgy4/z444/Cs88+K5w5c0b405/+pD2uUqmEAQMGCIIgtPqcJVRVVQl//etfhRs3bmj/g4mhbqWlpUJYWJhQWlqqc7y537/WnutoarVaGDx4sHD8+HFBEATht99+EyZMmGDzdWuc4NujLtZSz5ZY1WqSpsrKyoKPjw9kMs0utzKZDN7e3sjKyoKnp6eFo2ueWq3Gjh07EB4ejqysLPj7+2vPeXp6Qq1Wo6ioqNXnFApFh9YHANavX48pU6YgMDBQe0wMdbtx4wYUCgU2btyIY8eOoVOnTliwYAGcnJya/P0TBKFV5zr691YikWDdunWIjY2FXC5HWVkZtm7d2uz/LVupW732qIs11tMQm++Dt1UrV66EXC7HtGnTLB2KWZw6dQopKSl46qmnLB2K2dXW1uLGjRvo1asXvvnmGyxatAjz5s1DeXm5pUNrs5qaGmzZsgUffPAB9u3bh82bN2PhwoWiqBtZ2XrwpvLz80NOTg5qa2shk8lQW1uL3Nxc+Pn5WTq0ZsXFxeHatWuIj4+HVCqFn58fMjMztecLCgoglUqhUChafa6jJScnIy0tDePGjQMAZGdnIzo6GtOnT7f5uvn5+cHOzg6TJk0CAPTv3x8eHh5wcnJq8vdPEIRWneto58+fR25uLsLCwgAAYWFhcHZ2hqOjo83XrV5zeaK1dbHGehpi0y14Ly8vhIaG4vvvvwcAfP/99wgNDbWqP5HutHbtWqSkpGDTpk1wcHAAAPTp0weVlZU4fvw4AOCLL75AREREm851tJiYGBw8eBBJSUlISkqCr68vtm/fjpkzZ9p83Tw9PTFkyBAcOnQIgGb0hEqlQnBwcJO/f839blrT762vry+ys7Nx5coVAEBaWhpUKhW6du1q83Wr19p4ba2eBlmi49+cLl++LDz22GPChAkThMcee0xIS0uzdEhNSk1NFXr27ClMmDBBmDJlijBlyhQhNjZWEARBOHHihDBp0iRh/PjxwnPPPacdpdGWc5bU+CGXGOp2/fp1Ydq0acKkSZOEqKgoYf/+/YIgNP/719pzHS0xMVGYNGmSMHnyZGHy5MnCzz//3GKM1lq3lStXCqNGjRJCQ0OF4cOHC4888ki71cWafoZN4Y5OREQiZdNdNERE1DQmeCIikWKCJyISKSZ4IiKRYoInIhIpJngiIpFigiciEikmeCIikfr/+C+7RjWqO6oAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "77CgIkYjnEXk"
      },
      "source": [
        "# Manual Testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XoZhIC8U7_GV",
        "outputId": "0ddb10ca-fa42-4f1f-9f67-4424e24068b0"
      },
      "source": [
        "test_sentence = test_dataset[0]['translation']['en']\n",
        "print('Raw input text:', test_sentence)\n",
        "\n",
        "input_ids = encode_input_str(\n",
        "    text = test_sentence,\n",
        "    target_lang = 'ja',\n",
        "    tokenizer = tokenizer,\n",
        "    seq_len = model.config.max_length,\n",
        "    lang_token_map = LANG_TOKEN_MAPPING)\n",
        "input_ids = input_ids.unsqueeze(0).cuda()\n",
        "\n",
        "print('Truncated input text:', tokenizer.convert_tokens_to_string(\n",
        "    tokenizer.convert_ids_to_tokens(input_ids[0])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Raw input text: It has been confirmed that eight thoroughbred race horses at Randwick Racecourse in Sydney have been infected with equine influenza.\n",
            "Truncated input text: <jp> It has been confirmed that eight thoroughbred race horses at Randwick Racecourse</s>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "44BvtXAOHnWL",
        "outputId": "d59fc7bb-09bb-4774-d0f2-112636448354"
      },
      "source": [
        "output_tokens = model.generate(input_ids, num_beams=10, num_return_sequences=3)\n",
        "# print(output_tokens)\n",
        "for token_set in output_tokens:\n",
        "  print(tokenizer.decode(token_set, skip_special_tokens=True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ランディウィック・レースコーストで8匹のトロピカル・レース・\n",
            "ランディッキーのレース場で8匹のトロピカル・レース・ホッケーが\n",
            "ランディック・レースコーストで8匹のトロピカル・レース・ホッ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dwposz-R5ikY"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfN3QjHmPZJ8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "cellView": "form",
        "outputId": "9e113642-c0f3-43ce-9480-6557c8636603"
      },
      "source": [
        "#@title Slick Blue Translate\n",
        "input_text = 'A surfboarder ran into a shark' #@param {type:\"string\"}\n",
        "output_language = 'ja' #@param [\"en\", \"ja\", \"zh\"]\n",
        "\n",
        "input_ids = encode_input_str(\n",
        "    text = input_text,\n",
        "    target_lang = output_language,\n",
        "    tokenizer = tokenizer,\n",
        "    seq_len = model.config.max_length,\n",
        "    lang_token_map = LANG_TOKEN_MAPPING)\n",
        "input_ids = input_ids.unsqueeze(0).cuda()\n",
        "\n",
        "output_tokens = model.generate(input_ids, num_beams=20, length_penalty=0.2)\n",
        "print(input_text + '  ->  ' + \\\n",
        "      tokenizer.decode(output_tokens[0], skip_special_tokens=True))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "A surfboarder ran into a shark  ->  水泳選手はサメの群れの中にいた。\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vIMaG0pU5lLA"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}