Task 7_Hannan_Kai.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[{"file_id":"1Kj8JGNLklKYvOxiDhuqeP5Jg7jOx6ZKx","timestamp":1669467807102}],"collapsed_sections":["i6HIA1eSDTy5","lTLrEiSdrx8l","uUMJzbCRr_w6","UKubGKZasiSj","DXVoj6dps0tG","efYqw75eDyKU","bQu78hzIUQKW","KyYKoPtFGpfq","i-z86QvL-bt3"],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU","gpuClass":"standard"},"cells":[{"cell_type":"markdown","source":["# Assignment 7: Attention-based Neural Machine Translation - Kai Ponel & Hannan Mahadik\n","\n"],"metadata":{"id":"u0U1NkXx_7_b"}},{"cell_type":"markdown","source":["# Imports"],"metadata":{"id":"hDy1NIbADRZU"}},{"cell_type":"code","execution_count":137,"metadata":{"id":"fPMRqkKj_2lW","executionInfo":{"status":"ok","timestamp":1669568799305,"user_tz":-60,"elapsed":6316,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"colab":{"base_uri":"https://localhost:8080/"},"outputId":"7db5c5bb-6be8-4be1-fea1-6c218d7bfa4d"},"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: tensorflow-text>=2.10 in /usr/local/lib/python3.7/dist-packages (2.11.0)\n","Requirement already satisfied: tensorflow<2.12,>=2.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text>=2.10) (2.11.0)\n","Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-text>=2.10) (0.12.0)\n","Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.21.6)\n","Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.27.0)\n","Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.14.1)\n","Requirement already satisfied: flatbuffers>=2.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (22.11.23)\n","Requirement already satisfied: tensorboard<2.12,>=2.11 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.11.0)\n","Requirement already satisfied: tensorflow-estimator<2.12,>=2.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.11.0)\n","Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.4.0)\n","Requirement already satisfied: keras<2.12,>=2.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.11.0)\n","Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.15.0)\n","Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.1.0)\n","Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (57.4.0)\n","Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (14.0.6)\n","Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.3.0)\n","Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.50.0)\n","Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.2.0)\n","Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.1.0)\n","Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (4.1.1)\n","Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.3.0)\n","Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (21.3)\n","Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.19.6)\n","Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.6.3)\n","Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.38.4)\n","Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.5.2)\n","Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.23.0)\n","Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.0.1)\n","Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.14.1)\n","Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.8.1)\n","Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.4.6)\n","Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.6.1)\n","Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.4.1)\n","Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.2.8)\n","Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (5.2.0)\n","Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (4.9)\n","Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.3.1)\n","Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (4.13.0)\n","Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.10.0)\n","Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (0.4.8)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (1.24.3)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2.10)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (2022.9.24)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.0.4)\n","Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.12,>=2.11->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.2.2)\n","Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tensorflow<2.12,>=2.11.0->tensorflow-text>=2.10) (3.0.9)\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Requirement already satisfied: einops in /usr/local/lib/python3.7/dist-packages (0.6.0)\n"]}],"source":["!pip install \"tensorflow-text>=2.10\"\n","!pip install einops"]},{"cell_type":"code","source":["import numpy as np\n","\n","import typing\n","from typing import Any, Tuple\n","\n","import einops\n","import matplotlib.pyplot as plt\n","import matplotlib.ticker as ticker\n","\n","import tensorflow as tf\n","import tensorflow_text as tf_text\n","\n","import pathlib"],"metadata":{"id":"Zr9T8V4cBYOO","executionInfo":{"status":"ok","timestamp":1669568799305,"user_tz":-60,"elapsed":15,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":138,"outputs":[]},{"cell_type":"code","source":["import os\n","from google.colab import drive\n","drive.mount('/content/drive', force_remount=False)\n","\n","project_dir = '/content/drive/MyDrive/IDL_Hannan_Kai/Task 7'\n","# project_dir = '/content/drive/MyDrive/ColabNotebooks/IDL_Hannan_Kai/Task 7'"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ojVspIceBjTs","executionInfo":{"status":"ok","timestamp":1669568801899,"user_tz":-60,"elapsed":2603,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"06aa807f-a5ec-4be9-8d91-0c6a88217449"},"execution_count":139,"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"]}]},{"cell_type":"markdown","source":["# Shape checker"],"metadata":{"id":"eIqHJGRCuuNg"}},{"cell_type":"code","source":["#@title\n","class ShapeChecker():\n","  def __init__(self):\n","    # Keep a cache of every axis-name seen\n","    self.shapes = {}\n","\n","  def __call__(self, tensor, names, broadcast=False):\n","    if not tf.executing_eagerly():\n","      return\n","\n","    parsed = einops.parse_shape(tensor, names)\n","\n","    for name, new_dim in parsed.items():\n","      old_dim = self.shapes.get(name, None)\n","      \n","      if (broadcast and new_dim == 1):\n","        continue\n","\n","      if old_dim is None:\n","        # If the axis name is new, add its length to the cache.\n","        self.shapes[name] = new_dim\n","        continue\n","\n","      if new_dim != old_dim:\n","        raise ValueError(f\"Shape mismatch for dimension: '{name}'\\n\"\n","                         f\"    found: {new_dim}\\n\"\n","                         f\"    expected: {old_dim}\\n\")"],"metadata":{"id":"pMFOKjkTu8EY","executionInfo":{"status":"ok","timestamp":1669568801900,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":140,"outputs":[]},{"cell_type":"markdown","source":["# Data Stuff"],"metadata":{"id":"i6HIA1eSDTy5"}},{"cell_type":"markdown","source":["## Importing dataset"],"metadata":{"id":"z4oZnlEZDcBu"}},{"cell_type":"code","source":["path_to_file = pathlib.Path(os.path.join(project_dir, \"deu.txt\"))"],"metadata":{"id":"Z-YyJcR1CHja","executionInfo":{"status":"ok","timestamp":1669568801901,"user_tz":-60,"elapsed":13,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":141,"outputs":[]},{"cell_type":"code","source":["def load_data(path):\n","  text = path.read_text(encoding='utf-8')\n","\n","  lines = text.splitlines()\n","  pairs = [line.split('\\t')[:-1] for line in lines]\n","\n","  context = np.array([context for target, context in pairs])\n","  target = np.array([target for target, context in pairs])\n","\n","  return target, context"],"metadata":{"id":"sZBIkYuBCEOX","executionInfo":{"status":"ok","timestamp":1669568801901,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":142,"outputs":[]},{"cell_type":"code","source":["target_raw, context_raw = load_data(path_to_file)\n","print(context_raw[-1])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"B4uCjqjiCYjA","executionInfo":{"status":"ok","timestamp":1669568803922,"user_tz":-60,"elapsed":2030,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"1a24104d-4489-4054-e97c-48f246447fd9"},"execution_count":143,"outputs":[{"output_type":"stream","name":"stdout","text":["Ohne Zweifel findet sich auf dieser Welt zu jedem Mann genau die richtige Ehefrau und umgekehrt; wenn man jedoch in Betracht zieht, dass ein Mensch nur Gelegenheit hat, mit ein paar hundert anderen bekannt zu sein, von denen ihm nur ein Dutzend oder weniger nahesteht, darunter höchstens ein oder zwei Freunde, dann erahnt man eingedenk der Millionen Einwohner dieser Welt leicht, dass seit Erschaffung ebenderselben wohl noch nie der richtige Mann der richtigen Frau begegnet ist.\n"]}]},{"cell_type":"code","source":["print(target_raw[-1])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6VTuxTk1CaBd","executionInfo":{"status":"ok","timestamp":1669568803923,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"8a4f281c-b0da-420f-e3f6-c9457eb174aa"},"execution_count":144,"outputs":[{"output_type":"stream","name":"stdout","text":["Doubtless there exists in this world precisely the right woman for any given man to marry and vice versa; but when you consider that a human being has the opportunity of being acquainted with only a few hundred people, and out of the few hundred that there are but a dozen or less whom he knows intimately, and out of the dozen, one or two friends at most, it will easily be seen, when we remember the number of millions who inhabit this world, that probably, since the earth was created, the right man has never yet met the right woman.\n"]}]},{"cell_type":"markdown","source":["## Creating Tf.Data Dataset\n"],"metadata":{"id":"jrOUbXkjDXAf"}},{"cell_type":"code","source":["BUFFER_SIZE = len(context_raw)\n","BATCH_SIZE = 64\n","\n","is_train = np.random.uniform(size=(len(target_raw),)) < 0.8\n","\n","train_raw = (\n","    tf.data.Dataset\n","    .from_tensor_slices((context_raw[is_train], target_raw[is_train]))\n","    .shuffle(BUFFER_SIZE)\n","    .batch(BATCH_SIZE))\n","val_raw = (\n","    tf.data.Dataset\n","    .from_tensor_slices((context_raw[~is_train], target_raw[~is_train]))\n","    .shuffle(BUFFER_SIZE)\n","    .batch(BATCH_SIZE))"],"metadata":{"id":"_t5Ns7a1DhF3","executionInfo":{"status":"ok","timestamp":1669568805268,"user_tz":-60,"elapsed":1351,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":145,"outputs":[]},{"cell_type":"code","source":["for example_context_strings, example_target_strings in train_raw.take(1):\n","  print(example_context_strings[:5])\n","  print()\n","  print(example_target_strings[:5])\n","  break"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"95W6FufRDkuV","executionInfo":{"status":"ok","timestamp":1669568805911,"user_tz":-60,"elapsed":649,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"69966b28-9001-4420-c28c-a2a2499536f1"},"execution_count":146,"outputs":[{"output_type":"stream","name":"stdout","text":["tf.Tensor(\n","[b'Darum geht es hier nicht.' b'Wir wissen jetzt, dass das nicht stimmt.'\n"," b'Der Himmel ist fast jeden Tag klar.'\n"," b'Tom und Maria spielten Brettspiele.'\n"," b'Seine Technik war einzigartig und absolut unglaublich.'], shape=(5,), dtype=string)\n","\n","tf.Tensor(\n","[b\"This isn't about that.\" b\"We now know that's not true.\"\n"," b'The sky is clear almost every day.' b'Tom and Mary played board games.'\n"," b'His technique was unique and absolutely amazing.'], shape=(5,), dtype=string)\n"]}]},{"cell_type":"markdown","source":["## Text Preprocessing"],"metadata":{"id":"lTLrEiSdrx8l"}},{"cell_type":"markdown","source":["### Standardization"],"metadata":{"id":"uUMJzbCRr_w6"}},{"cell_type":"code","source":["example_text = tf.constant('Tom klang glücklich.')\n","\n","print(example_text.numpy())\n","print(tf_text.normalize_utf8(example_text, 'NFKD').numpy())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bx2AraH0Orib","executionInfo":{"status":"ok","timestamp":1669568805912,"user_tz":-60,"elapsed":16,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"e60fa299-353f-41dd-c1a7-57ffb4d61bbc"},"execution_count":147,"outputs":[{"output_type":"stream","name":"stdout","text":["b'Tom klang gl\\xc3\\xbccklich.'\n","b'Tom klang glu\\xcc\\x88cklich.'\n"]}]},{"cell_type":"code","source":["def tf_lower_and_split_punct(text):\n","  # Split accented characters.\n","  text = tf_text.normalize_utf8(text, 'NFKD')\n","  text = tf.strings.lower(text)\n","  # Keep space, a to z, and select punctuation.\n","  text = tf.strings.regex_replace(text, '[^ a-z.?!,¿]', '')\n","  # Add spaces around punctuation.\n","  text = tf.strings.regex_replace(text, '[.?!,¿]', r' \\0 ')\n","  # Strip whitespace.\n","  text = tf.strings.strip(text)\n","\n","  text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n","  return text"],"metadata":{"id":"46nF2NW7q1vm","executionInfo":{"status":"ok","timestamp":1669568805912,"user_tz":-60,"elapsed":10,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":148,"outputs":[]},{"cell_type":"code","source":["print(example_text.numpy().decode())\n","print(tf_lower_and_split_punct(example_text).numpy().decode())"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dMBR97kgO721","executionInfo":{"status":"ok","timestamp":1669568805913,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"a8adb5ce-96d4-4b5e-9a68-8e201d87853d"},"execution_count":149,"outputs":[{"output_type":"stream","name":"stdout","text":["Tom klang glücklich.\n","[START] tom klang glucklich . [END]\n"]}]},{"cell_type":"markdown","source":["### Vectorization"],"metadata":{"id":"Wmcy-p_gUcmh"}},{"cell_type":"code","source":["max_vocab_size = 5000\n","\n","context_text_processor = tf.keras.layers.TextVectorization(\n","    standardize=tf_lower_and_split_punct,\n","    max_tokens=max_vocab_size,\n","    ragged=True)\n","\n","context_text_processor.adapt(train_raw.map(lambda context, target: context))\n","\n","print(context_text_processor.get_vocabulary()[:10])\n","\n","target_text_processor = tf.keras.layers.TextVectorization(\n","    standardize=tf_lower_and_split_punct,\n","    max_tokens=max_vocab_size,\n","    ragged=True)\n","\n","target_text_processor.adapt(train_raw.map(lambda context, target: target))\n","print(target_text_processor.get_vocabulary()[:10])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"WrrCyO8EUzuH","executionInfo":{"status":"ok","timestamp":1669568823557,"user_tz":-60,"elapsed":17650,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"3719b922-cf7a-4a51-8d51-2ce459f1a60d"},"execution_count":150,"outputs":[{"output_type":"stream","name":"stdout","text":["['', '[UNK]', '[START]', '[END]', '.', ',', 'ich', 'tom', '?', 'nicht']\n","['', '[UNK]', '[START]', '[END]', '.', 'tom', 'to', 'you', 'the', 'i']\n"]}]},{"cell_type":"code","source":["example_tokens = context_text_processor(example_context_strings)\n","example_tokens[:3, :]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6V2UeeLAsbEZ","executionInfo":{"status":"ok","timestamp":1669568823558,"user_tz":-60,"elapsed":27,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"bf693764-1d4a-4a02-d67e-4c9ddebfa354"},"execution_count":151,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tf.RaggedTensor [[2, 600, 139, 15, 56, 9, 4, 3], [2, 23, 148, 92, 5, 22, 11, 9, 502, 4, 3],\n"," [2, 18, 936, 10, 218, 224, 161, 814, 4, 3]]>"]},"metadata":{},"execution_count":151}]},{"cell_type":"code","source":["context_vocab = np.array(context_text_processor.get_vocabulary())\n","tokens = context_vocab[example_tokens[0].numpy()]\n","' '.join(tokens)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"75tzHspOsc6l","executionInfo":{"status":"ok","timestamp":1669568823558,"user_tz":-60,"elapsed":18,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"8011a41a-db45-4240-f785-562b5e292331"},"execution_count":152,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'[START] darum geht es hier nicht . [END]'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":152}]},{"cell_type":"markdown","source":["## Process dataset"],"metadata":{"id":"UKubGKZasiSj"}},{"cell_type":"code","source":["def process_text(context, target):\n","  context = context_text_processor(context).to_tensor()\n","  target = target_text_processor(target)\n","  targ_in = target[:,:-1].to_tensor()\n","  targ_out = target[:,1:].to_tensor()\n","  return (context, targ_in), targ_out\n","\n","\n","train_ds = train_raw.map(process_text, tf.data.AUTOTUNE)\n","val_ds = val_raw.map(process_text, tf.data.AUTOTUNE)"],"metadata":{"id":"ClJuwXuasm4S","executionInfo":{"status":"ok","timestamp":1669568824087,"user_tz":-60,"elapsed":541,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":153,"outputs":[]},{"cell_type":"code","source":["for (ex_context_tok, ex_tar_in), ex_tar_out in train_ds.take(1):\n","  print(ex_context_tok[0, :10].numpy()) \n","  print()\n","  print(ex_tar_in[0, :10].numpy()) \n","  print(ex_tar_out[0, :10].numpy()) "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EDUMfbSZsoQY","executionInfo":{"status":"ok","timestamp":1669568824519,"user_tz":-60,"elapsed":439,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"4c19bccc-ff32-4f16-ca68-dfc61ef1ddd9"},"execution_count":154,"outputs":[{"output_type":"stream","name":"stdout","text":["[  2  23 851  11   9  71   4   3   0   0]\n","\n","[  2  49  41 820   6  15  13   4   0   0]\n","[ 49  41 820   6  15  13   4   3   0   0]\n"]}]},{"cell_type":"markdown","source":["# Encoder"],"metadata":{"id":"DXVoj6dps0tG"}},{"cell_type":"code","source":["class Encoder(tf.keras.layers.Layer):\n","  def __init__(self, text_processor, units):\n","    super(Encoder, self).__init__()\n","    self.text_processor = text_processor\n","    self.vocab_size = text_processor.vocabulary_size()\n","    self.units = units\n","    \n","    # The embedding layer converts tokens to vectors\n","    self.embedding = tf.keras.layers.Embedding(self.vocab_size, units,\n","                                               mask_zero=True)\n","\n","    # The RNN layer processes those vectors sequentially.\n","    self.rnn = tf.keras.layers.Bidirectional(\n","        merge_mode='sum',\n","        layer=tf.keras.layers.GRU(units,\n","                            # Return the sequence and state\n","                            return_sequences=True,\n","                            recurrent_initializer='glorot_uniform'))\n","\n","  def call(self, x):\n","    # shape_checker = ShapeChecker()\n","    # shape_checker(x, 'batch s')\n","\n","    # 2. The embedding layer looks up the embedding vector for each token.\n","    x = self.embedding(x)\n","    # shape_checker(x, 'batch s units')\n","\n","    # 3. The GRU processes the sequence of embeddings.\n","    x = self.rnn(x)\n","    # shape_checker(x, 'batch s units')\n","\n","    # 4. Returns the new sequence of embeddings.\n","    return x\n","\n","  def convert_input(self, texts):\n","    texts = tf.convert_to_tensor(texts)\n","    if len(texts.shape) == 0:\n","      texts = tf.convert_to_tensor(texts)[tf.newaxis]\n","    context = self.text_processor(texts).to_tensor()\n","    context = self(context)\n","    return context"],"metadata":{"id":"bUcDH1OOs6mq","executionInfo":{"status":"ok","timestamp":1669568824520,"user_tz":-60,"elapsed":16,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":155,"outputs":[]},{"cell_type":"markdown","source":["# Attention Layers"],"metadata":{"id":"GIw3dY77vDDF"}},{"cell_type":"code","source":["class CrossAttention(tf.keras.layers.Layer):\n","  def __init__(self, units, **kwargs):\n","    super().__init__()\n","    self.mha = tf.keras.layers.MultiHeadAttention(key_dim=units, num_heads=1, **kwargs)\n","    self.layernorm = tf.keras.layers.LayerNormalization()\n","    self.add = tf.keras.layers.Add()\n","\n","  def call(self, input, return_attention_scores=True):\n","    x = input[0]\n","    context = input[1]\n","\n","    shape_checker = ShapeChecker()\n"," \n","    shape_checker(x, 'batch t units')\n","    shape_checker(context, 'batch s units')\n","\n","    attn_output, attn_scores = self.mha(\n","        query=x,\n","        value=context,\n","        return_attention_scores=True)\n","    \n","    shape_checker(x, 'batch t units')\n","    shape_checker(attn_scores, 'batch heads t s')\n","    \n","    # Cache the attention scores for plotting later.\n","    attn_scores = tf.reduce_mean(attn_scores, axis=1)\n","    # shape_checker(attn_scores, 'batch t s')\n","    self.last_attention_weights = attn_scores\n","\n","    x = self.add([x, attn_output])\n","    x = self.layernorm(x)\n","    if not return_attention_scores:\n","      return x\n","    return (x, self.last_attention_weights) # return (x, last_attention_weights)"],"metadata":{"id":"p92BUOaUvFyO","executionInfo":{"status":"ok","timestamp":1669568824520,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":156,"outputs":[]},{"cell_type":"markdown","source":["# Decoder"],"metadata":{"id":"BqJt1EkYtE4E"}},{"cell_type":"code","source":["class Decoder(tf.keras.layers.Layer):\n","  # @ #classmethod\n","  # def add_method(cls, fun):\n","  #   setattr(cls, fun.__name__, fun)\n","  #   return fun\n","\n","  def __init__(self, text_processor, units, attention):\n","    super(Decoder, self).__init__()\n","    self.text_processor = text_processor\n","    self.vocab_size = text_processor.vocabulary_size()\n","    self.word_to_id = tf.keras.layers.StringLookup(\n","        vocabulary=text_processor.get_vocabulary(),\n","        mask_token='', oov_token='[UNK]')\n","    self.id_to_word = tf.keras.layers.StringLookup(\n","        vocabulary=text_processor.get_vocabulary(),\n","        mask_token='', oov_token='[UNK]',\n","        invert=True)\n","    self.start_token = self.word_to_id('[START]')\n","    self.end_token = self.word_to_id('[END]')\n","\n","    self.last_attention_weights = None\n","\n","    self.units = units\n","\n","\n","    # 1. The embedding layer converts token IDs to vectors\n","    self.embedding = tf.keras.layers.Embedding(self.vocab_size,\n","                                               units, mask_zero=True)\n","\n","    # 2. The RNN keeps track of what's been generated so far.\n","    self.rnn = tf.keras.layers.GRU(units,\n","                                   return_sequences=True,\n","                                   return_state=True,\n","                                   recurrent_initializer='glorot_uniform')\n","\n","    # 3. The RNN output will be the query for the attention layer.\n","    self.attention = attention(units)\n","\n","    # 4. This fully connected layer produces the logits for each\n","    # output token.\n","    self.output_layer = tf.keras.layers.Dense(self.vocab_size)\n","\n","\n","#------------------------------------------------------------------------------\n","  # @Decoder.add_method\n","  def get_initial_state(self, context):\n","    batch_size = tf.shape(context)[0]\n","    start_tokens = tf.fill([batch_size, 1], self.start_token)\n","    done = tf.zeros([batch_size, 1], dtype=tf.bool)\n","    embedded = self.embedding(start_tokens)\n","\n","    return start_tokens, done, self.rnn.get_initial_state(embedded)[0]\n","\n","\n","#------------------------------------------------------------------------------\n","  # @Decoder.add_method\n","  def tokens_to_text(self, tokens):\n","    words = self.id_to_word(tokens)\n","    result = tf.strings.reduce_join(words, axis=-1, separator=' ')\n","    result = tf.strings.regex_replace(result, '^ *\\[START\\] *', '')\n","    result = tf.strings.regex_replace(result, ' *\\[END\\] *$', '')\n","\n","    return result\n","\n","#------------------------------------------------------------------------------\n","# @Decoder.add_method\n","  def get_next_token(self, context, next_token, done, state, temperature = 0.0):\n","    logits, state = self(\n","      context, next_token,\n","      state = state,\n","      return_state=True) \n","    \n","    # if temperature == 0.0:\n","    next_token = tf.argmax(logits, axis=-1)\n","    # else:\n","      # logits = logits[:, -1, :]/temperature\n","      # next_token = tf.random.categorical(logits, num_samples=1)\n","\n","    # If a sequence produces an `end_token`, set it `done`\n","    done = done | (next_token == self.end_token)\n","    # Once a sequence is done it only produces 0-padding.\n","    next_token = tf.where(done, tf.constant(0, dtype=tf.int64), next_token)\n","    \n","    return next_token, done, state\n","\n","#------------------------------------------------------------------------------ \n","  def call(self, context, x, state=None, return_state=False):  \n","    # shape_checker = ShapeChecker()\n","    # shape_checker(x, 'batch t')\n","    # shape_checker(context, 'batch s units')\n","\n","    # 1. Lookup the embeddings\n","    x = self.embedding(x)\n","    # shape_checker(x, 'batch t units')\n","\n","    # 2. Process the target sequence.\n","    x, state = self.rnn(x, initial_state=state)\n","    # shape_checker(x, 'batch t units')\n","\n","    # 3. Use the RNN output as the query for the attention over the context.\n","    (x, self.last_attention_weights) = self.attention([x, context], return_attention_scores=True) \n","    # print(x)\n","    # print(self.last_attention_weights)\n","    # shape_checker(x, 'batch t units')\n","    # shape_checker(self.last_attention_weights, 'batch t s')\n","\n","    # Step 4. Generate logit predictions for the next token.\n","    logits = self.output_layer(x)\n","    # shape_checker(logits, 'batch t target_vocab_size')\n","\n","    if return_state:\n","      return logits, state\n","    else:\n","      return logits"],"metadata":{"id":"NONBI74IvoB6","executionInfo":{"status":"ok","timestamp":1669568824521,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":157,"outputs":[]},{"cell_type":"markdown","source":["# Model"],"metadata":{"id":"tRBk5Oz8wApR"}},{"cell_type":"code","source":["class Translator(tf.keras.Model):\n","  # @ #classmethod\n","  # def add_method(cls, fun):\n","  #   setattr(cls, fun.__name__, fun)\n","  #   return fun\n","\n","  def __init__(self, units,\n","               context_text_processor,\n","               target_text_processor,\n","               attention_mechanism):\n","    super().__init__()\n","    # Build the encoder and decoder\n","    encoder = Encoder(context_text_processor, units)\n","    # encoder = Encoder(context_text_processor2, units)\n","    decoder = Decoder(target_text_processor, units, attention=attention_mechanism)\n","    # decoder = Decoder(target_text_processor2, units, attention=attention_mechanism)\n","\n","    self.encoder = encoder\n","    self.decoder = decoder\n","\n","#-----------------------------------------------------------------------------\n","\n","  def translate(self,texts, *, max_length=50, temperature=0.0):\n","    # Process the input texts\n","    context = self.encoder.convert_input(texts)\n","    batch_size = tf.shape(texts)[0]\n","\n","    # Setup the loop inputs\n","    tokens = []\n","    attention_weights = []\n","    next_token, done, state = self.decoder.get_initial_state(context)\n","\n","    for _ in range(max_length):\n","      # Generate the next token\n","      next_token, done, state = self.decoder.get_next_token(\n","          context, next_token, done,  state, temperature)\n","          \n","      # Collect the generated tokens\n","      tokens.append(next_token)\n","      attention_weights.append(self.decoder.last_attention_weights)\n","      \n","      if tf.executing_eagerly() and tf.reduce_all(done):\n","        break\n","\n","    # Stack the lists of tokens and attention weights.\n","    tokens = tf.concat(tokens, axis=-1)   # t*[(batch 1)] -> (batch, t)\n","    self.last_attention_weights = tf.concat(attention_weights, axis=1)  # t*[(batch 1 s)] -> (batch, t s)\n","\n","    result = self.decoder.tokens_to_text(tokens)\n","    return result\n","\n","#-----------------------------------------------------------------------------\n","\n","  def plot_attention(self, text, **kwargs):\n","    assert isinstance(text, str)\n","    output = self.translate([text], **kwargs)\n","    output = output[0].numpy().decode()\n","\n","    attention = self.last_attention_weights[0]\n","\n","    context = tf_lower_and_split_punct(text)\n","    context = context.numpy().decode().split()\n","\n","    # output = tf_lower_and_split_punct(output)\n","    output = tf_lower_and_split_punct(output)\n","    output = output.numpy().decode().split()[1:]\n","\n","    fig = plt.figure(figsize=(10, 10))\n","    ax = fig.add_subplot(1, 1, 1)\n","\n","    ax.matshow(attention, cmap='viridis', vmin=0.0)\n","\n","    fontdict = {'fontsize': 14}\n","\n","    ax.set_xticklabels([''] + context, fontdict=fontdict, rotation=90)\n","    ax.set_yticklabels([''] + output, fontdict=fontdict)\n","\n","    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n","    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n","\n","    ax.set_xlabel('Input text')\n","    ax.set_ylabel('Output text')\n","\n","#-----------------------------------------------------------------------------\n","\n","  def call(self, inputs):\n","    context, x = inputs\n","    context = self.encoder(context)\n","    logits = self.decoder(context, x)\n","\n","    #TODO(b/250038731): remove this\n","    try:\n","      # Delete the keras mask, so keras doesn't scale the loss+accuracy. \n","      del logits._keras_mask\n","    except AttributeError:\n","      pass\n","\n","    return logits"],"metadata":{"id":"i0YDzRadwEGO","executionInfo":{"status":"ok","timestamp":1669568824521,"user_tz":-60,"elapsed":13,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":158,"outputs":[]},{"cell_type":"markdown","source":["# Loss and Accuracy"],"metadata":{"id":"tJv_dmu2rcQp"}},{"cell_type":"code","source":["def masked_loss(y_true, y_pred):\n","    # Calculate the loss for each item in the batch.\n","    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(\n","        from_logits=True, reduction='none')\n","    loss = loss_fn(y_true, y_pred)\n","\n","    # Mask off the losses on padding.\n","    mask = tf.cast(y_true != 0, loss.dtype)\n","    loss *= mask\n","\n","    # Return the total.\n","    return tf.reduce_sum(loss)/tf.reduce_sum(mask)"],"metadata":{"id":"6fRcw2xOwVb6","executionInfo":{"status":"ok","timestamp":1669568824521,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":159,"outputs":[]},{"cell_type":"code","source":["def masked_acc(y_true, y_pred):\n","    # Calculate the loss for each item in the batch.\n","    y_pred = tf.argmax(y_pred, axis=-1)\n","    y_pred = tf.cast(y_pred, y_true.dtype)\n","    \n","    match = tf.cast(y_true == y_pred, tf.float32)\n","    mask = tf.cast(y_true != 0, tf.float32)\n","    \n","    return tf.reduce_sum(match)/tf.reduce_sum(mask)"],"metadata":{"id":"IENeuMC5wWpw","executionInfo":{"status":"ok","timestamp":1669568824521,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":160,"outputs":[]},{"cell_type":"markdown","source":["# Training"],"metadata":{"id":"P7q0JVrGwSiu"}},{"cell_type":"markdown","source":["## Step-by-step (Encoding -> AttentionLayer -> Decode)"],"metadata":{"id":"8H-HSqStfftt"}},{"cell_type":"markdown","source":["### Encode:"],"metadata":{"id":"iBJcApGhrkxF"}},{"cell_type":"code","source":["UNITS = 1024\n","# Encode the input sequence.\n","encoder = Encoder(context_text_processor, UNITS)\n","ex_context = encoder(ex_context_tok)\n","\n","print(f'Context tokens, shape (batch, s): {ex_context_tok.shape}')\n","print(f'Encoder output, shape (batch, s, units): {ex_context.shape}')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FIY7UtYutDcY","executionInfo":{"status":"ok","timestamp":1669568824522,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"92964b6a-f3f5-4c02-d3e2-aa921d57e618"},"execution_count":161,"outputs":[{"output_type":"stream","name":"stdout","text":["Context tokens, shape (batch, s): (64, 20)\n","Encoder output, shape (batch, s, units): (64, 20, 1024)\n"]}]},{"cell_type":"markdown","source":["### Attention Layer:"],"metadata":{"id":"tEVmXgrWrz3t"}},{"cell_type":"code","source":["initializer = tf.keras.initializers.HeNormal()\n","regularizer = tf.keras.regularizers.L2(1e-4)"],"metadata":{"id":"knv1QoUGr7vW","executionInfo":{"status":"ok","timestamp":1669568824523,"user_tz":-60,"elapsed":9,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":162,"outputs":[]},{"cell_type":"code","source":["# attention_layer = tf.keras.layers.AdditiveAttention(UNITS)\n","\n","attention_layer = CrossAttention(UNITS)\n","\n","# Attend to the encoded tokens\n","embed = tf.keras.layers.Embedding(target_text_processor.vocabulary_size(),\n","                                  output_dim=UNITS, mask_zero=True, embeddings_regularizer = regularizer, embeddings_initializer=initializer)\n","ex_tar_embed = embed(ex_tar_in)\n","\n","(x, last_attention_weights) = attention_layer([ex_tar_embed, ex_context], return_attention_scores=True)\n","\n","print(f'Context sequence, shape (batch, s, units): {ex_context.shape}')\n","print(f'Target sequence, shape (batch, t, units): {ex_tar_embed.shape}')\n","print(f'Attention result, shape (batch, t, units): {x.shape}')\n","print(f'Attention weights, shape (batch, t, s):    {last_attention_weights.shape}')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HDsRVmgKvJq8","executionInfo":{"status":"ok","timestamp":1669568824524,"user_tz":-60,"elapsed":9,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"7f200024-5cf3-4692-e568-0898955a3963"},"execution_count":163,"outputs":[{"output_type":"stream","name":"stdout","text":["Context sequence, shape (batch, s, units): (64, 20, 1024)\n","Target sequence, shape (batch, t, units): (64, 18, 1024)\n","Attention result, shape (batch, t, units): (64, 18, 1024)\n","Attention weights, shape (batch, t, s):    (64, 18, 20)\n"]}]},{"cell_type":"markdown","source":["### Decode:"],"metadata":{"id":"rE1V6GR0sODl"}},{"cell_type":"code","source":["# decoder = Decoder(target_text_processor, UNITS, attention=tf.keras.layers.AdditiveAttention)\n","decoder = Decoder(target_text_processor, UNITS, attention=CrossAttention)\n","\n","logits = decoder(ex_context, ex_tar_in)\n","\n","print(f'encoder output shape: (batch, s, units) {ex_context.shape}')\n","print(f'input target tokens shape: (batch, t) {ex_tar_in.shape}')\n","print(f'logits shape shape: (batch, target_vocabulary_size) {logits.shape}')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"TuRZbV8UvyUH","executionInfo":{"status":"ok","timestamp":1669568825519,"user_tz":-60,"elapsed":565,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"969780d8-8fac-445a-c3ef-bbd0105905a5"},"execution_count":164,"outputs":[{"output_type":"stream","name":"stdout","text":["encoder output shape: (batch, s, units) (64, 20, 1024)\n","input target tokens shape: (batch, t) (64, 18)\n","logits shape shape: (batch, target_vocabulary_size) (64, 18, 5000)\n"]}]},{"cell_type":"markdown","source":["## Different Model specifications:"],"metadata":{"id":"W_j-p1B2tA5L"}},{"cell_type":"markdown","source":["### Using CrossAttention (as per Tutorial)"],"metadata":{"id":"MHg6Dni3d8IE"}},{"cell_type":"code","source":["model = Translator(UNITS, context_text_processor, target_text_processor, attention_mechanism=CrossAttention)\n","\n","logits = model((ex_context_tok, ex_tar_in))\n","\n","print(f'Context tokens, shape: (batch, s, units) {ex_context_tok.shape}')\n","print(f'Target tokens, shape: (batch, t) {ex_tar_in.shape}')\n","print(f'logits, shape: (batch, t, target_vocabulary_size) {logits.shape}')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t7TDaanZwQCr","executionInfo":{"status":"ok","timestamp":1669568826041,"user_tz":-60,"elapsed":529,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"2a819752-d64b-4061-f597-041a79f7f7d0"},"execution_count":165,"outputs":[{"output_type":"stream","name":"stdout","text":["Context tokens, shape: (batch, s, units) (64, 20)\n","Target tokens, shape: (batch, t) (64, 18)\n","logits, shape: (batch, t, target_vocabulary_size) (64, 18, 5000)\n"]}]},{"cell_type":"code","source":["model.compile(optimizer='adam',\n","              loss=masked_loss,\n","              metrics=[masked_acc, masked_loss])"],"metadata":{"id":"fdevnPq5wY4k","executionInfo":{"status":"ok","timestamp":1669568826041,"user_tz":-60,"elapsed":3,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":166,"outputs":[]},{"cell_type":"markdown","source":["#### Model Training"],"metadata":{"id":"k5PINvPJtJAX"}},{"cell_type":"code","source":["history = model.fit(\n","    train_ds.repeat(), \n","    epochs=20,\n","    steps_per_epoch = 100,\n","    validation_data=val_ds,\n","    validation_steps = 20,\n","    callbacks=[\n","        tf.keras.callbacks.EarlyStopping(patience=3)])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mGaaDEKdwtIc","executionInfo":{"status":"ok","timestamp":1669569042603,"user_tz":-60,"elapsed":216565,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"6484a16c-367c-4fc7-9d78-2f67b6355dd6"},"execution_count":167,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/20\n","100/100 [==============================] - 47s 272ms/step - loss: 4.0340 - masked_acc: 0.3873 - masked_loss: 4.0340 - val_loss: 3.2307 - val_masked_acc: 0.4588 - val_masked_loss: 3.2307\n","Epoch 2/20\n","100/100 [==============================] - 10s 101ms/step - loss: 2.9516 - masked_acc: 0.5069 - masked_loss: 2.9516 - val_loss: 2.7201 - val_masked_acc: 0.5412 - val_masked_loss: 2.7201\n","Epoch 3/20\n","100/100 [==============================] - 9s 90ms/step - loss: 2.5312 - masked_acc: 0.5605 - masked_loss: 2.5312 - val_loss: 2.4005 - val_masked_acc: 0.5720 - val_masked_loss: 2.4005\n","Epoch 4/20\n","100/100 [==============================] - 9s 85ms/step - loss: 2.2846 - masked_acc: 0.5939 - masked_loss: 2.2846 - val_loss: 2.2084 - val_masked_acc: 0.6057 - val_masked_loss: 2.2084\n","Epoch 5/20\n","100/100 [==============================] - 8s 83ms/step - loss: 2.1025 - masked_acc: 0.6196 - masked_loss: 2.1025 - val_loss: 2.0623 - val_masked_acc: 0.6209 - val_masked_loss: 2.0623\n","Epoch 6/20\n","100/100 [==============================] - 8s 82ms/step - loss: 1.9697 - masked_acc: 0.6362 - masked_loss: 1.9697 - val_loss: 1.9950 - val_masked_acc: 0.6316 - val_masked_loss: 1.9950\n","Epoch 7/20\n","100/100 [==============================] - 8s 84ms/step - loss: 1.8611 - masked_acc: 0.6507 - masked_loss: 1.8611 - val_loss: 1.7645 - val_masked_acc: 0.6590 - val_masked_loss: 1.7645\n","Epoch 8/20\n","100/100 [==============================] - 8s 82ms/step - loss: 1.7696 - masked_acc: 0.6650 - masked_loss: 1.7696 - val_loss: 1.7344 - val_masked_acc: 0.6733 - val_masked_loss: 1.7344\n","Epoch 9/20\n","100/100 [==============================] - 9s 87ms/step - loss: 1.6841 - masked_acc: 0.6753 - masked_loss: 1.6841 - val_loss: 1.6482 - val_masked_acc: 0.6822 - val_masked_loss: 1.6482\n","Epoch 10/20\n","100/100 [==============================] - 9s 87ms/step - loss: 1.6337 - masked_acc: 0.6829 - masked_loss: 1.6337 - val_loss: 1.6249 - val_masked_acc: 0.6806 - val_masked_loss: 1.6249\n","Epoch 11/20\n","100/100 [==============================] - 8s 83ms/step - loss: 1.5888 - masked_acc: 0.6876 - masked_loss: 1.5888 - val_loss: 1.5867 - val_masked_acc: 0.6833 - val_masked_loss: 1.5867\n","Epoch 12/20\n","100/100 [==============================] - 8s 85ms/step - loss: 1.5609 - masked_acc: 0.6937 - masked_loss: 1.5609 - val_loss: 1.5522 - val_masked_acc: 0.6968 - val_masked_loss: 1.5522\n","Epoch 13/20\n","100/100 [==============================] - 8s 79ms/step - loss: 1.4867 - masked_acc: 0.7035 - masked_loss: 1.4867 - val_loss: 1.5081 - val_masked_acc: 0.6901 - val_masked_loss: 1.5081\n","Epoch 14/20\n","100/100 [==============================] - 8s 83ms/step - loss: 1.4660 - masked_acc: 0.7053 - masked_loss: 1.4660 - val_loss: 1.4230 - val_masked_acc: 0.7116 - val_masked_loss: 1.4230\n","Epoch 15/20\n","100/100 [==============================] - 8s 80ms/step - loss: 1.4582 - masked_acc: 0.7085 - masked_loss: 1.4582 - val_loss: 1.4399 - val_masked_acc: 0.7081 - val_masked_loss: 1.4399\n","Epoch 16/20\n","100/100 [==============================] - 8s 80ms/step - loss: 1.4368 - masked_acc: 0.7096 - masked_loss: 1.4368 - val_loss: 1.3944 - val_masked_acc: 0.7213 - val_masked_loss: 1.3944\n","Epoch 17/20\n","100/100 [==============================] - 8s 80ms/step - loss: 1.4005 - masked_acc: 0.7160 - masked_loss: 1.4005 - val_loss: 1.3862 - val_masked_acc: 0.7122 - val_masked_loss: 1.3862\n","Epoch 18/20\n","100/100 [==============================] - 8s 80ms/step - loss: 1.3950 - masked_acc: 0.7175 - masked_loss: 1.3950 - val_loss: 1.3944 - val_masked_acc: 0.7168 - val_masked_loss: 1.3944\n","Epoch 19/20\n","100/100 [==============================] - 8s 79ms/step - loss: 1.3876 - masked_acc: 0.7172 - masked_loss: 1.3876 - val_loss: 1.3559 - val_masked_acc: 0.7226 - val_masked_loss: 1.3559\n","Epoch 20/20\n","100/100 [==============================] - 8s 80ms/step - loss: 1.3349 - masked_acc: 0.7282 - masked_loss: 1.3349 - val_loss: 1.2795 - val_masked_acc: 0.7302 - val_masked_loss: 1.2795\n"]}]},{"cell_type":"markdown","source":["#### Graphs:"],"metadata":{"id":"dvBJKbc2tL5L"}},{"cell_type":"code","source":["plt.plot(history.history['loss'], label='loss')\n","plt.plot(history.history['val_loss'], label='val_loss')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"id":"zo0GTQtIxcbd","colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569042603,"user_tz":-60,"elapsed":24,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"b6450892-6434-4def-8104-74ce5ba048a9"},"execution_count":168,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb4452872d0>"]},"metadata":{},"execution_count":168},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1dX48e9KcjOQBAKZIWEmIDIbQUUGUZE6gK1VnOrQVn91tlpbq2/VWjvYydbh0Tq9RV+rOFasWidmFSRgmAeZSZgykoSQef3+OAcM4SYEkpMbctfnec5zzz1733NXLpesnL332VtUFWOMMcErJNABGGOMCSxLBMYYE+QsERhjTJCzRGCMMUHOEoExxgS5sEAHcKwSEhK0d+/egQ7DGGNOKEuXLs1X1UR/ZSdcIujduzdZWVmBDsMYY04oIrKtsTJrGjLGmCBnicAYY4KcJQJjjAlyJ1wfgTEmOFVXV5OTk0NFRUWgQ2nXIiMjSUtLw+fzNfs1nicCEQkFsoBcVb2wQVkE8BJwClAATFfVrV7EUVlTy9KtRZzeLx4R8eItjDEeysnJITY2lt69e9v/4UaoKgUFBeTk5NCnT59mv64tmobuANY2UvYjoEhV+wOPAY96FcSs7J1c+fxi1u4q9eotjDEeqqioID7e/pBriogQHx9/zFdNniYCEUkDLgCeb6TKNGCGu/8mcLZ49K88IcMZPjt3w14vTm+MaQOWBI7ueD4jr68I/gb8HKhrpLwHsANAVWuAfUB8w0oicqOIZIlIVl5e3nEFktQ5ksGpnZm7/vheb4wxHZVniUBELgT2qurSlp5LVZ9V1UxVzUxM9HtjXLNMHJjIsm1FlFRUtzQkY0wQiomJCXQInvDyimAsMFVEtgKvAZNE5P8a1MkF0gFEJAzogtNp7ImJA5OoqVO+2Jjv1VsYY8wJx7NEoKq/VNU0Ve0NXA7MVtWrG1SbBVzr7n/frePZkmkje8YRGxFmzUPGmBZRVe655x6GDBnC0KFDmTlzJgC7du1i/PjxjBgxgiFDhrBgwQJqa2u57rrrDtV97LHHAhz9kdr8PgIReRjIUtVZwAvAyyKyESjESRie8YWGcOaABOZtyENVrePJmBPUr99bzZqdJa16zsHdO/PgRSc3q+7bb79NdnY2y5cvJz8/n1NPPZXx48fzr3/9i/POO4/777+f2tpaysvLyc7OJjc3l1WrVgFQXFzcqnG3hjZJBKo6F5jr7j9Q73gFcGlbxHDQxIGJfLhqNxv2lDEwJbYt39oY00EsXLiQK664gtDQUJKTk5kwYQJLlizh1FNP5Yc//CHV1dVcfPHFjBgxgr59+7J582Zuu+02LrjgAiZPnhzo8I8QdHcWjz84jHT9XksExpygmvuXe1sbP3488+fP5/333+e6667jrrvu4pprrmH58uV89NFHPPPMM7z++uu8+OKLgQ71MEE311BqlygGpcRaP4Ex5riNGzeOmTNnUltbS15eHvPnz2f06NFs27aN5ORkbrjhBn784x+zbNky8vPzqaur45JLLuGRRx5h2bJlgQ7/CEF3RQAwYWAiLy7cQlllDTERQfkRGGNa4Lvf/S5ffvklw4cPR0T44x//SEpKCjNmzOBPf/oTPp+PmJgYXnrpJXJzc7n++uupq3Nup/r9738f4OiPJB4O0vFEZmamtnRhmi825XPlc4t59genMPnklFaKzBjjpbVr13LSSScFOowTgr/PSkSWqmqmv/pB1zQEkNmrG9HhoczdYM1DxhgTlIkgPCyEsf0TmLfeGUZqjDHBLCgTATj9BLnFB9iUVxboUIwxJqCCNhFMHJgEYKOHjDFBL2gTQY+4KAYkxTDP+gmMMUEuaBMBOGsULN5cSHlVTaBDMcaYgAnqRDBxYBJVtXV8ucmzCU+NMabdC+pEcGqfrnQKD7XmIWNMq2tq7YKtW7cyZMiQNoymaUGdCCLCQjmjXzxzbRipMSaIBf38ChMyEvl07V625O+nb2LHXH3ImA7nw3th98rWPWfKUPjOHxotvvfee0lPT+eWW24B4KGHHiIsLIw5c+ZQVFREdXU1jzzyCNOmTTumt62oqOCmm24iKyuLsLAw/vrXv3LWWWexevVqrr/+eqqqqqirq+Ott96ie/fuXHbZZeTk5FBbW8uvfvUrpk+f3qIfGywRuMNIVzN3fZ4lAmNMo6ZPn86dd955KBG8/vrrfPTRR9x+++107tyZ/Px8TjvtNKZOnXpMa5089dRTiAgrV65k3bp1TJ48mQ0bNvDMM89wxx13cNVVV1FVVUVtbS0ffPAB3bt35/333wdg3759rfKzBX0iSO/Wib6J0czbkMcPz+wT6HCMMc3RxF/uXhk5ciR79+5l586d5OXl0bVrV1JSUvjpT3/K/PnzCQkJITc3lz179pCS0vw5zBYuXMhtt90GwKBBg+jVqxcbNmzg9NNP57e//S05OTl873vfY8CAAQwdOpS7776bX/ziF1x44YWMGzeuVX62oO4jOGhCRiKLNhdQUV0b6FCMMe3YpZdeyptvvsnMmTOZPn06r7zyCnl5eSxdupTs7GySk5OpqKholfe68sormTVrFlFRUZx//vnMnj2bjIwMli1bxtChQ/mf//kfHn744VZ5L88SgYhEishXIrJcRFaLyK/91LlORPJEJNvdfuxVPE2ZODCJypo6vtxsw0iNMY2bPn06r732Gm+++SaXXnop+/btIykpCZ/Px5w5c9i2bdsxn3PcuHG88sorAGzYsIHt27czcOBANm/eTN++fbn99tuZNm0aK1asYOfOnXTq1Imrr76ae+65p9XWNvCyaagSmKSqZSLiAxaKyIequqhBvZmqequHcRzVmD7diPSFMG99Hme5U08YY0xDJ598MqWlpfTo0YPU1FSuuuoqLrroIoYOHUpmZiaDBg065nPefPPN3HTTTQwdOpSwsDD++c9/EhERweuvv87LL7+Mz+cjJSWF++67jyVLlnDPPfcQEhKCz+fj6aefbpWfq03WIxCRTsBC4CZVXVzv+HVA5rEkgtZYj8Cf6/73K7YVlDPnZxNb/dzGmJaz9Qiar12tRyAioSKSDewFPqmfBOq5RERWiMibIpLeyHluFJEsEcnKy/Pm5q+JGYlsyd/PtoL9npzfGGPaK08TgarWquoIIA0YLSINb6V7D+itqsOAT4AZjZznWVXNVNXMxMRET2I9OBup3WVsjGktK1euZMSIEYdtY8aMCXRYR2iT4aOqWiwic4ApwKp6x+v3zj4P/LEt4vGnd0I0veI7MXd9Htec3jtQYRhjmqCqxzRGP9CGDh1KdnZ2m77n8TT3ezlqKFFE4tz9KOBcYF2DOqn1nk4F1noVT3NMzEjki035NozUmHYoMjKSgoICmw6mCapKQUEBkZGRx/Q6L68IUoEZIhKKk3BeV9X/iMjDQJaqzgJuF5GpQA1QCFznYTxHNXFgEjO+3MaSrYWMG+BNE5Qx5vikpaWRk5ODV/2EHUVkZCRpaWnH9BrPEoGqrgBG+jn+QL39XwK/9CqGY3Va33jCw0KYuz7PEoEx7YzP56NPH7v73wt2Z3E9UeGhjOnTjbnr9wY6FGOMaTOWCBqYODCJTXn72VFYHuhQjDGmTVgiaGBChtMkZMNIjTHBwhJBA/0So0nrGsXc9ZYIjDHBwRJBAyLCxIHOMNLKGhtGaozp+CwR+DExI4nyqlqWbi0KdCjGGOM5SwR+nN4vnvDQEOZaP4ExJggETyKo2g9Z/wvNuCsxOiKMU/t0tWGkxpigEDyJYPU78J87YdVbzao+MSOJDXvK2Fl8wOPAjDEmsIInEQy/ArqPhI/uh4qSo1afMNCGkRpjgkPwJIKQULjgL1C2B+Y9etTqA5Ji6N4l0pqHjDEdXvAkAoAep8Ap18Kip2HPmiariggTBibx+cYCqmvr2ihAY4xpe8GVCADOfhAiO8MHPztqx/GEjETKKmtYus2GkRpjOq7gSwSdusE5D8G2z2HlG01WHds/nrAQsbuMjTEdWvAlAoCR1zjNRB//D1Tsa7RabKSPzN42jNQY07EFZyIICYHz/wxle2HO75usOiEjiXW7S9lTUtFGwRljTNsKzkQA0GMUZF4PX/0Ddq9stNrEg8NIrXnIGNNBeblmcaSIfCUiy0VktYj82k+dCBGZKSIbRWSxiPT2Kh6/Jv0KorrC+413HA9KiSW5cwRzN1jzkDGmY/LyiqASmKSqw4ERwBQROa1BnR8BRaraH3gMOPoA/9bUqRuc82vYsQiWv+a3iogwISORBd/kU2PDSI0xHZBniUAdZe5Tn7s1/LN7GjDD3X8TOFtExKuY/BpxFaSdCp/8Cg4U+60ycWASpRU1fL3Df7kxxpzIPO0jEJFQEckG9gKfqOriBlV6ADsAVLUG2AfE+znPjSKSJSJZeXmt3FZ/sOO4vADm/M5vlbH9EwgNERs9ZIzpkDxNBKpaq6ojgDRgtIgMOc7zPKuqmaqamZiY2LpBAnQfAZk/giXPwa4VRxR3ifIxqmeczTtkjOmQ2mTUkKoWA3OAKQ2KcoF0ABEJA7oABW0R0xEm3Q9R3Zw7juuO7AuYODCJVbkltqi9MabD8XLUUKKIxLn7UcC5wLoG1WYB17r73wdmqzZjwQAvRHWFcx+GHYth+atHFE8d3p1O4aHc/cZyausCE6IxxnjByyuCVGCOiKwAluD0EfxHRB4WkalunReAeBHZCNwF3OthPEc3/ApIHwOfPAAHDp9fKL1bJ34zbQhfbSnkidnfBChAY4xpfRKoP8CPV2ZmpmZlZXn3BrtWwLMTnD6DC/58RPFPZ2bzbnYur95wGmP6HtGvbYwx7ZKILFXVTH9lwXtncWNSh8GpN0DWC7Az+4ji31w8hJ7dOnHnzGyK9lcFIEBjjGldlgj8Oes+6BTvt+M4JiKMJ64YRX5ZJT9/awUn2hWVMcY0ZInAn6g4OPc3kLMEsl85onhoWhd+MWUQn6zZw8uLtgUgQGOMaT2WCBoz/HLoeTp8+iCUFx5R/KMz+3DWwEQeeX8ta3YefQ1kY4xprywRNEbEueP4QDHMfsRPsfDnS4cTF+XjtleXUV5VE4AgjTGm5SwRNCVlCIy+EbJehJ1fH1EcHxPB36aPYHP+fh6atToAARpjTMtZIjias34J0Ynw/t1+7zg+o38CN0/sx+tZOcxavjMAARpjTMtYIjiayC4w+RHIXQpfv+y3yp3nZDCqZxz3vb2S7QU2BYUx5sRiiaA5hl0GPc+ATx+CfblHFPtCQ/j75SMJEbjtta+ptnULjDEnEEsEzSECFz4GdTXw0jRnreMG0rt14tFLhrF8RzF//nh9AII0xpjjY4mguZIGwVVvQEmukwz8DCn9ztBUrhzTk3/M28x8m7LaGHOCsERwLHqeBle8BgWb4OWL/a5o9sCFg8lIjuGu17PZW1oRgCCNMebYWCI4Vn0nwPT/gz1r4JVLobLssOJIXyhPXjmK0ooa7n59OXU2ZbUxpp2zRHA8MibD9190RhK9ejlUHT5SKCM5lgcuGsyCb/J5dsHmAAVpjDHNY4ngeA2eCt/9B2xdCDOvhprKw4qvHN2T7wxJ4c8frefr7UWNnMQYYwLPEkFLDLsUpj4Bmz6DN66H2upDRSLCH743jOTOkdz+2teUVFQ3cSJjjAkcSwQtNeoHzpxE69+Ht2+EutpDRV06+Xj8ipHsLK7g/ndW2ZTVxph2ycs1i9NFZI6IrBGR1SJyh586E0Vkn4hku9sDXsXjqdE3ONNWr34b3r31sKkoTunVlbvOzeC95Tt5IysngEEaY4x/YR6euwa4W1WXiUgssFREPlHVNQ3qLVDVCz2Mo22MvR2qD8Dc34EvEi74q3MjGvCTCf34fGM+D85aTWpcJOMGJAY4WGOM+ZZnVwSquktVl7n7pcBaoIdX79cuTPg5jL3Tma30o/vBbQoKDRH+dvkIesV34toXv+LZ+ZusmcgY0260SR+BiPQGRgKL/RSfLiLLReRDETm5kdffKCJZIpKVl9eO79gVgXMegtH/DxY9BXN+e6goKTaSt246gylDUvjdB+u447VsDlTVNnoqY4xpK54nAhGJAd4C7lTVhkt5LQN6qepw4Ang3/7OoarPqmqmqmYmJrbzZhURmPIHGHUNzP8TzP/zoaLoiDCeunIUP58ykPdW7OSSp79gR6HNVmqMCSxPE4GI+HCSwCuq+nbDclUtUdUyd/8DwCciCV7G1CZCQuDCv8HQy2D2b+DLpw4ViQg3T+zPi9edSk5ROVOfXMjnG/MDGKwxJtg1OxGIyBkicqWIXHNwO0p9AV4A1qrqXxupk+LWQ0RGu/EUND/8diwkFC5+Gk6aCh/dB0teOKz4rIFJzLr1TBJjI/jBC4t5fsFm6zcwxgREs0YNicjLQD8gGzjYsK3AS028bCzwA2CliGS7x+4DegKo6jPA94GbRKQGOABcrh3pt2FoGFzyAsy8Ct6/C3xRMOLKQ8W9E6J5++ax/Oz15Tzy/lpW5e7jD5cMI9IXGsCgjTHBRprze1dE1gKD28Mv6czMTM3Kygp0GMemugL+dRlsXQBjboKz7oOImEPFqspTczbyl082MDi1M//4wSmkde0UwICNMR2NiCxV1Ux/Zc1tGloFpLReSEHGFwlXvAqjrnVGEz01BtZ9cKhYRLh10gBeuDaT7YXlTH3yc77YZP0Gxpi20dxEkACsEZGPRGTWwc3LwDqc8Gi46G/ww48hsjO8dgW8dtVhS19OGpTMu7eMpVt0OD944SteXLjF+g2MMZ5rbtPQBH/HVXVeq0d0FCdk01BDtdXwxRMw71EICYNJv3KmqQhx+gZKK6q5+/XlfLxmD98b1YPffXeo9RsYY1qkxU1D7i/8rYDP3V+Ccw+AOR6hPhh3F9y8CNLHwH9/Ac+fDbuWAxAb6eOZq0/hp+dk8PayXC595kt2Fh8IcNDGmI6qWYlARG4A3gT+4R7qQSM3f5lj0K0PXP2WM7JoXy48O9GZmqKyjJAQ4Y5zBvDcNZlsyd/PRU8sZPHmjjGy1hjTvjS3j+AWnOGgJQCq+g2Q5FVQQUUEhn4fbv3K6Uz+8kmnM3n9hwCcOziZf98yli6dfFz5/GJ++/4aSm1tA2NMK2puIqhU1aqDT0QkDOc+AtNaorp+25kcEessgTnzaijZSf+kGP59y1i+PyqN5xduYdJf5vHO1znWkWyMaRXNTQTzROQ+IEpEzgXeAN7zLqwg1nMM/L/5cPaD8M0n8ORoWPwPOoeH8Oj3h/HOzWPp3iWSn85czmX/+JLVO/cFOmJjzAmuuaOGQoAfAZMBAT5S1ec8js2vDjFqqLkKtzh3JG+aDd1HwkV/h9Th1NUpbyzdwaP/XU9xeRVXjenF3ZMziOsUHuiIjTHtVFOjhpqbCB5W1QfqPQ8FXlLVq1ovzOYJqkQAzpoGq96C//4SyvPh5O/CmXdByhD2lVfz2KcbeOnLrXSJ8vHzKYO4LDOd0BAJdNTGmHamNe4sTheRX7onC8eZUfSbVorPNKV+Z/Lpt8KGj+CZsfCv6XTJX8ZDU0/m/dvHMSA5ll++vZKLn/qcZduLAh21MeYE0twrAgFeAVYCZwEfqupjHsfmV9BdETRUXghfPQeLn4YDRdDrTBh3F9r3LGat2MXvPljLnpJKLj0ljZ9PGURibESgIzbGtAPH3TQkIqPqPfXh3EfwOc700hxcirItBX0iOKiyDJbNgC+ehNKdkDoCxt1FWd/v8MScTby4cAuRvlDuOjeDH5zWi7DQNlmMzhjTTrUkEcxp4ryqqpNaGtyxskTQQE0lLH8NPv8bFG6GhAwYeyebUs/nofc3sOCbfAYmx/LQ1JM5vV98oKM1xgRIizuL2xNLBI2oq4U1/4YFf4U9q6BLOnrGbXwadR6//nAzOUUHuHBYKrdNGsDAlNhAR2uMaWOtMWqoC/AgMN49NA94WFXbfBC7JYKjUHXuP1jwF9ixCKITqT71JzxXMYnHP99DRXUdZ/SL5/qxfZg0KMlGGBkTJFojEbyFsybBDPfQD4Dhqvq9VouymSwRHINtXzgJYeOnENGZiuHX8t+qYTy+JprNJdCzWyeuOb0Xl52aTudIX6CjNcZ4qDUSQbaqjjjasQbl6ThLWSbjTEfxrKr+vUEdAf4OnA+UA9cdrQPaEsFx2JkNCx+DNe8CikooJV0G8mVVXz4qTmdd2EBOHZXJtWP70C8x5qinM8aceJpKBM1asxg4ICJnqupC94RjcdYYbkoNcLeqLhORWGCpiHyiqmvq1fkOMMDdxgBPu4+mNXUfAZfNgP0FkJuF5CyhS84SpuTMY0p4KQBFX8fw9dL+ZHcbTv+RExk6ehIhneICHLgxpi00NxH8BHjJ7SsAKAKubeoFqroL2OXul7rrHvcA6ieCaTh3KCuwSETiRCTVfa1pbdHxkHGes4HTwZy/AXZ8ReSWxQzf/CVd971EyNwZ1M0ViqP7EN3vdHy9xkDaqZA48NDiOcaYjqO5iaBEVYeLSGcAVS0RkT7NfRMR6Q2MBBY3KOoB7Kj3PMc9ZomgLYSEQtJJkHQSUadcSxRQVVZE1hefsjl7Hqklqxi1fBZdV7zi1I/sApN/CyOvdu54NsZ0CM1NBG8Bo1S1pN6xN4FTjvZCEYlxX39ng9c3m4jcCNwI0LNnz+M5hWmm8JiunDH5Us6YfClfby/iwYVbWL3qa4bLN9wY+QWDZt1K7ZYFhF74V4iw/gRjOoImE4GIDAJOBrqISP0RQp2ByKOdXER8OEngFVV920+VXCC93vM099hhVPVZ4FlwOouP9r6mdYzs2ZWRV3Zl977BvLJ4G9csnsjl1W9w58rX2bthETvPfZqho86wIajGnOCOdmfxNOBiYCowq15RKfCaqn7RxGsFZ7hpoare2UidC4BbcUYNjQEeV9XRTQVso4YCp6a2js83FbD68/e4bOtDRHOAP4fdgIy4imkj0xjSozNiTUbGtEstmWLiCuBjIENVvzzGNz0TWIAzUV2de/g+oCeAqj7jJosngSk4w0evV9Umf8tbImgfDhTuZP+r15OQt4h36sZxf9X1pCTGM214D6aN6E7vhOhAh2iMqaclieAXwHk4E859BnwIfKUBnJfCEkE7UlcL8/+Mzv09JdG9+U3UL3gzpzMAw9PjuHhEdy4YlkpS7FFbEY0xHmuNG8pigXNw/nIfDawF/ouzUtmeVoz1qCwRtEOb58FbP4bKUorO+i2vV0/g38t3sXZXCSECY/snMG1ED847OZlYu4PZmIBo9UnnRGQwzs1gk1X1vBbGd0wsEbRTpXvg7R/Dlvkw7HK44C9sKFZmZe/k3eW57Cg8QHhYCGP7xXP2SclMGpRE97ioQEdtTNBoSdPQ1ar6f+7+WFX9vF7Zrar6ZKtHexSWCNqxulqY/yeY+wdIGACXzoDkwagqy7YX897ynXy2bg87Cp2b0gendubsk5KYNCiJ4WlxhNjoI2M805JEsExVRzXc9/e8rVgiOAHUayri/D8ddgOaqrIpr4xP1+5l9tq9ZG0rpE4hISaciQOTOOekJM4ckEhMRHNvcTHGNEdLEsHXqjqy4b6/523FEsEJwk9Tkb8b0IrLq5i3IY9P1+5l3vq9lFTU4AsVTusbz9mDkjj7pGTSu3UKwA9gTMdiVwQmMBppKmpMdW0dS7cVMXvdXj5bu4dNefsBGJAUw6STkjjnpGRGpsfZspvGHIeWJIJyYCMgQD93H/d5X1Vt88HilghOQJvnwls3QEUxdOsLMckQm3LkY2yqs+9eOWzN389n6/Yye90eFm8upKZOiY0MY2y/BMZnJDI+I4G0rna1YExztCQRDMBZT2BHg6J0YLeqbjzyVd6yRHCCKt0DXzwORVuhbI/zvGw31FYdWTc85ogkURGZwLr9sXxY0of3toawc18FAP0So92kkMhpfeKJCrfZUY3xpyWJ4D/AL1V1ZYPjQ4HfqepFrRppM1gi6EBU4UCRmxh2fZsc/D1W7//2ZQkDKe4+nkUhI3mzoBcLt5RSWVNHeFgIo3t3Y4KbGDKSY2zKC2NcLUkES1T11EbKVqrq0FaKsdksEQSpylLnamLzPNj0GWz9HGorISyK2l5j2dLlND6uGsI726L4xu1bSOkcybgBTjPSmf0T6BodHtifwZgAakki+EZVBzRStlFV+7dSjM1micAAUFUO2z531mPe+CkUuK2UcT3Znz6RZeGn8E5xfz7dtJ+SihpEYFhaHEO6d2ZAUgwDkmMZkBRDYmyEXTWYoNCSpSqzROQGVX2uwQl/DCxtrQCNOWbhnWDAuc4GULjFuVLYOJvo9W8zruolxoWEob3GkJswlrk1w3h3N7y3fCclFTWHTtM5MuxQUuhfL0Gkdom0BGGCxtGuCJKBd4Aqvv3FnwmEA99V1d2eR9iAXRGYo6qpgh2L3auFz2CP28XVKQGN60lVeBz7iCW/LpqdVZ3YWh7BhtJwciqjKNJYijSG6oiupCV1IyMphgHJMQxIiqV/Ugzd46Js/QVzQmqNSefOAoa4T1er6uxWjO+YWCIwx6x0N2yaDVsXOh3T5YVQXuB0VFc2vmhehURQrLEU1EVTqLEUEcseSeBApx4Q15PIxD7EpfYlPTmRvonRJFkzk2nHWn3SuUCyRGBaVU2VkxDKC+BAYb0kcXC/kOrSPCpL89H9+UQd2E2YVh92igKNJUcT2S2JlEb2oLZzGmHxvYhN6U9iWj96piTRLTrckoQJqJb0ERjTsYWFQ2yyszXC524A1NU5VxXF26kr3k7Jrk1U5W0hsWgbPcpy6VK5DF9eNeQB65yXFGoMaySJ4vBUyiNTqIpORWNTCe2aTmS3nsQk9iChczSJsRFEh4dawjBtzhKBMcciJAQ6p0LnVEJ6jiFuGMTVL6+rg/151BRupTB3I6W7N1FVsB1fyXZ6H9hOt5KlRJVUwK5vX1KjIewljvUaz17i2ReexP6IFKqiU9DOPQiNS6NTt1QSO3diSI8udje1aXWeJQIReRG4ENirqkP8lE8E3gW2uIfeVtWHvYrHmDYREgKxyYTFJpPUawxJDctVoaKYmqIcyvK2UZ63neqiHHRfDillO+ldsZPYyq8J318J+4G9zsuqNQlZSfoAABDlSURBVJQ9dGV7XRJZ4WmExPclLm0QvTOGkd5vMOKztR3M8fPyiuCfOOsRv9REnQWqeqGHMRjTvohAVFfCoroS133o4VcTBx2847okF/blUlu8g6rCHUQXbKd/3maGli0mdu/HTpJYBnUIhWGJVHbuTWTyAOLSBhES3x/i+0HX3hAW0bY/oznheJYIVHW+iPT26vzGdFgi0Kmbs6UMJRSIdreD9EAROzetZvvGlezLXQ+Fm0nOz6V3wbuErH3l23oIdElD4vtBfH9IPw36nQXRCW39U5l2LNB9BKeLyHJgJ/AzVV0d4HiMOSFIVFd6DDmTHkPOPHQst/gA87YUsnLjVnZvXUNY0Wb6hOymX9EeTirfRfq2JUQseR5F2N/tZA70nAj9zyaq7+lER9kNdMHM0+Gj7hXBfxrpI+gM1KlqmYicD/y9ieksbgRuBOjZs+cp27Zt8yxmYzqK/LJKsrYWsnhLIV9tKWT9rmJOZjPjQ1YwPnQFI2UjYVJHqUaxSE9maegIVkVlUhbdky5RPuI6+egS9e3WOcpHpC+U8NAQwsOE8NBQfKFCeFgIvtAQItzH8HqP4e6j3YQXeAG7j6CpROCn7lYgU1Xzm6pn9xEYc3wOVNVSWF5FcXkV+w5UU15SSMSOz4nfvYDuBV8QV+kMZdoT1p2ssJEs1OHMrRzE7sowWvprIkQgPCyE2EgfcYeSTPihZHPoWKdw4tzEE9fJR1xUOLGRYbaedStol/cRiEgKsEdVVURGAyFAQaDiMaajiwoPpUd4FD3iDo4wSoCRGcD1Tgd1wSbY9BnJGz/jgq1zuKD6fQjxoRmjqex1FvtSx1Ip4dQcKEErSqmrKIHKUqSiBKpKkcpSQqvLCKlyHsNqyvBV78dXW4avZj/hteWUajd2V6Wxo6Y7W/alsKE6mS+rkvimKp7qRn4diXAoWSR1jiQtLoruh7ZI0rpGkdolimhb5/q4eXZFICKvAhOBBGAP8CDufTmq+oyI3ArcBNQAB4C7VPWLo53XrgiMaQM1lbB90aGJ/A7N19SU8BiI6AwRsQ0295gvypnuo2Cjsx0oPPRSlVBqu/SksnMfymJ6UxjZk7zwdHJDu7OrrivFFbUUlVeze98BdhZXsLukgtq6w393xXXy0b1LFD26Osmue1wk3eOc/R5xUSTERAT1lYVNMWGMaZnSPc603yKH/3I/uIXHQMgxrg5XXuhchRRshMJN3yaIgk1QXf5tvbAoZyhst74Q2QXCIqgLCWd/bSgl1cK+KqGoSiisEPIPQP4B2FteR0l1CJWEUYWPKg1DQ31EhQlRITVESQ2RUkNkSK37WE241BEp1URQQ7jUEEE14VJDONWEU4OPGsp9cewLT6Uksjtlkansj+qORMTU6xsRfKFuH0loCL4wqbcfQlpcFL0TovEFYN3tdtk0ZIw5gcQmw5Dvte45Dw6RTW+w9pWqs2Jd/cRQsBH2roWqMqitIqSmitjaSmJrq+jh79yCM0eyP3XNC6+WEKrxUS0+qvFRQyhxuo9wDp9rqlBj2aGJ5GgCOZrIJk06tJ+rCRwg8rD6vlChX2IMGcmxDEyJdR6TY0nrGhWwKxZLBMaY9kUEOnd3tj7jm66r6qx7XVPpPNbfP+yxEmqrQUKd+aVCI+o9RkBouPvoO3QsNCSUUDj817g7hQjF26F4GxRvp1vxdroWb2dY8XYoXo7UVh4WYk1kPFWxaVRE92Bnp4Es42QW7O/K0m1FzFq+81C9KF8oGckNEkRKbJvMamtNQ8YY01r8JIpDW9EWKNzs1PNFQ8/TqEw7na2xI8mu6cO6vAo27Cll/e4y8su+TSZdonwMTI4lIyWGcwenMCEj8bhCs6YhY4xpC+5cU8QmH9nkBVCW5/S1bF0IWxcSMe8RBgIDfdGQPhoyzoTJ4yjoMoYN+VV8s7eU9btL2bCnlHezd5IYE3nciaApdkVgjDGBsj+/XmL4HPa6kyuERUHPMdD7TOh1JvQ4BQ31UV2rhIcdX0ezXREYY0x7FJ0Ag6c5G8D+AicxHEwOsx9xjodFIemjCT/lWhhySauHYYnAGGPai+h4GDzV2cAZYrvtc+dqYetC2JfrydtaIjDGmPaqUzc46SJnA1o810cj2v6uBmOMMcfHo2GklgiMMSbIWSIwxpggZ4nAGGOCnCUCY4wJcpYIjDEmyFkiMMaYIGeJwBhjgpwlAmOMCXKWCIwxJsh5lghE5EUR2SsiqxopFxF5XEQ2isgKERnlVSzGGGMa5+UVwT+BKU2UfwcY4G43Ak97GIsxxphGeJYIVHU+UNhElWnAS+pYBMSJSKpX8RhjjPEvkH0EPYAd9Z7nuMeOICI3ikiWiGTl5eW1SXDGGBMsTojOYlV9VlUzVTUzMbH1l2kzxphgFshEkAuk13ue5h4zxhjThgKZCGYB17ijh04D9qnqrgDGY4wxQcmzFcpE5FVgIpAgIjnAg4APQFWfAT4Azgc2AuXA9V7FYowxpnGeJQJVveIo5Qrc4tX7G2OMaZ4TorPYGGOMdywRGGNMkLNEYIwxQc4SgTHGBDlLBMYYE+QsERhjTJCzRGCMMUHOEoExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKBMcYEOUsExhgT5CwRGGNMkLNEYIwxQc4SgTHGBDlLBMYYE+Q8TQQiMkVE1ovIRhG510/5dSKSJyLZ7vZjL+MxxhhzJC/XLA4FngLOBXKAJSIyS1XXNKg6U1Vv9SoOY4wxTfPyimA0sFFVN6tqFfAaMM3D9zPGGHMcvEwEPYAd9Z7nuMcaukREVojImyKS7u9EInKjiGSJSFZeXp4XsRpjTNAKdGfxe0BvVR0GfALM8FdJVZ9V1UxVzUxMTGzTAI0xpqPzMhHkAvX/wk9zjx2iqgWqWuk+fR44xcN4jDHG+OFlIlgCDBCRPiISDlwOzKpfQURS6z2dCqz1MB5jjDF+eDZqSFVrRORW4CMgFHhRVVeLyMNAlqrOAm4XkalADVAIXOdVPMYYY/wTVQ10DMckMzNTs7KyAh2GMcacUERkqapm+isLdGexMcaYALNEYIwxQc4SgTHGBDlLBMYYE+QsERhjTJCzRGCMMUHOEoExxgQ5SwTGGBPkLBEYY0yQs0RgjDFBzhKBMcYEOUsExhgT5CwRGGNMkLNEYIwxQc4SgTHGBDlLBMYYE+QsERhjTJDzNBGIyBQRWS8iG0XkXj/lESIy0y1fLCK9vYzHGGPMkTxLBCISCjwFfAcYDFwhIoMbVPsRUKSq/YHHgEe9iscYY4x/Xl4RjAY2qupmVa0CXgOmNagzDZjh7r8JnC0i4mFMxhhjGgjz8Nw9gB31nucAYxqro6o1IrIPiAfy61cSkRuBG92nZSKy/jhjSmh47namvccH7T9Gi69lLL6Wac/x9WqswMtE0GpU9Vng2ZaeR0SyVDWzFULyRHuPD9p/jBZfy1h8LdPe42uMl01DuUB6vedp7jG/dUQkDOgCFHgYkzHGmAa8TARLgAEi0kdEwoHLgVkN6swCrnX3vw/MVlX1MCZjjDENeNY05Lb53wp8BIQCL6rqahF5GMhS1VnAC8DLIrIRKMRJFl5qcfOSx9p7fND+Y7T4Wsbia5n2Hp9fYn+AG2NMcLM7i40xJshZIjDGmCDXIRNBe57aQkTSRWSOiKwRkdUicoefOhNFZJ+IZLvbA20Vn/v+W0VkpfveWX7KRUQedz+/FSIyqg1jG1jvc8kWkRIRubNBnTb//ETkRRHZKyKr6h3rJiKfiMg37mPXRl57rVvnGxG51l8dj+L7k4isc/8N3xGRuEZe2+T3wcP4HhKR3Hr/juc38tom/797GN/MerFtFZHsRl7r+efXYqraoTacjulNQF8gHFgODG5Q52bgGXf/cmBmG8aXCoxy92OBDX7imwj8J4Cf4VYgoYny84EPAQFOAxYH8N96N9Ar0J8fMB4YBayqd+yPwL3u/r3Ao35e1w3Y7D52dfe7tlF8k4Ewd/9Rf/E15/vgYXwPAT9rxnegyf/vXsXXoPwvwAOB+vxaunXEK4J2PbWFqu5S1WXufimwFucO6xPJNOAldSwC4kQkNQBxnA1sUtVtAXjvw6jqfJyRb/XV/57NAC7289LzgE9UtVBVi4BPgCltEZ+qfqyqNe7TRTj3+gREI59fczTn/3uLNRWf+7vjMuDV1n7fttIRE4G/qS0a/qI9bGoL4ODUFm3KbZIaCSz2U3y6iCwXkQ9F5OQ2DQwU+FhElrrTezTUnM+4LVxO4//5Avn5HZSsqrvc/d1Asp867eWz/CHOVZ4/R/s+eOlWt+nqxUaa1trD5zcO2KOq3zRSHsjPr1k6YiI4IYhIDPAWcKeqljQoXobT3DEceAL4dxuHd6aqjsKZOfYWERnfxu9/VO5NilOBN/wUB/rzO4I6bQTtcqy2iNwP1ACvNFIlUN+Hp4F+wAhgF07zS3t0BU1fDbT7/08dMRG0+6ktRMSHkwReUdW3G5araomqlrn7HwA+EUloq/hUNdd93Au8g3P5XV9zPmOvfQdYpqp7GhYE+vOrZ8/BJjP3ca+fOgH9LEXkOuBC4Co3WR2hGd8HT6jqHlWtVdU64LlG3jfQn18Y8D1gZmN1AvX5HYuOmAja9dQWbnviC8BaVf1rI3VSDvZZiMhonH+nNklUIhItIrEH93E6FFc1qDYLuMYdPXQasK9eE0hbafSvsEB+fg3U/55dC7zrp85HwGQR6eo2fUx2j3lORKYAPwemqmp5I3Wa833wKr76/U7fbeR9m/P/3UvnAOtUNcdfYSA/v2MS6N5qLzacUS0bcEYT3O8eexjnCw8QidOksBH4CujbhrGdidNEsALIdrfzgZ8AP3Hr3AqsxhkBsQg4ow3j6+u+73I3hoOfX/34BGfRoU3ASiCzjf99o3F+sXepdyygnx9OUtoFVOO0U/8Ip9/pM+Ab4FOgm1s3E3i+3mt/6H4XNwLXt2F8G3Ha1w9+Dw+OpOsOfNDU96GN4nvZ/X6twPnlntowPvf5Ef/f2yI+9/g/D37v6tVt88+vpZtNMWGMMUGuIzYNGWOMOQaWCIwxJshZIjDGmCBnicAYY4KcJQJjjAlylghMUBOR2gazmbba7JUi0rv+bJXNqB8tIp+6+wvdm5WM8Zx90UywO6CqIwIdhOt04Ev3xrL9+u2EcMZ4yq4IjPHDnUP+j+488l+JSH/3eG8Rme1OhPaZiPR0jye7c/ovd7cz3FOFishz4qw98bGIRPl5r37uXPb/B1wJLAWGu1coSW30I5sgZonABLuoBk1D0+uV7VPVocCTwN/cY08AM1R1GM4kbY+7xx8H5qkz0d0onLtIAQYAT6nqyUAxcEnDAFR1k3tVshRnHpoZOHeujlBnfhpjPGV3FpugJiJlqhrj5/hWYJKqbnYnCdytqvEiko8z1UG1e3yXqiaISB6QpqqV9c7RG2etgQHu818APlV9pJFYlqjqqSLyFnCHNjJ/jTGtza4IjGmcNrJ/LCrr7dfip19ORJ5xO5UHuE1EU4D/iMhPj/M9jTkmlgiMadz0eo9fuvtf4MxwCXAVsMDd/wy4CUBEQkWkS3PfRFV/Avwa+A3OKmbvu81Cj7UsfGOax0YNmWAXJYcvOv5fVT04hLSriKzA+av+CvfYbcD/isg9QB5wvXv8DuBZEfkRzl/+N+HMVtlcE4CXcFa7mndcP4kxx8n6CIzxw+0jyFTV/EDHYozXrGnIGGOCnF0RGGNMkLMrAmOMCXKWCIwxJshZIjDGmCBnicAYY4KcJQJjjAly/x/BE/lc7q+KegAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["plt.plot(history.history['masked_acc'], label='accuracy')\n","plt.plot(history.history['val_masked_acc'], label='val_accuracy')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"id":"h4rMHE7Mxdli","colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569042604,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"c6257ebe-3851-44db-bce1-fbb4a755e40e"},"execution_count":169,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb4452bc790>"]},"metadata":{},"execution_count":169},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9bn48c+TyR5CIAsQAiHsmxiRCAIKKKLYWlArAu64cG2rdbm9LdVWqe3ttcutV/ujrWhdqLV4i1fFvaIIKigERPZ9S1hDyAZkn+f3x5nEECYkQs5Mknner9e85izfc84zJ5PzzPmec75fUVWMMcaErrBgB2CMMSa4LBEYY0yIs0RgjDEhzhKBMcaEOEsExhgT4sKDHcA3lZycrBkZGcEOwxhjWpVVq1YdUdUUf/NaXSLIyMggOzs72GEYY0yrIiJ7GppnVUPGGBPiLBEYY0yIa3VVQ8YYEzK8XijZD0d3wdGdkH4hpPRv9s1YIjDGmGCqroTCvV8f7Aucdz26Cy3YTVh1eW3RzZk/ZcA1s5o9BEsExhjjJlUoLYDifVCw2zng+w72HN0FRTmg3triFWHR7JNUtlUms8t7GXu0C4fDU4nu1Idr+49ggAshWiIwxrRcx/Jg7zLYsxwOfAXxnSFlwNevxF4QHhm8+KoroeQgFO93qnCKD9R5P+CbfgCqyk5arCqqA0Ux3cmlL5tjRrG6pAM7qlLYo10ojUrinLQEhqQlcE5aAhPSEshIiiMsTFz7GJYIjDEtR+Fe2LPs61f+Nmd6eAx0GQL718CG1wFfq8lh4ZDYGzrVJIf+zntSHwiPAqCiysuh4jIOFpdxoqKaiiqv86p2hst94zXvFVXVhJUXEVmWT0T5UaIqCoipPEpMRSEdq4/QofoIHauc9/jqQsI4uQXnSomkKDyZoohkisN7UZwwnOLIThR4kvmqJIEl+e04UhYLRRAfFc7gtPYMGZzAzb6Dv9sHfX8sERhjgkMV8rZ8/Yt/zzIoznXmRSdA+kgYehP0GA2pmV//8q844SSIvC3o4U1UHtyI5q4lcuObCE4VSzVhHPB0ZZs3jQ2VqWz1dmOndiGaCpKkmCQpIZFi33BxneESOlJChFT7DblY4smXJA6FJbEhrCdHPInkSRJHwhLJI4lDkkiRxuPFqe3RckXLwetr7r9Xcjuu7Z9Q+4u/R2JswA/6/lgiMMY0zOuFPZ/C6nmw+R3whEN0B4jp4Byso33vdcdjOp4yryqyPccrlOoDX+HJ+ZyI3OVE7V+Bp+woAJUxKZR0Hk5R/zs5mpJFcXxfKrxQVa1U5XupPHy49pf9/sJS9heVsr+wE/sL4ymvygIgigp6yQEGhu9naPQhBnj2Mdibw1jNJgz/B3ZvZDwam4TGpSCx6YS1S0biUiAuGWKTIS4J4lKc4dgk2kdE0x7oGaj9HyCuJgIRmQg8CXiAZ1X18XrznwAu8Y3GAp1UtYObMRljmqDkINWrX0K+fImwwl14I9tzrM8kKiQS74kiKCsgrLAIT0UuEZXFRFWVEOEtb3B14UCchhEuzi/2Pd5OrPCewwodwArvAPaUdYYCgc0AxcAqv+sRgU7xUXTtEMOgru25bGAnunaIoWuHGNI6xJCaEE1iXCQidX5lV5VD/g44ugMiYp2DfFwKxCYR5qs+CnXiVg9lIuIBtgITgFxgJTBdVTc2UP5eYKiq3n669WZlZak1MWFM0+SVlLPlYAmbDxazI+84JyqqKKuspqzSS3mV815W6dSVV1ZUMKxqFZOqFzGG1YSLl8+9A5lfdQnveodTTsMXZWMjPSRGKalR5XSKKKNTRCnJ4WUkek7QQU6QIMeJlXJKEvpzNGkYlXGpRHiE8LAwwj1SbziM8DDfu296hEeIDA8jKS6KyHB7DvZMiMgqVc3yN8/NM4LhwHZV3ekLYj4wGfCbCIDpwKMuxmNMm1VaUc3WQyW+g75z4N9ysIT84xW1ZTrGRtA+JoLocA9REWFEh3uIjw5nYFQ+4068x4Vl75HgzedYRCLZnW5ia+pkyhJ6MjTcw8iIMKIjPLSLCqddVDjx0RHER4fTPjqCuCgP4R47OLdmbiaCNCCnznguMMJfQRHpgVPt9lED82cCMwHS09ObN0pjWrLqSjiw1qmnP7gOb0wiBRFd2FudyKYTHVhdHM+qvDB2Hz1Bzcl9dEQY/TrHM35gJ/p3ac+ALvH07xJPcrs61SCVZbD5LVj9IuxaChIGfSbA+bfQrt8VXOiJ4MLgfGITBC3lYvE0YIGq+r2io6pzgbngVA0FMjBjAqmyvJSCbZ9TsfNTInOX0zF/NRHVpQDkhaUQU32MJCklCRgK3ACUSzTHOnTB274bUckZtOvUk7CO6ZDQHTokQXxHCPM4Gzi0wbnw+9V8KCuEDulwyc/gvBsgIS1YH9sEmZuJYB/Qvc54N980f6YBP3AxFmOCruaulwNFZRwoKuVgURl5BUXEHF5F14LV9C1byzneLXSSSgA2e7vzrvci1oQNJif+PGISu9InJY7MJBgUW0iP8Hyiju8nqjCHqKK9UJgDu96FDfknbzgsHNp3dS6U5m0GTyQM/A6cfwtkjIEwq9YJdW4mgpVAXxHpiZMApuH8gDmJiAwAOgLLXYzFmGanqhSXVnHkeDn5xyrIP1bOkePOe/6xCvKPl3PENz3/eAWFJyqJpYxhYVsZEbaJEWGbOE92ECHVeAljf3QfNnS8jhOpI5CMUaR06srVCdHcEh1+8l0wjak4DkW5TmKoSRBFOXAiH4bdBudOhdhEt3aLaYVcSwSqWiUi9wDv49w++pyqbhCRx4BsVV3oKzoNmK9u3b5kQpuqc1A8ssV5alU8EB7tPJwUHu38Og6Pdp5CDY8Cj/OunkiOlAs5RdXsLKxi79EycgtKOVJSyvFjxZQfL6TieDHRWkqclNGOUuIoI07KiKOUbhEVZEZU0DG8gg6ecuLjykiILSLl+FbCtBoVD97U8/Bk/AAyLiIs/UK6RSfQrTk+c2Sc7wnb5m+l0rRNrt0+6ha7fdT4VV3lNOSVt8U56OdtdapBjmyDyuNnvfpK9eAVD1FUNF64RmQ75xXle4/pAGlZ0GMUdB/hTDcmQIJ1+6gxza+y1Dm4H9nqHPTzNjvD+TvAW/l1ufZpkNwPzr+Z4+17s5M01pcmsa/gOIeOFnGksJiC4hI83koipYooKojzVNO1nZAaG0anOOgUIyRFe+kYBQnh1URQBRFxXx/Yo+J9B/uaafFfz4uItbp302pYIjAtW/F+2LkEdi2BvcuhYA+1DY5JGHTs6VSB9JuIN7kfhyJ7sL68M2uPeNm4v5iNa4o5UFTT8mM+8dHh9EhKoUf3DAYmxtIjKZb0xDh6JMXSpX10i2j3xZhAs0RgWpYTR2H3J18f/PO3O9NjEiFjNGROh+R+VCT2ZVtlJzYcrmDjgWI27ihm06fFlJQXAUV4woTeKXGM6JnIoK7tGZSawIDUeJLqNz9gjLFEYIKs/JjzS3/XEufgf3AdoE71So9RMGwG1Rlj2FDdjRW7C2t/5W8/nEOVdy8AcZEeBqa255rz0xiU2p5BXdvTr3M80RGe4H42Y1oJSwQmsKoqIHelc+DftdQZ9lY5d+90Gw6XPIT2HMO28H4s21XEZ9vz+fxfBygpcx5S79w+ikGp7Rk/sBODuyYwKLU96S2kKV9jWitLBMZ9qrDlHVj5V+fXf+UJp34/9TwYeQ/acyw57Ybw2d5Slu3IZ/knRzhyzHmsJD0xlm8PSWVk7yRG9kqiU/voIH8YY9oeSwTGPaqw+W1Y8hs4uBYS0p2ORnqO5VBiFsv2V7Fsez7LFuSzr3AF4DQxfFGfZEb1TmZk7yS6J8YG+UMY0/ZZIjDNz+uFLTUJYB0k9uL4lX/kk+hL+GxXEZ+9c4SdeU578wkxEYzslcTdY3sxsncyvVPi7GKuMQFmicA0H6/XadFyyW/g0HpOxGfwUa9HeaZwGGtfP4bqWmIjPQzvmcj0C9IZ2TuJQantrX7fmCCzRGDOntdL9aaFVCz6L2IKNnPAk8Yfqr/P/+WNJOxoOEO7R3L/+H6M7pNEZvcORFjb9ca0KJYIzBlRVXbllZDz2Sv02TSHtIpd7Pem8seq77M15XJG9e3Ms32TGZ6RSFyUfc2MacnsP9Q02eGSMpZtz+ezbYeJ3PoWN1e8wtiwHPZIGv/o9gixw6bwcJ/OpMRbP7DGtCaWCEyDvF5l/f4iFm06zIebDrFpfyHfCvuC+yNfpw85FMX3JG/0HNJHTKOHx75KxrRW9t9rTlJaUc1n24/w4eZDLN50kNhjexgctpu7Eg4yJnEViSd2ocn9YexfSRh8zdc9XxljWi1LBIZDxWV8tGEfm9etpCL3S/p5d3GdZw+Phu0hOsrpJpHySEjNhG/9HBl0tSUAY9oQSwQhSCtOsHPDCnavX05l7peklm7jWskhSiohDKojY5EuQwjrerNz8E/NhOT+Tmcuxpg2xxJBKCjeT9W2D8lb9yFy8CuSy3bTGy+9gRJpR2HiII73mEBk7wuQrufhSexlv/iNCSGWCNqiylLY8xnsWIx32yLCjmwmHAjX9mygDycSbyChVxYDz7+YxK69ibcneY0JaZYI2gJVOLwJdnwI2z+EPcugupwqiSRbB/Bh5Q3kd76IiZdeytgBnYgKt1/7xpivWSJorY7nw87FsOMj51VyAIDKxH6sSrqa5w72YmlFP0YP6M7d43pzQUZikAM2xrRUriYCEZkIPAl4gGdV9XE/Za4HZuP0P/iVqt7gZkytltcLOZ87v/h3fAj71wAK0R2g9yUcShnF3P0ZvLi+CgUmZ3bljbG96d8lPtiRG2NaONcSgYh4gDnABCAXWCkiC1V1Y50yfYGfAqNVtUBEOrkVT6t2cB289SDkrgDxQLcLYNxPoc94Vldl8Oelu/ngvUNER1Rz04U9uPPinnTraM03G2Oaxs0zguHAdlXdCSAi84HJwMY6Ze4C5qhqAYCqHnYxntanvAQW/xq++IvTZ+93noJBk9HoBD7emsef397Bil1fkBATwQ/H9+XWkT1IamfNOxhjvhk3E0EakFNnPBcYUa9MPwAR+Qyn+mi2qr5Xf0UiMhOYCZCenu5KsC2KKmx4Dd5/CEoOQtYMuPTnVEV14O11B/jzx2vZfLCE1IRofn7VIKZd0N0adjPGnLFgHz3Cgb7AOKAbsFREhqhqYd1CqjoXmAuQlZWlgQ4yoPJ3wDs/ci4AdzkXpr6Epg1j4Vf7+f2/1pBztJQ+ndrxu+vOZfJ5aUSGW5POxpiz42Yi2Ad0rzPezTetrlzgC1WtBHaJyFacxLDSxbhapsoy+PQJ5xUeBVf+Fi64k+1HTvDIs1+wbEc+g7u2Z+7Ng7hsYGfrzMUY02zcTAQrgb4i0hMnAUwD6t8R9DowHXheRJJxqop2uhhTy7R9Ebz9IyjYBedcB1f8Jyeiknnq/W389dOdxER4+OXV53DD8HQ8lgCMMc3MtUSgqlUicg/wPk79/3OqukFEHgOyVXWhb97lIrIRqAb+Q1Xz3YqpxSneD+/Ngo1vQFIfuOUNtOdY3t9wkMfeXML+ojKuG9aNWVcOINkuAhtjXCKqravKPSsrS7Ozs4MdxtmproIVTzt3BHmr4OIfwegfsruwikcXbmDJ1jwGdInnl1efYw+CGWOahYisUtUsf/OCfbE49Oz9At5+EA6th76Xw5W/pSw+nT8t3sFfluwg0hPGz68axK0jexBuffsaYwLAEkGgHD8Ci2bDl3+D9mkw9SUYcBUfbTnM7GeXsvfoCSZlduXhbw+kc/voYEdrjAkhlgjcVlnmPBD2yX9D5QkY9UMY+xNyT4Txi7+t4oONh+idEsfLd45gVJ/kYEdrjAlBlgjcogob/s85CyjcC/0mwoTHKO/Yh2c/2cUfP9qGIPxk4gDuuKinPQ9gjAkaSwRuyFnhPBWcuxI6D4Fb3oBe4/h02xEeefETdh45zsTBXfj5dwaR1iEm2NEaY0KcJYLmVLDbOQPY8Bq06wKT50DmdFTC+P37m5mzeAc9kmJ5YcYFjOtv7esZY1oGSwTNobQQPvk9fPE0hIXD2Fkw+ocQGYeq8p9vb+LZT3cx7YLuzJ40mOgI6xjGGNNyWCI4G9WVkP08fPxfUFoA590Ilz4M7bsC4PUqv3hzAy8u38OtI3swe9JgxLqFNMa0MJYIzoQqbHkXPngE8rdBzzFw+X9C6rm1Rbxe5eHX1/OPFXu56+KePPStgZYEjDEtkiWCb+rAV/D+w7D7E0juB9NfgX5XQJ2DfLVX+cmra1mwKpcfXNKbH13e35KAMabFskTQVCeOwr9+BmtehthE+NbvYdht4Ik4qVhVtZd//+dXvLFmP/df1pf7xve1JGCMadEsETSFKrz+Pae/4FH3wsX/DjEdTilWWe3l/vlreHvdAf7jiv784JI+QQjWGGO+GUsETfHVfNj6Hlzxaxj5A79FyququeflL/lg4yF+9u2B3HlxrwAHaYwxZ8YSQWOKD8B7P4HuF8KIu/0WKaus5nsvrWLxljx+MWkwt47KCGyMxhhzFiwRnI4qvHkfVFXA1X+CsFPv/y+tqGbm37L5dPsRfn3NEG4YEQJ9Khtj2hRLBKfz1T9g2/twxX9BUu9TZh8vr+KOF1fyxa6j/Pa75zIlq7uflRhjTMtmiaAhxfvh3VmQPtJvlVBJWSUznl/J6r0FPHH9eVw9NC0IQRpjzNmzROBPTZVQdYXTXlDYyS2DFpVWcutzK1i/r4g/Tj+fb5+bGqRAjTHm7Fki8GfNy7DtXzDx8VOqhApPVHDTX79gy8ES/nTj+Vw+uEuQgjTGmOZhiaC+4v3w3k8hfRQM/7eTZuUfK+fGZ79g55HjzL05i0sGWAuixpjWz9XeUERkoohsEZHtIjLLz/zbRCRPRNb4Xne6GU+jVGHhD31VQv/vpCqhI8fKmTb3c3YdOc6zt1gSMMa0Ha6dEYiIB5gDTABygZUislBVN9Yr+oqq3uNWHN/Imr/D9g9g4m9OqRL6wwdb2ZN/ghduv4BRva1LSWNM2+HmGcFwYLuq7lTVCmA+MNnF7Z2don1OlVCP0TB85kmz8krKWbAql+uyulkSMMa0OW4mgjQgp854rm9afd8VkbUiskBE/N6ILyIzRSRbRLLz8vKaP1JVePOH4K06pUoI4MVlu6ms9nKXNRthjGmDgt1j+ptAhqqeC3wAvOivkKrOVdUsVc1KSUlp/ii+fAm2L4LLZkPiyQf74+VV/O3zPUwc3IWeyXHNv21jjAkyNxPBPqDuL/xuvmm1VDVfVct9o88Cw1yMx7+ifU5H8z0uggvuOmX2KytzKCqtZOYYOxswxrRNbiaClUBfEekpIpHANGBh3QIiUvdJrEnAJhfjOVUjVUKV1V7++ukuhvdMZGh6x4CGZowxgeLaXUOqWiUi9wDvAx7gOVXdICKPAdmquhD4oYhMAqqAo8BtbsXjV02V0JW/g8Sep8x+e+0B9hWW8surBwc0LGOMCSRR1WDH8I1kZWVpdnb22a+oKBf+NBJSM+GWhaecDagqVz75CV5V3rtvDGFh1suYMab1EpFVqprlb16wLxYHR82DY95qmPTHU5IAwNJtR9h8sISZY3pbEjDGtGmh2cTEl3+DHR86/Q77qRICeHrJDrq0j2ZSZtcAB2eMMYEVemcERbnw/sOQcTFk3eG3yLrcIpbtyOf2izKIDA+9XWSMCS2hdZSrWyXk5y6hGk8v3UF8VDjTh1tvY8aYti+0EsHqeU6V0IRfQMcMv0X25p/gnXUHuPHCHsRHRwQ2PmOMCYLQSQSFOY1WCQE8++lOwsPCmDE6I3CxGWNMEIVOIljzMqj3tFVC+cfK+d/sHK4e2pXO7aMDHKAxxgRH6Nw1NPbHMOS6BquEAOYt30NZpdeakzDGhJTQOSMQOaWPgbpKK6qZt3w3lw3sTJ9O8YGLyxhjgix0EkEj/rkqh4ITldw91s4GjDGhxRIBUFXt5ZlPdjKsR0eyMhKDHY4xxgSUJQLg3fUHyTlaatcGjDEhqckXi0VkFJBRdxlVnedCTAGlqjy9dAe9kuOYMLBzsMMxxpiAa1IiEJG/Ab2BNUC1b7ICrT4RLNuRz/p9xTx+7RBrXM4YE5KaekaQBQzS1tZmdRP8ZckOUuKjuHqov+6UjTGm7WvqNYL1QBc3AwmGjfuL+WTbEWaMziA6whPscIwxJiiaekaQDGwUkRVATR/DqOokV6IKkLlLdxAX6eHGET2CHYoxxgRNUxPBbDeDCIbcghO8ufYAM0ZlkBBjjcsZY0JXkxKBqi4RkR5AX1VdJCKxOP0Qt1p//XQXAtx+kf+OaYwxJlQ06RqBiNwFLACe9k1KA153Kyi3FRyvYP6KHCad15WuHWKCHY4xxgRVUy8W/wAYDRQDqOo2oFNjC4nIRBHZIiLbRWTWacp9V0RURPx2rNzcXvp8D6WV1fYAmTHG0PREUK6qFTUjIhKO8xxBg0TEA8wBrgQGAdNFZJCfcvHAfcAXTQ36bJRVVvPCst2M65/CgC7tA7FJY4xp0ZqaCJaIyENAjIhMAP4JvNnIMsOB7aq605dE5gOT/ZT7JfAboKyJsZyVV1fnkn+8gn8b03BLpMYYE0qamghmAXnAOuDfgHdU9eFGlkkDcuqM5/qm1RKR84Huqvr26VYkIjNFJFtEsvPy8poY8qmqvcozS3eS2S2BC3tZ43LGGANNTwSzVfUZVZ2iqtcBz4nI389mwyISBvwB+PfGyqrqXFXNUtWslJSUM97mvzYcZHf+Cf5tbG9ErDkJY4yBpieC7iLyUwARiQReBbY1ssw+oHud8W6+aTXigXOAj0VkN3AhsNCtC8aqyl+W7KBHUixXDG5zD0kbY8wZa2oiuB0Y4ksGbwFLVHV2I8usBPqKSE9f8pgGLKyZqapFqpqsqhmqmgF8DkxS1exv+iGa4otdR/kqt4i7Lu6FxxqXM8aYWqdNBCJyvq8efyjwJDAV50xgiW96g1S1CrgHeB/YBPyvqm4QkcdEJOBNU2w7fIyuCdFcN6xboDdtjDEtmpyuQVERWXyaZVVVL23+kE4vKytLs7PP7KShospLZLj1xWOMCT0iskpV/Va9n7aJCVW9xJ2QgsOSgDHGnKqpTUwkiMgfam7hFJH/FpEEt4Mzxhjjvqb+RH4OKAGu972KgefdCsoYY0zgNLUZ6t6q+t06478QkTVuBGSMMSawmnpGUCoiF9WMiMhooNSdkIwxxgRSU88I7gbm1bkuUADc6k5IxhhjAqmpiaBYVTNFpD2AqhaLiPXoYowxbUBTq4ZeBScBqGqxb9oCd0IyxhgTSKc9IxCRAcBgIEFErq0zqz0Q7WZgxhhjAqOxqqH+wFVAB+A7daaXAHe5FZQxxpjAaSwRxAI/Auaq6vIAxGOMMSbAGksE6Ti9kUWIyIfAu8AKPV0DRcYYY1qV014sVtXf+BqW+xbwFU5z1KtF5GURuUVEOgciSGOMMe5p0u2jqloCvOZ74euE/kpgHnCFa9EZY4xxXWP9EdxUZ3h0zbCqbgTKVdWSgDHGtHKNPUfwYJ3hP9abd3szx2KMMSYIGksE0sCwv3FjjDGtUGOJQBsY9jdujDGmFWrsYvEAEVmL8+u/t28Y33gvVyMzxhgTEI0lgkygM5BTb3p34KArERljjAmoxqqGngCKVHVP3RdQ5Jt3WiIyUUS2iMh2EZnlZ/7dIrJORNaIyKe+21KNMcYEUGOJoLOqrqs/0Tct43QLiogHmIPzvMEgYLqfA/3LqjpEVc8Dfgv8oamBG2OMaR6NJYIOp5kX08iyw4HtqrpTVSuA+cDkugXqNGkNEIddgDbGmIBrLBFki8gprYyKyJ3AqkaWTePkawu5vmn11/UDEdmBc0bww0bWaYwxppk1drH4fuA1EbmRrw/8WUAkcE1zBKCqc4A5InID8DP8dIEpIjOBmQDp6enNsVljjDE+p00EqnoIGCUilwDn+Ca/raofNWHd+3DuLqrRzTetIfOBPzcQx1xgLkBWVpZVHxljTDNqaqNzi4HF33DdK4G+vr6N9wHTgBvqFhCRvqq6zTf6bWAbxhhjAqqpndd/Y6paJSL3AO8DHuA5Vd0gIo8B2aq6ELhHRC4DKoEC/FQLGWOMcZdriQBAVd8B3qk37ZE6w/e5uX1jjDGNa+yuIWOMMW2cJQJjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCY4wJcZYIjDEmxFkiMMaYEGeJwBhjQpwlAmOMCXGWCIwxJsRZIjDGmBBnicAYY0KcJQJjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCY4wJcZYIjDEmxFkiMMaYEOdqIhCRiSKyRUS2i8gsP/MfFJGNIrJWRD4UkR5uxmOMMeZUriUCEfEAc4ArgUHAdBEZVK/Yl0CWqp4LLAB+61Y8xhhj/HPzjGA4sF1Vd6pqBTAfmFy3gKouVtUTvtHPgW4uxmOMMcYPNxNBGpBTZzzXN60hdwDvuhiPMcYYP8KDHQCAiNwEZAFjG5g/E5gJkJ6eHsDIjDGm7XPzjGAf0L3OeDfftJOIyGXAw8AkVS33tyJVnauqWaqalZKS4kqwxhgTqtxMBCuBviLSU0QigWnAwroFRGQo8DROEjjsYizGGGMa4FoiUNUq4B7gfWAT8L+qukFEHhORSb5ivwPaAf8UkTUisrCB1RljjHGJq9cIVPUd4J160x6pM3yZm9s3xhjTOHuy2BhjQpwlAmOMCXGWCIwxJsRZIjDGmBBnicAYY0KcJQJjjAlxlgiMMSbEWSIwxpgQZ4nAGGNCnCUCY4wJcZYIjDEmxFkiMMaYEGeJwBhjQpwlAmOMCXEtoqvKs1VZWUlubi5lZWXBDsUA0dHRdOvWjYiIiGCHYoxpgjaRCHJzc4mPjycjIwMRCXY4IU1Vyc/PJzc3l549ewY7HGNME7SJqqGysjKSkpIsCbQAIkJSUpKdnRnTirSJRABYEmhB7G9hTOvSZhKBMcaYM2OJwBhjQpwlglamqqoq2CEYY9oYV+8aEpGJwJOAB3hWVR+vN38M8D/AucA0VV1wttv8xZsb2Li/+GxXc5JBXdvz6HcGN1ru6quvJicnh7KyMu6779KuD70AAA+qSURBVD5mzpzJe++9x0MPPUR1dTXJycl8+OGHHDt2jHvvvZfs7GxEhEcffZTvfve7tGvXjmPHjgGwYMEC3nrrLV544QVuu+02oqOj+fLLLxk9ejTTpk3jvvvuo6ysjJiYGJ5//nn69+9PdXU1P/nJT3jvvfcICwvjrrvuYvDgwTz11FO8/vrrAHzwwQf86U9/4rXXXmvWfWSMab1cSwQi4gHmABOAXGCliCxU1Y11iu0FbgN+5FYcgfTcc8+RmJhIaWkpF1xwAZMnT+auu+5i6dKl9OzZk6NHjwLwy1/+koSEBNatWwdAQUFBo+vOzc1l2bJleDweiouL+eSTTwgPD2fRokU89NBDvPrqq8ydO5fdu3ezZs0awsPDOXr0KB07duT73/8+eXl5pKSk8Pzzz3P77be7uh+MMa2Lm2cEw4HtqroTQETmA5OB2kSgqrt987zNtdGm/HJ3y1NPPVX7SzsnJ4e5c+cyZsyY2vvpExMTAVi0aBHz58+vXa5jx46NrnvKlCl4PB4AioqKuPXWW9m2bRsiQmVlZe167777bsLDw0/a3s0338xLL73EjBkzWL58OfPmzWumT2yMaQvcTARpQE6d8VxgxJmsSERmAjMB0tPTzz4yF3z88ccsWrSI5cuXExsby7hx4zjvvPPYvHlzk9dR97bL+vfhx8XF1Q7//Oc/55JLLuG1115j9+7djBs37rTrnTFjBt/5zneIjo5mypQptYnCGGOglVwsVtW5qpqlqlkpKSnBDsevoqIiOnbsSGxsLJs3b+bzzz+nrKyMpUuXsmvXLoDaqqEJEyYwZ86c2mVrqoY6d+7Mpk2b8Hq9p63DLyoqIi0tDYAXXnihdvqECRN4+umnay8o12yva9eudO3alV/96lfMmDGj+T60MaZNcDMR7AO61xnv5pvWJk2cOJGqqioGDhzIrFmzuPDCC0lJSWHu3Llce+21ZGZmMnXqVAB+9rOfUVBQwDnnnENmZiaLFy8G4PHHH+eqq65i1KhRpKamNritH//4x/z0pz9l6NChJ91FdOedd5Kens65555LZmYmL7/8cu28G2+8ke7duzNw4ECX9oAxprUSVXVnxSLhwFZgPE4CWAncoKob/JR9AXirKXcNZWVlaXZ29knTNm3aZAe4Rtxzzz0MHTqUO+64IyDbs7+JMS2LiKxS1Sx/81w7I1DVKuAe4H1gE/C/qrpBRB4TkUm+wC4QkVxgCvC0iJySJMzZGzZsGGvXruWmm24KdijGmBbI1auGqvoO8E69aY/UGV6JU2VkXLRq1apgh2CMacFaxcViY4wx7rFEYIwxIc4SgTHGhDhLBMYYE+IsERhjTIizRBAE7dq1C3YIxhhTq+01OvPuLDi4rnnX2WUIXPl44+VamaqqKmt3yBhjZwTNYdasWSe1HTR79mx+9atfMX78eM4//3yGDBnCG2+80aR1HTt2rMHl5s2bV9t8xM033wzAoUOHuOaaa8jMzCQzM5Nly5axe/duzjnnnNrlfv/73zN79mwAxo0bx/33309WVhZPPvkkb775JiNGjGDo0KFcdtllHDp0qDaOGTNmMGTIEM4991xeffVVnnvuOe6///7a9T7zzDM88MADZ7zfjDEthKq2qtewYcO0vo0bN54yLZBWr16tY8aMqR0fOHCg7t27V4uKilRVNS8vT3v37q1er1dVVePi4hpcV2Vlpd/l1q9fr3379tW8vDxVVc3Pz1dV1euvv16feOIJVVWtqqrSwsJC3bVrlw4ePLh2nb/73e/00UcfVVXVsWPH6ve+973aeUePHq2N65lnntEHH3xQVVV//OMf63333XdSuZKSEu3Vq5dWVFSoqurIkSN17dq1fj9HsP8mxpiTAdnawHHV6gWawdChQzl8+DD79+8nLy+Pjh070qVLFx544AGWLl1KWFgY+/bt49ChQ3Tp0uW061JVHnrooVOW++ijj5gyZQrJycnA130NfPTRR7X9C3g8HhISEhrt6Kam8TtwOryZOnUqBw4coKKiorbvhIb6TLj00kt56623GDhwIJWVlQwZMuQb7i1jTEtjiaCZTJkyhQULFnDw4EGmTp3K3//+d/Ly8li1ahURERFkZGSc0seAP2e6XF3h4eF4vV/39XO6vg3uvfdeHnzwQSZNmsTHH39cW4XUkDvvvJNf//rXDBgwwJq0NqaNsGsEzWTq1KnMnz+fBQsWMGXKFIqKiujUqRMREREsXryYPXv2NGk9DS136aWX8s9//pP8/Hzg674Gxo8fz5///GcAqqurKSoqonPnzhw+fJj8/HzKy8t56623Tru9mr4NXnzxxdrpDfWZMGLECHJycnj55ZeZPn16U3ePMaYFs0TQTAYPHkxJSQlpaWmkpqZy4403kp2dzZAhQ5g3bx4DBgxo0noaWm7w4ME8/PDDjB07lszMTB588EEAnnzySRYvXsyQIUMYNmwYGzduJCIigkceeYThw4czYcKE02579uzZTJkyhWHDhtVWO0HDfSYAXH/99YwePbpJXWwaY1o+1/ojcIv1RxB8V111FQ888ADjx49vsIz9TYxpWYLSH4FpewoLC+nXrx8xMTGnTQLGmNbFLhYHybp162qfBagRFRXFF198EaSIGtehQwe2bt0a7DCMMc2szSQCVUVEgh1Gkw0ZMoQ1a9YEOwxXtLbqRmNCXZuoGoqOjiY/P98OQC2AqpKfn090dHSwQzHGNFGbOCPo1q0bubm55OXlBTsUg5OYu3WzHkiNaS3aRCKIiIiofSLWGGPMN+Nq1ZCITBSRLSKyXURm+ZkfJSKv+OZ/ISIZbsZjjDHmVK4lAhHxAHOAK4FBwHQRGVSv2B1Agar2AZ4AfuNWPMYYY/xz84xgOLBdVXeqagUwH5hcr8xkoKZdgwXAeGlNt/4YY0wb4OY1gjQgp854LjCioTKqWiUiRUAScKRuIRGZCcz0jR4TkS1nGFNy/XW3MBbf2bH4zl5Lj9HiO3M9GprRKi4Wq+pcYO7ZrkdEsht6xLolsPjOjsV39lp6jBafO9ysGtoHdK8z3s03zW8ZEQkHEoB8F2MyxhhTj5uJYCXQV0R6ikgkMA1YWK/MQuBW3/B1wEdqT4UZY0xAuVY15Kvzvwd4H/AAz6nqBhF5DKfLtIXAX4G/ich24ChOsnDTWVcvucziOzsW39lr6TFafC5odc1QG2OMaV5toq0hY4wxZ84SgTHGhLg2mQhactMWItJdRBaLyEYR2SAi9/kpM05EikRkje/1SKDi821/t4is82072898EZGnfPtvrYicH8DY+tfZL2tEpFhE7q9XJuD7T0SeE5HDIrK+zrREEflARLb53v327Skit/rKbBORW/2VcSG234nIZt/f7zUR6dDAsqf9Lrgc42wR2Vfn7/itBpY97f+7i/G9Uie23SLit135QO3Ds6KqbeqFc2F6B9ALiAS+AgbVK/N94C++4WnAKwGMLxU43zccD2z1E9844K0g7sPdQPJp5n8LeBcQ4ELgiyD+rQ8CPYK9/4AxwPnA+jrTfgvM8g3PAn7jZ7lEYKfvvaNvuGMAYrscCPcN/8ZfbE35Lrgc42zgR034Dpz2/92t+OrN/2/gkWDuw7N5tcUzghbdtIWqHlDV1b7hEmATzhPWrclkYJ46Pgc6iEhqEOIYD+xQ1T1B2PZJVHUpzp1vddX9nr0IXO1n0SuAD1T1qKoWAB8AE92OTVX/papVvtHPcZ7zCZoG9l9TNOX//aydLj7fseN64B/Nvd1AaYuJwF/TFvUPtCc1bQHUNG0RUL4qqaGAv/4pR4rIVyLyrogMDmhgoMC/RGSVr3mP+pqyjwNhGg3/8wVz/9XorKoHfMMHgc5+yrSEfXk7zhmeP419F9x2j6/66rkGqtZawv67GDikqtsamB/sfdiotpgIWgURaQe8CtyvqsX1Zq/Gqe7IBP4IvB7g8C5S1fNxWo79gYiMCfD2G+V7SHES8E8/s4O9/06hTh1Bi7tXW0QeBqqAvzdQJJjfhT8DvYHzgAM41S8t0XROfzbQ4v+f2mIiaPFNW4hIBE4S+Luq/l/9+aparKrHfMPvABEikhyo+FR1n+/9MPAazul3XU3Zx267Elitqofqzwj2/qvjUE2Vme/9sJ8yQduXInIbcBVwoy9RnaIJ3wXXqOohVa1WVS/wTAPbDup30Xf8uBZ4paEywdyHTdUWE0GLbtrCV5/4V2CTqv6hgTJdaq5ZiMhwnL9TQBKViMSJSHzNMM5FxfX1ii0EbvHdPXQhUFSnCiRQGvwVFsz9V0/d79mtwBt+yrwPXC4iHX1VH5f7prlKRCYCPwYmqeqJBso05bvgZox1rztd08C2m/L/7qbLgM2qmutvZrD3YZMF+2q1Gy+cu1q24txN8LBv2mM4X3qAaJwqhe3ACqBXAGO7CKeKYC2wxvf6FnA3cLevzD3ABpw7ID4HRgUwvl6+7X7li6Fm/9WNT3A6HdoBrAOyAvz3jcM5sCfUmRbU/YeTlA4AlTj11HfgXHf6ENgGLAISfWWzgGfrLHu777u4HZgRoNi249St13wHa+6i6wq8c7rvQgD3399836+1OAf31Pox+sZP+X8PRHy+6S/UfO/qlA3KPjyblzUxYYwxIa4tVg0ZY4z5BiwRGGNMiLNEYIwxIc4SgTHGhDhLBMYYE+IsEZiQJiLV9VozbbbWK0Uko25rlU0oHycii3zDn/oeVjLGdfZFM6GuVFXPC3YQPiOB5b4Hy47r143CGeMqOyMwxg9fG/K/9bUjv0JE+vimZ4jIR76G0D4UkXTf9M6+dv2/8r1G+VblEZFnxOl74l8iEuNnW719bdm/BNwArAIyfWconQL0kU0Is0RgQl1MvaqhqXXmFanqEOD/Af/jm/ZH4EVVPRenobanfNOfApao09Dd+ThPkQL0Beao6mCgEPhu/QBUdYfvrGQVTjs0L+I8uXqeOu3TGOMqe7LYhDQROaaq7fxM3w1cqqo7fY0EHlTVJBE5gtPUQaVv+gFVTRaRPKCbqpbXWUcGTl8DfX3jPwEiVPVXDcSyUlUvEJFXgfu0gfZrjGludkZgTMO0geFvorzOcDV+rsuJyF98F5X7+qqIJgJvicgDZ7hNY74RSwTGNGxqnfflvuFlOC1cAtwIfOIb/hD4HoCIeEQkoakbUdW7gV8Av8TpxextX7XQE2cXvjFNY3cNmVAXIyd3Ov6eqtbcQtpRRNbi/Kqf7pt2L/C8iPwHkAfM8E2/D5grInfg/PL/Hk5rlU01FpiH09vVkjP6JMacIbtGYIwfvmsEWap6JNixGOM2qxoyxpgQZ2cExhgT4uyMwBhjQpwlAmOMCXGWCIwxJsRZIjDGmBBnicAYY0Lc/wcAm9OQ9G1jmgAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### Translate"],"metadata":{"id":"YAT4UMHHxgvc"}},{"cell_type":"code","source":["result = model.translate(['Ich gehe heute nach Hause.'])\n","result[0].numpy().decode()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"id":"6h7xbHVzxmfl","executionInfo":{"status":"ok","timestamp":1669569043161,"user_tz":-60,"elapsed":565,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"58253e4a-c8b7-4688-e52a-5a24845dfdbf"},"execution_count":170,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'im going to be home today . '"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":170}]},{"cell_type":"markdown","source":["#### Attention Plot"],"metadata":{"id":"l8kwMkLjtsr6"}},{"cell_type":"code","source":["model.plot_attention('Ich gehe heute nach Hause.')"],"metadata":{"id":"CUU35xI3xyYO","colab":{"base_uri":"https://localhost:8080/","height":650},"executionInfo":{"status":"ok","timestamp":1669569043823,"user_tz":-60,"elapsed":671,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"8f452e39-d675-4a9d-a608-f840ef91ac7b"},"execution_count":171,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 720x720 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAmwAAAJ5CAYAAAATjmwNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd7wcZ33v8c9XlmS5UW3AkFBsqjHFIFzogYQeSgwhwTRTHIO5F3AMoYQLl1yqTSgBAg64UMMNJRAuJQlgimMwphpk04wbYHDDtiwXIf3uHzMn2nM4ko5k7c5ztJ/367Uv7ZSd+e1oz9nveWaeZ1JVSJIkqV1Lhi5AkiRJm2ZgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElq3NKhCxAk+bOteNlnq+qqbV6MJElqTryX6PCSrN/ClxRwu6o6axz1SJKkttjC1o6bVdVvFrJikivGXYwkSWqH17C14URgS05vfgC4fEy1SJKkxnhKVJIkqXG2sDUiybokNxm6DkmS1B4DWzsydAGSJKlNBjZJkqTG2Uu0LX+eZJOdCarqfZMqRpIktcFOB43ox2JbQzfG2sZUVV1vQiVJkqRGGNga0Qe2BY/FJkmSpofXsLXD5CxJkuZlYGuHvUQlaRtKcpckb0/y2SR79vMem2S/oWuTtpSBrR2bvNtBkpVJPjfBeiRp0UryEOCbwC2ABwE79Yv2Bl45VF0aTpIbbelj6JpHGdgaUVWHAgcmOTrJa5PsBZDk9kk+CXxj2AqlNiV5eJJPJ1mV5A/7ec9K8uCha9Og/g44sqoeB1w7Mv8kYP9BKtLQLgIu3ILHb2a+i1vgsB6NSPI04HjgEuBGwDOTPB94N/Bx4O5VdfqAJUrNSXII8C7gPcCDgWX9oh2AFwNfGKg0DW9f4DPzzJ/5Havp9Hi6z8DmhPk/P4MxsLXjhcDLqur1Sf4c+GfgRcA9qupnw5YmNevFwLOr6p+TPGtk/teBVw9Uk9pwCd3p0LPnzL8HcP7Eq1ELzgG+UlUXL2TlJGcBa8db0sJ5SrQdewMf6Z9/FFhH15xvWJM27nbAKfPMXw04ZuF0+xBwdJI/oOuFvzTJA4BjAAcgn0JVdZuFhrV+/X2r6rxx1rQlDGzt2AW4EqCq1gNXA818UNSOJLsnOSDJjkPX0oBfArefZ/79Af/YmW5/C/ycrlVlV2AV8EXga8BrBqxL2iqeEm3LI5Nc1j9fAjw0ya9HV6iqj0++LLUgyW7Ae+muwSi61qWzkrwLuKCqXjVgeUM5FnjbyOnQP0xyP+CNwKsGq0qDq6q1wCFJXkF3GnQJ8J2q+smwlWloSQI8FTgY2Ivu9+lZwL8AH6xG7yjgnQ4a0d/pYHOqqnYYezFqUpJ3AncDjqBrJbhrVZ2V5FHAa6rqboMWOJAkr6G7BnRFP+sa4JiqesVwValFSW4LnF9VVw9di4aT5OPAY4HT6VpeA+xD11HlE1V18IDlbZSBTVokkpwPPK6qvpnkCuBufWDbG/huVe02cImDSbIz3S/cJcCqqlo9cEkTl2QF8Hy63rI3Yc4lL1V11yHqGkqS1wI/qqoT+xaVf6c7NpcBD6sqh0qaQn3P8ncDf1ZV/z5n2UOBjwGHVdWHhqhvU7yGbRFJ8sdD16BB3RCY74LZ3eg6qUydJMcl2a2q1lTVaVV1alWtTrJLkuOGrm/C3gm8hK5X5L/SffGMPqbNIcCP+ucPB+4OHEjX4eD1QxWlwT0ZeMPcsAZQVZ8Hju7XaY4tbI1LcgvgUOAZwK08JTq9kpwE/GtVvaVvYbtrVf08yT/SfTYeMWyFk5dkHbBnVf1mzvzd6a7rm5rrdJNcAvx5Vf3n0LW0IMnVwG2r6vwkb6f7vjuiPy16WlXdYOASNYAkvwT+tKq+tZHlK4FPVdXNJ1vZ5k3NL7PFJMkOwGOAZwIPAb5PNzjovwxZlwb3MuDzSe5M97N7ZP98f7pekVOjv2VM+scNk/xuZPEOwCOBX8/32u3YGuxZPupi4FZ0Y649hK71EbqfHe/dPL1uDPxqE8t/RaMDKxvYGpLkDsCz6HqvXEk3jtBDgKdU1aoha9Pwquq/ktwbOIpuyIoHA98GDprCu2BcRNezq+guGp6rmL77Rb6RLsQf3movtwn7GPChJD+m+wL+fD//7sBPB6tKQ1vGpgfD/R0b7pjSFE+JNiLJV+l6qHwMeH9Vfbmfv5bu4nIDm9TrB0AN3bhaBzP7VjPXAudU1S+HqG2Sknxqzqz7011Uv4o5X0pV9ehJ1dWCJEvpOmHcEjihqr7Tz38hcEVVvWfI+jSMfkSG4+hapOezM3Boi5cfGdga0Z/SeQdwbFX9cGS+gU2zJLk58/cC/PYwFQ0nya2Ac6e1RSnJ8Qtdt6oOHWct0mLQXwu82d8XVfVH469myxjYGpFkP7rToU+i6+X1PuDDdNekGNg08xn5AHBHfv8anKkcoy/JPTa1fBpDrDp+NrS9MbA1ph9L6Ql0vULvS9eK8hLgPVV16ZC1aVhJvkl3IfWr6W7JNOuHt6rOGaKuIfWnN4rZAfa/j8s0hdi+A8oOVfX9OfPvCvxu2v7o87Oh7Y2BrRFJbgmcN3pqp+9+PtMJ4cbAF6vq4QOVqIEluRLYr6p+PHQtrehPiY5aBuwHvBx4aVV9dvJVDSPJycA75g74meQvgOdV1X2HqWwYfjY0nyRHLmS9qvr7cdeypQxsjdjYeFL9sh2ARwHPqKrHTLw4NSHJ14EXV9VXhq6ldUkeAryyqu4zdC2T0o/Nt19V/XTO/L2Bb1fV9YeprC3T+NnQBkl+vonFBdwM2LHFFliH9WjHRscFqqp1wCf7h6ZIP97YjJcBb0zyt3T3wJvbC3C0p+S0+znd8A3TZB0wXyi7IY47NmoaPxvqVdVt5pufZC/gNXSXJDU55qmBTWrbzHhjM2buiTh3XtENGDtV5gRa6I7FnsCr2HBbomnxZeDlSZ7Q/5E3M7TFy4Gpa5X1s6GFSHJj4BXA4cDJwIFVddqwVc3PwNaWo5Js8qbVVfXqSRXTiv7G3ndn/qEsPj5IUZPTXNfyxswNtNB9MZ8HPHHy5QzqxcDXgJ8m+Vo/777ArkzZnTB6fja0UUl2Ao6k+7k5G3hc69c1eg1bI/oeTT+iG2V5Y6qq7jqhkprQ3/D+w3SdLuaayqEstEE/gO6o9cCFwE+ralM/S9ulJHsCz2PDKb/vAO+chkGE5/KzofkkWUJ328f/TXdZySvoBqtvPgwZ2BrRB7abzdfpYJol+SHwTeBl0/ilM1eSuwB/BexN1wnlV0keSzey/3eGrU6S2pZkFd09Zt8G/ANw9XzrtXhNsIGtEZvqJTrN+qEs7lpVPxu6lqH1vds+BXwWeARwp6o6K8lfA/erqscOWuBADLGz9XfCuCWwfHT+tPYu9nhoVN84MmO+ABQaPXvjNWztsBfX/E4G7kB3s/Np93fAkVX1zn4IhxknAX89TEnDmhNiHwTs1C/aG3g6MDUhtg8mH6K7Xm1mwNjRL6TmvoDGyeOhjVi01wUb2Nrxv4FNdjiYFnNuKfMu4Jj+l+98Q1lM0+1l9gU+M8/8S4C5PeKmhSF2g7fQDe2xD91lBA8Dbkp3Z4wXDljXUDwe+j1V9eWha9haBrZ2vImudWDNzIwkdwJeRNfL6+NV9c8D1TZpp/H7t5Q5dp71pm0oi0uAW9D1aBp1D+D8iVfTBkPsBg8AHllVZyYp4MKqOjnJNXTB9j+GLW/iPB76PUkOA06sqmv66TsDP5rpiJJkF+Bvqup/DVjmvJZsfhVNyD/StbIBkGR34Kt0dzi4A/DBJE8aqLZJuw2wV//vph57DVXgQD4EHJ3kD+jC6tK+J9wxwPsGrWw4MyF2rmkMsTvRDWUB3XG5Sf98FTBVvct7Hg/N5x+ZPcD0KXTXOM7YlW7swubYwtaOg+gunJ7xFOBaugvLL0vyBrru+h+a78Xbk2m8ifkC/S1wAnAOXevjKro/uj5IN0L3NJoJsX/O74fY4wetbPLOBO5I1wL7XeDwJOcBRwC/GLCuoXg8NJ+514svmuvH7SXaiL435J2r6ux++lN0vdz+Rz+9D/DlqtpjuConL8lrgPOq6l1z5h8O3KKqXjFMZcPpb6FyD7qw9p2q+snAJQ0myTK6EPsXdL9417MhxB46TeNtJTkEWFZVJ/TXgX4O2B24BnhaVf3fQQucMI+H5jN3CK3+2te7VdVZ/fRNgV+22EvUwNaIJBcCD6yqH/bTFwB/XVUf7Kf3Ak6vql0GLHPikpwLPKGqvjFn/r2Aj1bVrYapbPKSHLeRRUU3ltBPgY9M43h1htjf198h5I7AuVV10ebW3955PASLO7B5SrQd3wMOpbs91QOBPYAvjizfG5i6L2K6604unGf+xXQ9vqbJHsD96FqRftDP25euZelbwJ8Br05yv6r67jAlTl6SJwIPZsOty56cdGc5qurRA5Y2cfMci5n5U3cswOOhjXpkksv650uAhyb5dT99g4Fq2iwDWzv+Dvhsfy3OHsAJVfWrkeWPo7tP4LQ5ly6knDVn/v2ZvovKT6Yb+uWZVbUG/rvV4J/oAv8j6DofvInuS2q7l+Ro4AXAl+j+oJnaUwYei9k8HtqE986Zfsec6SY/K54SbUg/jMdDgAuAf6mq9SPLDgNOnaaWE4B+FP+XA3/DhhbHBwOvA95QVW8cqrZJS/Ir4EFVdcac+fsAX6iqPZPsB/xnVc1379XtTv9X8RFV9dGhaxmax2I2j4e2N7awNSDJ/sC3+i/iM+Zbp6qOHVn/nsD3q2rtfOtuT6rqTf0QJ2+ju7VM6C4afitw9JC1DWBXYE9+/zNys34ZwOVM18/1EroegPJYzOXx0Cwj37XrFrh+U9+1jsPWhlPYskE+vwT84ZhqaU5VvZSud9eBwAHAHlX1kpq+5uFPAO9N8oQkt+4fT6Br3v94v87+wI8Hq3DyjgWePHQRjfBYzObx0FyL+rt2mv4Sb1mA1yVZs9k1O8s3v8ri1Q9p8uSqurx/Pt86wNRdVH448PfAB9jws/s74DjgqH76DODZky9tcpK8bWRyCXBIkj8Bvs/v37rsf06ytoHdAHjSNB8LPxtbJ8kZwO2qanvPBIv6u3Z7/89ZLL5C1wt0oU4BrhpTLS24mA0XfV48ZCEt6TsaHN5f1zfzeflZVV05ss40nAK6y5zpmfd8xznzp60Fdh88Fn42ts47gGm47nVRf9fa6UCSJKlxXsMmSZLUOANbw/qhPNTzeMzm8ZjN4zGbx2M2j8dsHo/ZFsPxMLC1rfkP0IR5PGbzeMzm8ZjN4zGbx2M2j8dszR8PA5skSVLj7HQwj+VZUTst2XXzK47ZtXU1y7Ni6DK43b6rhy4BgAsvXsceNx7+frw/WXW9oUsA4Nr1V7F8yU5DlwGN/A65tq5ieYY/HrV+QWNyjt3auoZl2XHoMprpj7mWa1hGA8ejH5JoaGvrapY18P3SxtGAa7mG5Q18Pi6vSy6qqj3mW+awHvPYacmuHLjzo4Yuoxmf/fzJQ5fQlEfs95ChS2jLNdcMXUFT1q++cvMrTZFa30hia0SW+bU7Ko0E2Fb8+1UfOGdjyzwlKkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY1btIEtyQlJPj10HZIkSeO2dOgCroPnAxm6CEmSpHFbtIGtqi4bugZJkqRJ2C5OiSY5Kck/JnlTkkuSXJjk+Ul2TPKOJL9Ncm6SpwxdtyRJ0pZatIFtHocAVwAHAK8H3gL8K/BjYCVwIvCeJHsOVqEkSdJW2J4C2w+r6lVV9RPg74GLgLVV9daq+inwarpr3u4z34uTHJbktCSnXVtXT65qSZKkzdieAtv3Z55UVQG/AU4fmbcWuBS4yXwvrqpjq2plVa1cnhXjrlWSJGnBtqfAtnbOdG1k3vb0niVJ0hQwvEiSJDXOwCZJktQ4A5skSVLjFvPAuU8fef7AeZbvO8+8m423KkmSpG3PFjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElq3NKhC2hSFaxbN3QVzXjEPg8YuoSmvOV7nxi6hKa84C4PH7qEpuyw+42HLkENW3fpb4cuoSnrr7lm6BIWDVvYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYtisCW5IFJKsnuQ9ciSZI0aYsisAH/BewJXDx0IZIkSZO2dOgCFqKqrgUuGLoOSZKkIUykhS3JLknel2R1kl8neWmSTyc5oV9+wyQnJrk0yVVJ/jPJnUdeP+uUaJKn99t6cJIfJLkyyZeS3GbOfl/a7291v/9XJjl7Eu9ZkiRpW5nUKdE3AQ8AHgc8CLgbcL+R5ScABwCPAfYH1gCfS7LTJra5I/BS4BnAQcANgHfNLEzyF8ArgZcD9wDOAI7cJu9GkiRpgsZ+SjTJrnSh6qlV9R/9vGcC5/fPbwc8GnhAVX2ln/cU4FzgEOA9G9n0UuCIqvpR/5pjgOOSpKoKeD5wQlXNvP51Sf4IuP1G6jwMOAxgRXa5bm9akiRpG5pEC9vewDLg1JkZVXUl8IN+8k7AeuCUkeWXAacD+2xiu9fMhLXeL4HlwA376TuO7rP3jY1trKqOraqVVbVyOTtu8g1JkiRNUuu9RGsTy363kXVbf0+SJElbZBLh5mfAWuBeMzOS7Azs20+e0ddx0Mjy6wF3AVZdh/2eObrP3v7XYXuSJEmDGPs1bFW1OslxwBuSXAT8CvhbupBWVfWTJJ8E3t1fR/Zb4DXA5cCHrsOu3wocn+SbwFfpOjwcAFx6HbYpSZI0cZMah+0oYBfgU8Bq4M3ATYGr++WHAm/pl68ATgYeVlVXbe0Oq+qfk+wFvB7YGfg4XS/Sx2ztNiVJkoYwkcBWVauBp/QPkuwIvAD4TL/8UuBpm3j9SUBGpk+gGwpko+v0814LvHZmOskngJ9u/TuRJEmavIkEtiT70fUGPRXYDfib/t+PjHGfOwPPAT5H10HhYLrWtYPHtU9JkqRxmOStqY4E7kAXnr4L3L+qzh/j/gp4OPAyYCfgJ8CTq+oTY9ynJEnSNjepU6LfAVZOYl8j+7wK+ONJ7lOSJGkcHLNMkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElq3NKhC2hRAVU1dBnNqCuvGrqEphxxyBFDl9CUZTucM3QJTfnV4/YauoSmrNlz6Aracpv/e+nQJTQlZ507dAltuXLji2xhkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWrcdhPYkpyU5O1D1yFJkrStbTeBTZIkaXu1XQS2JCcADwCOSFL949ZJ7p/kG0muTvLrJG9OsnzgciVJkrbIdhHYgOcDpwDHA3v2j7XAZ4HvAPsBzwT+EnjdQDVKkiRtle0isFXVZcC1wJqquqCqLgCeC/wSeG5VnVFVnwZeAjwvyc5zt5HksCSnJTltbV090folSZI2ZbsIbBtxJ+DrVbV+ZN7XgOXAbeeuXFXHVtXKqlq5LCsmVaMkSdJmbc+BbVNq6AIkSZIWansKbNcCO4xMnwEcmGT0Pd63X+9nkyxMkiTputieAtvZwP5979DdgXcCNwfemeROSR4JvB54e1WtGbBOSZKkLbI9BbZj6FrPVgEXAsuAh9P1EP0ucBzwYeBlQxUoSZK0NZYOXcC2UlU/Bg6aM/ts4IDJVyNJkrTtbE8tbJIkSdslA5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNWzp0AS1KQpYvH7qMZtRVVw1dQlOW/+LSoUtoyno/H7Msv7yGLqEpv73j0BU0ZqntJKNy85sOXUJbfrLxRX5yJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYt2sCW5KQkbx+6DkmSpHFbtIFNkiRpWhjYJEmSGrfYA9vSJG9Ncmn/ODrJEoAky5O8Icn5SdYk+WaShw5dsCRJ0pZa7IHtELr3cBDwV8BhwAv6ZccDDwCeBOwLnAj8W5K7DVCnJEnSVls6dAHX0a+A/1lVBZyZ5PbAkUk+CfwlcOuqOrdf9+1J/pgu2D137oaSHEYX+FiRXSZSvCRJ0kIs9ha2r/dhbcYpwC2A+wIBViVZPfMAHgnsPd+GqurYqlpZVSuXZ8XYC5ckSVqoxd7CtikF3AtYO2f+VQPUIkmStNUWe2A7IElGWtkOBH5J19IW4GZV9aXBqpMkSdoGFvsp0ZsDb0lyhySPB14EvLmqfgx8EDghyeOT7JVkZZKjkvzZoBVLkiRtocXewvZBYAfgG3SnQN8LvLlfdijwcuCNwB8AlwCnAra4SZKkRWWzgS3JjlV1zebmTVpVPXBk8nnzLF8LvKp/SJIkLVoLOSV6ygLnSZIkaQw22sKW5GZ0Q2TslGQ/uov4Aa4H7DyB2iRJksSmT4k+FHg63fVfb2JDYLsceNl4y5IkSdKMjQa2qjoRODHJwVX1sQnWJEmSpBELuYbtsUmuPzOR5FZJvjDGmiRJkjRiIYHta8A3kjwiybOB/wDeMt6yJEmSNGOzw3pU1buT/JBu/LKLgP2q6oKxVyZJkiRgAS1sSZ4CHAc8FTgB+EySu425LkmSJPUWcqeDg4H7VtVvgA8n+QRwInD3sVYmSZIkYGGnRB8LkGTnqlpTVacm2X/8pUmSJAkWdkr0oCSrgDP76bthpwNJkqSJWUgv0bfQDaJ7MUBVfQ+4/ziLkiRJ0gYLCWxU1XlzZq0bQy2SJEmax0I6HZyX5N5AJVkGPB84Y7xlSZIkacZCWtgOB46guxH8L+h6hz53nEVJkiRpg4W0sN2hqg4ZnZHkPsDJ4ylJkiRJoxbSwvYPC5wnSZKkMdhoC1uSg4B7A3skOXJk0fWAHcZdmCRJkjqbOiW6HNi1X2e3kfmXA48fZ1GSJEnaYKOBraq+DHw5yQlVdc4Ea5IkSdKIzV7DZliTJEka1oIGzpUkSdJwFnIv0fssZJ4kSZLGw2E9JEmSGuewHpIkSY1zWA9JkqTGOayHJElS4xZyL9ETktTcmVX1oDHU04YqWLt26CqaUevWDV1CU2oHO1ePym67bX6lKbLiUn9eRu38h2uGLqEpueraoUtoy8W/HbqCRWMhge2okecrgIOB342nHEmSJM212cBWVd+aM+vkJKeOqR5JkiTNsdnAluRGI5NLgHsC1x9bRZIkSZplIadEvwUUELpToT8HnjnOoiRJkrTBQk6J3mYShUiSJGl+CzklugJ4LnBfupa2rwLvqqqrx1ybJEmSWNgp0fcBV7DhdlRPAt4PPGFcRUmSJGmDhQS2fatqn5HpLyVZNa6CJEmSNNtCRgD9dpIDZyaSHACcNr6SJEmSNGohLWz3BP4rybn99C2BHyU5HaiquuvYqpMkSdKCAtvDxl6FJEmSNmohge3/VNVTRmckef/ceZIkSRqPhVzDdufRiSRL6U6TSpIkaQI2GtiSvDTJFcBdk1ye5Ip++tfAJydWoSRJ0pTbaGCrqtdV1W7A0VV1vararX/cuKpeOsEaJUmSptpCrmH7bJL7z51ZVV8ZQz2SJEmaYyGB7UUjz1cA+9PdEP5BY6lIkiRJsyzk5u9/Ojqd5A+Bt4ytIkmSJM2ykF6ic50P3GlbFyJJkqT5bbaFLck/ANVPLgHuDnx7nEVJkiRpg4VcwzZ639DfAR+uqpPHVI8kSZLmWEhg+whw2/75T6vq6jHWI0mSpDk2NXDu0iRvpLtm7UTgfcB5Sd6YZNmkCpQkSZp2m+p0cDRwI+A2VXXPqroHsDdwA+CYSRQnSZKkTQe2RwHPrqorZmZU1eXAc4BHjLswSbQ792cAABAsSURBVJIkdTYV2Kqqap6Z69jQa1SSJEljtqnAtirJU+fOTPJk4MzxlSRJkqRRm+olegTw8STPoLsVFcBKYCfgceMuTJIkSZ2NBraq+gVwQJIHAXfuZ3+mqr6wJTtIchLwg6p63lZXKUmSNMUWci/RLwJfnEAtkiRJmsfW3EtUkiRJEzSpwLYkyWuTXJTkN0mOSbIEIMkNk5yY5NIkVyX5zyQzp2BJ8vQkq5M8PMmZSdYk+VSS6yd5fJKfJLksyfuT7DTyuiR5cZKf9ds9ve8wIUmStKhMKrAdQncf0nsDzwNeADyxX3YCcADwGGB/YA3wudHwBewI/HW/nQfTdX74GPA04GDgsXTjxj135DX/B3gmXeeJfYDXAe9O8sht/u4kSZLGaCH3Et0WVlXV/+qf/zjJs4EHJzkNeDTwgKr6CkCSpwDn0oWz94zUeURV/ahf50PAC4GbVtVF/bxPAn8EvCnJLsCRwEOq6qv9Nn6eZH+6APf/5haY5DDgMIAV2WWbvnlJkqTrYlKB7ftzpn8J3AS4E7AeOGVmQVVdluR0ulaxGdfMhLXer4ELZsLayLyZ1+wDrKBrqRsd5HcZcPZ8BVbVscCxANdfcmMHBpYkSc2YVGBbO2e62Pzp2NHQ9Lt5lm1qmzP//ilda92mapEkSWrapALbxpxBF64OAmZOiV4PuAtw/HXY7irgGuBW/bAkkiRJi9agga2qftJfe/bu/hqy3wKvAS4HPnQdtntFkmOAY5KELgzuChwIrO9Pf0qSJC0KLYzDdihwKvCp/t+dgYdV1VXXcbuvAF4FHAX8EPgPuh6lP7+O25UkSZqoVHl9/VzXX3LjOnDFI4Yuoxnrr7lm6BKassPetx66hLZctnroCpqy5l63HrqEplz0rDVDl9CUW77U36ez/ObioStoyucv+advVdXK+Za10MImSZKkTTCwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1LilQxfQogKqaugympGly4YuoSnrzz5v6BKacsWnbzl0CU3Z6fXrhi6hKbc64uKhS2jK+pvccOgSmrLkersOXUJbLtn4IlvYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYtmsCW5NNJThi6DkmSpEkbe2BLclKSt497P5IkSdurRdPCJkmSNK3GGtj6U5gPAI5IUv3j1knun+QbSa5O8uskb06yfOR1Oyc5IcnqfvnL5tn2k5N8M8kVSX6T5F+S3KJfliQ/TXLUnNfcrq/hHuN835IkSdvSuFvYng+cAhwP7Nk/1gKfBb4D7Ac8E/hL4HUjrzsG+BPgYODB/Xr3n7Pt5cArgbsBjwJ2Bz4MUFUFvBc4dM5rngF8t6q+vU3enSRJ0gSMNbBV1WXAtcCaqrqgqi4Angv8EnhuVZ1RVZ8GXgI8r29Z25UuxL24qj5fVT+gC17r52z7uKr6TFWdVVWnAs8B7pfkD/pVjgdun+RAgCQ7AE+lC3K/J8lhSU5LctraunobHwlJkqStN8Q1bHcCvl5VowHsa3QtZrcF9u6fnzKzsKpWA6ePbiTJPZJ8Msk5Sa4ATusX3bJ/zQXAp+la1QAeBtwI+OB8RVXVsVW1sqpWLsuK6/gWJUmStp3WOh3UQlZKsgvweWAN8BTgXnSBDLqwN+M9wBOT7EwX3D5RVZduu3IlSZLGbxKB7Vpgh5HpM4ADk4zu+779ej/rH2uBA2cW9gFt35H170h3zdrLquorVXUmcJN59v054HLgcOBPgeOu87uRJEmasEkEtrOB/fveobsD7wRuDrwzyZ2SPBJ4PfD2qlrTn/58L/CGJH+S5M50QWs09J0LXEN33dte/Tb+bu6Oq2pd/9rXAb8AvjC2dylJkjQmkwhsx9C1nq0CLgSWAQ+n6/n5XbpA9WFgdOiOo4AvAZ/o//0B8JWZhVV1IfA04LH9dl8JHLmR/R9Hd5r0+L73qCRJ0qKydNw7qKofAwfNmX02cMAmXnMlXY/Op25inY8AH5kzO/OsejNgHXDC5quVJElqz9gD21CS7AjsQXeq9BNVde7AJUmSJG2V1nqJbkt/CZxD1zlhY6dLJUmSmrfdBraqOqGqdqiqe1TVeUPXI0mStLW228AmSZK0vTCwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS45YOXUCLkpDly4cuox3r1g1dQVPWr1kzdAlNucFz/HzMsv7CoStoSq1fP3QJTbny1rsNXUJTrnyWvz9medTGF9nCJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUuKVDF9CKJIcBhwGsyC4DVyNJkrSBLWy9qjq2qlZW1crlWTF0OZIkSf/NwCZJktQ4A5skSVLjpiqwJXlekjOHrkOSJGlLTFVgA3YH7jB0EZIkSVtiqgJbVb2qqjJ0HZIkSVtiqgKbJEnSYmRgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXEGNkmSpMYZ2CRJkhpnYJMkSWqcgU2SJKlxBjZJkqTGGdgkSZIaZ2CTJElqnIFNkiSpcQY2SZKkxhnYJEmSGmdgkyRJapyBTZIkqXFLhy5A7cuOOw5dQlvWrBm6gqasu9GuQ5fQlu/9aOgK1LDdvpGhS2jK37zpy0OX0JRHb2KZLWySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS48YW2JKclKT6x4Hj2s8Cazl7pJbdh6xFkiRpS427he14YE/gWwAjoWnu4/B++QP76TOTLB3dUB+6jhqZHg2E1yb5VZLPJXlyksyp417AweN9q5IkSeMx7sC2pqouqKq1I/OeTRfiRh8nznndrYBnLmD7M4FwL+DRwCnAu4FPJNlhZqWquhC4ZGvfhCRJ0pCWbn6Vbe63VXXBZtZ5G/CqJB+oqis3sd6akW2dD3wzydeBzwFPpQt0kiRJi1qrnQ7+AVgLHLmlL6yqzwOn4ylQSZK0nRgisL0/yeo5j7vMWedq4BXAi5LssRX7WEV3mnTBkhyW5LQkp11bV2/FLiVJksZjiMD2IuDucx4/mme99wNn0wW3LRWgtuQFVXVsVa2sqpXLs2IrdilJkjQeQ1zDdkFV/XRzK1XV+iQvAf41yVu3cB/7AGdtVXWSJEmNafUaNgCq6jPAycBrFvqaJA8F9gU+Oq66JEmSJmmIFrYbJLnZnHmrq2r1RtZ/MfB1uk4Ic+3cb2sp3fAej+jX/yTwgW1UryRJ0qCGaGH7J+BXcx4v2djKVfVNutayHedZfGj/+rOAfwMOAg4HHldV67Zt2ZIkScOYaAtbVc29A8Hc5SfRdRiYO/+JwBPnzHvgtqxNkiSpVeNuYTusH7bjXmPezyYl+SHw2SFrkCRJ2lrjbGE7BNipf37eGPezEI8AlvXPvUWVJElaVMYW2KrqF+Pa9paqqnOGrkGSJGlrNT2shyRJkgxskiRJzTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUOAObJElS4wxskiRJjTOwSZIkNc7AJkmS1DgDmyRJUuMMbJIkSY0zsEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUuFTV0DU0J8mFwDlD1wHsDlw0dBEN8XjM5vGYzeMxm8djNo/HbB6P2Vo5Hreqqj3mW2Bga1iS06pq5dB1tMLjMZvHYzaPx2wej9k8HrN5PGZbDMfDU6KSJEmNM7BJkiQ1zsDWtmOHLqAxHo/ZPB6zLfh4JFm9rXee5NZJnrSlyxa47QcmufcWvszPx2wej9k8HrM1fzy8hk3S1Emyuqp23cbbfCBwVFU9akuWLXDbrwJWV9Ux16VGSYuXLWySplbfcnVSko8mOTPJB5OkX3Z2kjcmOT3JqUlu288/IcnjR7Yx01r3euB+Sb6b5IVzdjVrWZIdkhyd5JtJvp/kr/ptvTDJcf3zuyT5QZJ9gMOBF/avv994j4qkFi0dugBJGth+wJ2BXwInA/cBvtYvu6yq7pLkqcBbgE21kL2EjbeizVqW5LB+2/dKsiNwcpJ/B94KnJTkccDLgb+qqlVJ3oUtbNJUs4VN0rQ7tarOr6r1wHeBW48s+/DIvwdtw30+BHhqku8C3wBuDNyur+HpwPuBL1fVydtwn5IWMVvYJE27a0aer2P278Wa5/nv6P/YTbIEWL4V+wzwP6rq8/Msux2wGrj5VmxX0nbKFjZJ2rgnjvx7Sv/8bOCe/fNHA8v651cAu21kO3OXfR54TpJlAElun2SXJNcH3gbcH7jxyLVym9q2pClgYJOkjbthku8DzwdmOhL8E/CAJN+jO016ZT//+8C6JN+bp9PB3GXvAVYB307yA+DddC17bwbeUVU/Bp4JvD7JTYB/Ax5npwNpejmshyTNI8nZwMqqauH+gpKmnC1skiRJjbOFTZIkqXG2sEmSJDXOwCZJktQ4A5skSVLjDGySJEmNM7BJkiQ1zsAmSZLUuP8PWmnXLyHWLloAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["### Using tf.keras.layers.Attention"],"metadata":{"id":"g9ofukVYf-Y1"}},{"cell_type":"code","source":["model = Translator(UNITS, context_text_processor, target_text_processor, attention_mechanism=tf.keras.layers.Attention)\n","\n","logits = model((ex_context_tok, ex_tar_in))\n","\n","print(f'Context tokens, shape: (batch, s, units) {ex_context_tok.shape}')\n","print(f'Target tokens, shape: (batch, t) {ex_tar_in.shape}')\n","print(f'logits, shape: (batch, t, target_vocabulary_size) {logits.shape}')"],"metadata":{"id":"rTJKuIrYf-Y3","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569043824,"user_tz":-60,"elapsed":20,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"d974f344-082e-4fb0-acce-134679ce553b"},"execution_count":172,"outputs":[{"output_type":"stream","name":"stdout","text":["Context tokens, shape: (batch, s, units) (64, 20)\n","Target tokens, shape: (batch, t) (64, 18)\n","logits, shape: (batch, t, target_vocabulary_size) (64, 18, 5000)\n"]}]},{"cell_type":"code","source":["model.compile(optimizer='adam',\n","              loss=masked_loss,\n","              metrics=[masked_acc, masked_loss])"],"metadata":{"id":"yTNJ6S_3f-Y7","executionInfo":{"status":"ok","timestamp":1669569043825,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":173,"outputs":[]},{"cell_type":"markdown","source":["#### Model Training"],"metadata":{"id":"P0VTlq9zf-Y9"}},{"cell_type":"code","source":["history = model.fit(\n","    train_ds.repeat(), \n","    epochs=20,\n","    steps_per_epoch = 100,\n","    validation_data=val_ds,\n","    validation_steps = 20,\n","    callbacks=[\n","        tf.keras.callbacks.EarlyStopping(patience=3)])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569241848,"user_tz":-60,"elapsed":198036,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"6c504fec-e86d-4bca-f206-24e9bf26b855","id":"jJVJpqRsf-Y_"},"execution_count":174,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/20\n","120/120 [==============================] - 33s 157ms/step - loss: 5.0431 - masked_acc: 0.2518 - masked_loss: 5.0431 - val_loss: 4.3075 - val_masked_acc: 0.3221 - val_masked_loss: 4.3075\n","Epoch 2/20\n","120/120 [==============================] - 12s 97ms/step - loss: 4.0318 - masked_acc: 0.3606 - masked_loss: 4.0318 - val_loss: 3.7707 - val_masked_acc: 0.3954 - val_masked_loss: 3.7707\n","Epoch 3/20\n","120/120 [==============================] - 11s 90ms/step - loss: 3.5588 - masked_acc: 0.4235 - masked_loss: 3.5588 - val_loss: 3.3946 - val_masked_acc: 0.4397 - val_masked_loss: 3.3946\n","Epoch 4/20\n","120/120 [==============================] - 9s 72ms/step - loss: 3.2213 - masked_acc: 0.4623 - masked_loss: 3.2213 - val_loss: 2.9801 - val_masked_acc: 0.4975 - val_masked_loss: 2.9801\n","Epoch 5/20\n","120/120 [==============================] - 9s 72ms/step - loss: 2.8694 - masked_acc: 0.5084 - masked_loss: 2.8694 - val_loss: 2.6967 - val_masked_acc: 0.5312 - val_masked_loss: 2.6967\n","Epoch 6/20\n","120/120 [==============================] - 8s 71ms/step - loss: 2.5512 - masked_acc: 0.5526 - masked_loss: 2.5512 - val_loss: 2.3989 - val_masked_acc: 0.5729 - val_masked_loss: 2.3989\n","Epoch 7/20\n","120/120 [==============================] - 9s 72ms/step - loss: 2.3274 - masked_acc: 0.5860 - masked_loss: 2.3274 - val_loss: 2.1972 - val_masked_acc: 0.6024 - val_masked_loss: 2.1972\n","Epoch 8/20\n","120/120 [==============================] - 8s 69ms/step - loss: 2.0757 - masked_acc: 0.6216 - masked_loss: 2.0757 - val_loss: 1.9764 - val_masked_acc: 0.6340 - val_masked_loss: 1.9764\n","Epoch 9/20\n","120/120 [==============================] - 8s 69ms/step - loss: 1.9396 - masked_acc: 0.6416 - masked_loss: 1.9396 - val_loss: 1.8264 - val_masked_acc: 0.6599 - val_masked_loss: 1.8264\n","Epoch 10/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.8180 - masked_acc: 0.6604 - masked_loss: 1.8180 - val_loss: 1.6886 - val_masked_acc: 0.6734 - val_masked_loss: 1.6886\n","Epoch 11/20\n","120/120 [==============================] - 8s 70ms/step - loss: 1.7332 - masked_acc: 0.6736 - masked_loss: 1.7332 - val_loss: 1.6932 - val_masked_acc: 0.6804 - val_masked_loss: 1.6932\n","Epoch 12/20\n","120/120 [==============================] - 8s 70ms/step - loss: 1.6586 - masked_acc: 0.6856 - masked_loss: 1.6586 - val_loss: 1.6229 - val_masked_acc: 0.6917 - val_masked_loss: 1.6229\n","Epoch 13/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.5922 - masked_acc: 0.6943 - masked_loss: 1.5922 - val_loss: 1.5274 - val_masked_acc: 0.7035 - val_masked_loss: 1.5274\n","Epoch 14/20\n","120/120 [==============================] - 9s 71ms/step - loss: 1.5596 - masked_acc: 0.6979 - masked_loss: 1.5596 - val_loss: 1.5574 - val_masked_acc: 0.6958 - val_masked_loss: 1.5574\n","Epoch 15/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.5151 - masked_acc: 0.7041 - masked_loss: 1.5151 - val_loss: 1.5188 - val_masked_acc: 0.6987 - val_masked_loss: 1.5188\n","Epoch 16/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.4622 - masked_acc: 0.7125 - masked_loss: 1.4622 - val_loss: 1.4509 - val_masked_acc: 0.7082 - val_masked_loss: 1.4509\n","Epoch 17/20\n","120/120 [==============================] - 8s 69ms/step - loss: 1.4366 - masked_acc: 0.7151 - masked_loss: 1.4366 - val_loss: 1.4302 - val_masked_acc: 0.7205 - val_masked_loss: 1.4302\n","Epoch 18/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.4209 - masked_acc: 0.7189 - masked_loss: 1.4209 - val_loss: 1.4361 - val_masked_acc: 0.7148 - val_masked_loss: 1.4361\n","Epoch 19/20\n","120/120 [==============================] - 8s 68ms/step - loss: 1.4046 - masked_acc: 0.7197 - masked_loss: 1.4046 - val_loss: 1.3103 - val_masked_acc: 0.7406 - val_masked_loss: 1.3103\n","Epoch 20/20\n","120/120 [==============================] - 8s 69ms/step - loss: 1.3852 - masked_acc: 0.7228 - masked_loss: 1.3852 - val_loss: 1.3647 - val_masked_acc: 0.7230 - val_masked_loss: 1.3647\n"]}]},{"cell_type":"markdown","source":["#### Graphs:"],"metadata":{"id":"aJJzksqZf-ZA"}},{"cell_type":"code","source":["plt.plot(history.history['loss'], label='loss')\n","plt.plot(history.history['val_loss'], label='val_loss')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569241849,"user_tz":-60,"elapsed":24,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"07bd16e4-8849-4714-815e-6ca9707474f9","id":"mwNG8Nl3f-ZB"},"execution_count":175,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb3b2908150>"]},"metadata":{},"execution_count":175},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVfrH8c8zk0kvhAAJEEroLdSASLMDYkFBQESxoiuKuq6u7uquZXXX1bX8dnXFhooVVFQUBVRQZEE0QOhFQEoCCSFAep05vz/uAAkpBJLJJJPn/XrNa27mnpn7ZBi+uXPuveeIMQallFK+x+btApRSSnmGBrxSSvkoDXillPJRGvBKKeWjNOCVUspH+Xm7gNKaNWtm2rdv7+0ylFKqwVi9evUhY0zzitbVq4Bv3749iYmJ3i5DKaUaDBHZU9k67aJRSikfpQGvlFI+SgNeKaV8VL3qg1dKNT7FxcUkJydTUFDg7VLqtcDAQGJjY3E4HNV+jga8UsqrkpOTCQsLo3379oiIt8upl4wxZGRkkJycTFxcXLWf59EuGhHZLSIbRCRJRDxyekxeUQmPfL6RhRsPeOLllVIeVlBQQFRUlIZ7FUSEqKio0/6WUxd78OcZYw556sUD/ews3ZbOrwdzGN2rpac2o5TyIA33UzuT96jBH2S12YSrBsSyYmcG+w7nebscpZSqNzwd8AZYLCKrReTWihqIyK0ikigiienp6We0kfEDYhGBj1cn16RWpVQjFRoa6u0SPMLTAT/MGNMfuBi4Q0RGnNzAGPOqMSbBGJPQvHmFV9ueUusmQQzr1IyPVyfjcukEJkopBR4OeGNMivv+IPApMMhT25qQ0IaUo/ms3JXhqU0opXycMYb777+fXr16ER8fz5w5cwA4cOAAI0aMoG/fvvTq1Ysff/wRp9PJDTfccLzt888/7+Xqy/PYQVYRCQFsxphs9/JI4HFPbW9kj2jCA/2Ym7iPoZ2aeWozSikPeuyLTWzen1Wrr9mjVTiPXNazWm3nzZtHUlIS69at49ChQwwcOJARI0bw/vvvM2rUKB566CGcTid5eXkkJSWRkpLCxo0bATh69Git1l0bPLkHHw0sF5F1wM/AAmPMQk9tLNBhZ2zf1izcmEpmfrGnNqOU8mHLly9n8uTJ2O12oqOjOeecc/jll18YOHAgb775Jo8++igbNmwgLCyMDh06sGvXLmbMmMHChQsJDw/3dvnleGwP3hizC+jjqdevyISEWN75aQ9frNvPtYPb1eWmlVK1oLp72nVtxIgRLFu2jAULFnDDDTdw7733MnXqVNatW8eiRYuYOXMmc+fOZdasWd4utYwGf5pkafGtI+gWE8ZHejaNUuoMDB8+nDlz5uB0OklPT2fZsmUMGjSIPXv2EB0dzbRp07jllltYs2YNhw4dwuVyMX78eJ544gnWrFnj7fLL8amhCkSECQlt+NuXm9melk2X6DBvl6SUakCuvPJKVq5cSZ8+fRARnn76aWJiYnj77bd55plncDgchIaGMnv2bFJSUrjxxhtxuVwA/OMf//By9eWJMfXntMKEhART0wk/MnIKOevv33Hj0PY8dEmPWqpMKeUpW7ZsoXv37t4uo0Go6L0SkdXGmISK2vtUFw1AVGgAF3aP5tO1KRQ7Xd4uRymlvMbnAh6sg62HcopYuvWgt0tRSimv8cmAP6dLc5qHBTA3UQ+2KqUaL58MeD+7jXH9W7N020EOZuskAkqpxsknAx5gwoA2OF2Gz9ameLsUpZTyCp8N+E4tQunftglzE5OpT2cKKaVUXfHZgAeYmNCGHQdzSNpX/8aIUEopT/PpgL+kd0sCHTY92KqUqjVVjR2/e/duevXqVYfVVM2nAz4s0MGY+JZ8uW4/+UVOb5ejlFJ1yqeGKqjIxIQ2zFuTwsJNB7iyX6y3y1FKVeXrByF1Q+2+Zkw8XPxUpasffPBB2rRpwx133AHAo48+ip+fH0uXLuXIkSMUFxfzxBNPMHbs2NPabEFBAbfffjuJiYn4+fnx3HPPcd5557Fp0yZuvPFGioqKcLlcfPLJJ7Rq1YqJEyeSnJyM0+nkL3/5C5MmTarRrw2NIODPimtK26bBfJSYrAGvlCpn0qRJ3HPPPccDfu7cuSxatIi77rqL8PBwDh06xODBg7n88stPa+Lrl156CRFhw4YNbN26lZEjR7J9+3ZmzpzJ3XffzZQpUygqKsLpdPLVV1/RqlUrFixYAEBmZmat/G4+H/AiwoQBsTz7zXb2Hc6jTdNgb5eklKpMFXvantKvXz8OHjzI/v37SU9PJzIykpiYGH7/+9+zbNkybDYbKSkppKWlERMTU+3XXb58OTNmzACgW7dutGvXju3bt3P22Wfz5JNPkpyczLhx4+jcuTPx8fH84Q9/4IEHHuDSSy9l+PDhtfK7+XQf/DE6KbdSqioTJkzg448/Zs6cOUyaNIn33nuP9PR0Vq9eTVJSEtHR0RQU1M5Fk9dccw3z588nKCiIMWPGsGTJErp06cKaNWuIj4/n4Ycf5vHHa2fyu0YR8K10Um6lVBUmTZrEhx9+yMcff8yECRPIzMykRYsWOBwOli5dyp49e077NYcPH857770HwPbt29m7dy9du3Zl165ddOjQgbvuuouxY8eyfv169u/fT3BwMNdeey33339/rY0t3ygCHnRSbqVU5Xr27El2djatW7emZcuWTJkyhcTEROLj45k9ezbdunU77decPn06LpeL+Ph4Jk2axFtvvUVAQABz586lV69e9O3bl40bNzJ16lQ2bNjAoEGD6Nu3L4899hgPP/xwrfxePjcefGUKip0MevJbzuvWgv+7up9HtqGUOn06Hnz1Nfrx4Cujk3IrpRqbRhPwYJ0TX1ji4ot1+71dilKqAduwYQN9+/YtczvrrLO8XVY5vnGaZHYaBEaAI7DKZr1ahx+flPvawe3qqDil1KkYY07rHHNvi4+PJykpqU63eSbd6Q1/Dz7vMMwcCt8+csqmxyblXrfvKNvTsuugOKXUqQQGBpKRkaGjvlbBGENGRgaBgVXvxJ6s4e/BBzeFXlfBqpeh4wXQZWSVza/o24qnvt7CR4n7dFJupeqB2NhYkpOTSU9P93Yp9VpgYCCxsad3NX7DD3iACx+F3T/C59Ph9hUQ2qLSplGhAVzQzZqU+4+ju+GwN/wvMUo1ZA6Hg7i4OG+X4ZN8I90cgTD+DSjMhs9uB5eryuYTB+qk3Eop3+cbAQ/QohuMehJ2fAurZlbZdETn5rTQSbmVUj7OdwIeIOFm6DrGOuB6YH2lzaxJuWN1Um6llE/zrYAXgctfhKCm8MnNUJRXadMJCbE6KbdSyqf5VsADhETBuFfg0K+w+KFKm3VsHsqAdpF8pJNyK6V8lO8FPECHc2HIDEicBVu+rLTZxIRYftVJuZVSPso3Ax7g/L9Ayz4w/07Iqnhogkt6tyLIYdeDrUopn+TxgBcRu4isFZHKd6U9wc8fxs+CkkL49LYKT50MDfDTSbmVUj6rLvbg7wa21MF2ymvWCS5+Gn5bBiv+XWGTCQmxZBeWsGDDgTouTimlPMujAS8iscAlwOue3E6V+l0LPa6AJX+DlPKzpJwV15RuMWE8uWAz+w5XftaNUko1NJ7eg38B+CNQ6aWlInKriCSKSKJHxqIQgctegNAY69TJwpyTt8/L1w7A6TJMm51IbmFJ7deglFJe4LGAF5FLgYPGmNVVtTPGvGqMSTDGJDRv3twzxQRFwrhX4fBv8PUD5VbHNQvhxWv6sz0tm/s+WqfztiqlfIIn9+CHApeLyG7gQ+B8EXnXg9urWvuhMOI+SHoXNs4rt3pEl+b8eUx3vt6Yyn+W7PBCgUopVbs8FvDGmD8ZY2KNMe2Bq4ElxphrPbW9ajnnAYgdCF/cA0f3llt987A4xvVvzfPfbmfhxlQvFKiUUrXHd8+Dr4jdAeNeA+OCebeCq+ypkSLC36+Mp0+bJtw7N4mtqVleKlQppWquTgLeGPO9MebSutjWKTWNg0uehb0r4cdny60OdNh59boBhAb4MW12Ikdyi7xQpFJK1Vzj2oM/ps8kiJ8I3z8F+34utzo6PJBXrhtAWmYhd7y/hhJn1ePLK6VUfdQ4Ax7gkn9BRKx16mRBZrnV/dpG8vdx8azYmcETC7xznZZSStVE4w34wAgY/zpkpsCC+ypsctWAWG4eFsdbK3Yz95d9dVygUkrVTOMNeIA2g+DcB2HDXFj9VoVN/nRxN4Z3bsZDn21g9Z7DdVufUkrVQOMOeIDhf4CO51unTq55p9xqP7uN/0zuR6smQdz2zhoOZOZ7oUillDp9GvA2O1z9vhXy8++EX94o16RJsD+vT00gv6iE295ZTUGxjjyplKr/NOABHEFWyHcZDQvuhZ/KT9rdOTqMF67ux4aUTB78ZL3OAqWUqvc04I9xBMLEd6DbpbDwAVjxn3JNLuoRzR8u6sJnSft5ddkuLxSplFLVpwFfmp8/THgLel4Jix+u8EKoO87rxCXxLXlq4Va+33aw7mtUSqlq0oA/md0B4163LoT67nHrYqhS3TEiwjMTetMtJpwZH6xlZ3pOFS+mlFLeowFfEbsfXDkT+k6B7/9hTRZSKuSD/f14beoAHHYb02YnklVQ7MVilVKqYhrwlbHZ4fIXYcANVlfNN38pE/KxkcH8d0p/9mbkcfcHa3HqGPJKqXpGA74qNhtc+gIMutU66LrwwTIhP7hDFI9c3pOl29J5+LONGvJKqXrFz9sF1Hsi1sTdNgf89BI4i2DMs1b4A9cNbsf+o/m8/P1O0rML+PfkfgT769uqlPI+3YOvDhEY9SQMvQcSZ8EXM8qMJf/A6G48dnlPlmw9yORXfyI9u9CLxSqllEUDvrpE4MJHrVmh1r4Ln00vE/LXD2nPzGsHsC0tm3Ev/0/PrlFKeZ0G/OkQgfP+DOc9DOs/hHnTwFlyfPXInjF8MG0weYVOxr+8gl926+BkSinv0YA/E+fcDxc+Bhs/gY9vBOeJ0yT7tY1k3vQhRAb7M+X1VSxYf8CLhSqlGjMN+DM17B4Y9Q/YMh/mToWSE/3u7aJC+OT2IcS3juCO99fw2rJdOnaNUqrOacDXxNnTYcy/YNtX8NENZbprmob4894tZzEmPoYnv9rCY19s1tMolVJ1SgO+pgZNOxHy8+8E14n5WwMddl6c3J9b3LNC/e7d1eQX6VDDSqm6oQFfGwZNsw68rvsAFv2pzMVQNpvw8KU9eOSyHny7JY2rX/uJQzl6GqVSyvM04GvLiPtg8B2waib88HS51TcOjWPmtQPYeiCLcf9dwS49jVIp5WEa8LVFBEY+4R6g7O+w6pVyTUb1jOGDWweTU1jC+JdX6ByvSimP0oCvTTYbXPZva9KQr/8I6+aUa9K/bSTzbh9CRJCDa15bxdcb9DRKpZRnaMDXNrsfjH8D4kbAZ7fDtq/LNWnfLIR504fSs1U4099fwxvLf/NCoUopX6cB7wmOQGuO15Z9YO718NuP5Zo0DfHn/WmDGdUjhr99uZm/fbkZl55GqZSqRRrwnhIQBlM+hsj28MFk2L+2XJNAh52XpvTnhiHteWP5b/zho3UUO13lX0sppc6ABrwnhUTBdZ9CUCS8Ox7St5drYrcJj1zWg/tHdeXTtSlMm51IXlFJBS+mlFKnRwPe0yJaw9TPQGzwzhVwdF+5JiLCHed14qlx8Szbns6U11dxNK/IC8UqpXyJBnxdiOoI186Dwhwr5HPSK2x29aC2/HfKADbtz2LCzJUcyMyv40KVUr5EA76utOwN18yBzBR4dxwUZFbYbHSvGN6+cRCpmQWM/+8KdhzUC6KUUmdGA74utTsbJr0DBzfD+1dDccV76Gd3jOLD2wZT5DRMmLmCpH1H67hQpZQvqHbAi8gQEblGRKYeu52ifaCI/Cwi60Rkk4g8VvNyfUDni+DKV2DvSusUylJjyZfWs1UEn9x+NmGBDq557SeWba+4W0cppSpTrYAXkXeAfwHDgIHuW8IpnlYInG+M6QP0BUaLyOAa1Oo74q+CS56FXxdZF0O5Kj41sl1UCB/ffjbtokK4+e1f+DwppY4LVUo1ZH7VbJcA9DCnMWuFu+2xDmSH+6ZX8hwz8GYoOArfPQ6BTWDMM9Z4NidpERbInNsGc8vbidwzJ4kjuUXcMDTOCwUrpRqa6nbRbARiTvfFRcQuIknAQeAbY8yqCtrcKiKJIpKYnt7IuiGG3QtDZsAvr8FX95WZFaq08EAHs28axEXdo3n0i808u3ibzhCllDolqU5QiMhSrG6Wn7G6XgAwxlxerY2INAE+BWYYYzZW1i4hIcEkJiZW5yV9hzGw+GFY+aI1tMFVb1qnVVagxOnioU83MidxH5MHteWJK3pht5Xf61dKNR4istoYU2GXeXW7aB6tSQHGmKPuPxKjsb4NqGNEYNST0G4IfDYdXhkBl74AvSeUa+pnt/HU+Hiahfnz0tKdHMkt4oWr+xLosHuhcKVUfVetLhpjzA/AbsDhXv4FWFPVc0SkuXvPHREJAi4CttaoWl/W7RK4/X8QEw/zboHP7oCi3HLNRIT7R3XjL5f2YOGmVG5482eyCyo+E0cp1bhV9yyaacDHwLFZLFoDn53iaS2BpSKyHusPwjfGmC/PtNBGISIWrv8SRtwPSe/Bq+dCasVfeG4eFscLk/qSuPsIV7/6E+nZOg2gUqqs6h5kvQMYCmQBGGN+BVpU9QRjzHpjTD9jTG9jTC9jzOM1K7WRsPvB+Q/D1M+tq11fOx9+eb3MPK/HXNGvNa9fn8Cu9FyueOl/bDmQ5YWClVL1VXUDvtAYc3z0KxHxQ0959KwO58Dv/gdxw2HBH2DuVMg/Uq7ZuV1bMOe2wZS4XIx/eQWLNqV6oVilVH1U3YD/QUT+DASJyEXAR8AXnitLARDaHK75CC76G2z7CmaOgH0/l2vWO7YJ8+8cRufoMG57ZzUvLvlVT6NUSlU74B8E0oENwG3AV8aYhzxWlTrBZoOhd8FNi60zbmaNhh+fK3f1a3R4IHNuHcwVfVvxr8XbuevDJAqKnV4qWilVH1Q34B81xrxmjJlgjLkKmCUi73myMHWS2AHwux+h+2Xw3WPWiJQ5B8s0CXTYeX5SX/44uitfrt/PxFdWkppZ4KWClVLeVt2AbyMifwIQEX/gE+BXj1WlKhYYARPegsv+zxqs7OWhsHNJmSYiwvRzO/HadQnsPJjD5S8u19EolWqkqhvwNwHx7pD/EvjBGPOox6pSlROBATfAtKUQ3BTeGQffPlpuVMoLe0Qzb/pQ/P1sTHxlJZ+t1YHKlGpsqgx4EekvIv2BfsD/AZOw9tx/cD+uvCW6hxXy/afC8ufhrUsha3+ZJl1jwph/5zD6tWnCPXOS+OfCrbhcevBVqcaiyrFo3MMLVMYYY86vzWIa5Vg0tWHDxzD/LvAPhqtmQdyIMquLSlw8Mn8jH/y8jwu7t+CFq/sRGlDdUSqUUvVZVWPRVGuwsbqiAV8DB7fC3OsgYwdc8FcYek+Z4YeNMby9Yjd/W7CFjs1DeH3qQNpGBXuxYKVUbagq4Ks7VEGEiDx3bFhfEXlWRCJqt0xVIy26wbQl0GOs1Sf/4RTIP3FwVUS4YWgcb904kNTMAsa+tJyfdmV4r16llMdV9yDrLCAbmOi+ZQFveqoodYYCwqzhhkc/Zc0W9eq5cGB9mSbDOzfnszuGEhniz7Wvr+L9VXu9U6tSyuOqG/AdjTGPGGN2uW+PAR08WZg6QyIw+Ha44SsoKYA3LoK1ZS9Z6NA8lE+nD2VIp2b8+dMNPPL5RkqcFU8bqJRquKob8PkiMuzYDyIyFMj3TEmqVrQ9C277EWIHwufTrYOwxScueooIcjDr+gRuHhbH2yv3MP7lFazdW36sG6VUw1XdGZ36ALOBY/3uR4DrjTHrK3/W6dODrB7gLIGlT8Ly56wZoybOhsj2ZZrMX7efv325mfTsQsb1a80DF3cjOjzQO/UqpU5Ljc+iEZE4Y8xvIhIOYIzJOvZYbRaqAe9BW7+CT39ndeGMew26jCyzOqewhBeX7GDW8t9w2IU7zu/EzcPiCPDT2aKUqs9qfBYN1tAEGGOyjDHHBh3/uDaKU3Wk2xi47XuIaAPvT4AlT4LrxGBkoQF+PHhxNxb/fgRnd2zG0wu3MfL5ZXyzOU1HplSqgTrVlazdRGQ8ECEi40rdbgD0O3xD07QD3PIN9L0Wlj0N746H3LKnSrZvFsLr1ycw+6ZBOOw2ps1OZOqsn/k1LdtLRSulztSprmQdC1wBXA7ML7UqG/jQGLOiNovRLpo6tGY2LLgPQprDxLchtvw3vGKni3dW7uH5b7eTV+Rk6tntuOeCLkQEO7xQsFKqImfcBy8ik4HFQBdjzEoP1XecBnwd259kXf2adQDOfQAGTwf/kHLNMnIKefab7Xzw814ig/35w8guXD2wLXabVPCiSqm6VJOAfwAYBTiA74CvgZ+NhzplNeC9IO8wfHE3bJkPIS2sCb8HXA9+AeWabtqfyWPzN/Pz7sP0aBnOI5f14KwOUV4oWil1TG2cRRMGXAiMBgYBW4CFwCJjTFptFaoB70V7V8F3j8Oe5RDRFs59EHpPsiYBL8UYw4INB/j7gi3szyzgkt4t+fOY7rRuEuSlwpVq3Gp9sDER6QFcDIw0xoyqYX3HacB7mTGwa6kV9PvXQrMucN5D0P1ya+rAUvKLnMz8YSczf9iJCMw4vzO3juiAw17dE7OUUrXhjE+TFJFrSy0PPbZsjNkMFNZmuKt6QAQ6nm+NMz/pXRAbfHQ9vHYu/Pqt9QfALcjfzu8v6sKS+87l/G4teGbRNsa++D82pmR6r36lVBmn2t26t9Tyf05ad1Mt16LqCxFr7tfbV8CVr1ijUr43Ht4cA3vKnjjVukkQ/50ygFeuG0B6TiFjX/ofzyzaqhN+K1UPnCrgpZLlin5WvsZmhz5Xw52JcMmzcHgXvHkxvHuVdQZOKaN6xvDt78/hyn6teWnpTi7594+s3qNj2yjlTacKeFPJckU/K1/l5w8Db4G71sJFj0NKIrx6Dsy9HtK3H28WEezgXxP68PZNgygodnHVzBU8/sVm8opKvFi8Uo3XqU6TzAN2YO2td3Qv4/65gzGm/EnTNaAHWRuIgkxY+ZJ1K86DPpOts26atD3eJKewhKcXbmX2yj20bRrMU+PiGdKpmReLVso31eQ8+M5ANLDvpFVtgFRjzI7yzzpzGvANTO4ha8Lvn1+z+u2HzLCmCgwIPd5k1a4MHvhkPbsz8pg8qA1/GtOd8EC9Elap2lKTwcaeBzKNMXtK34BM9zrVmIU0g1FPwozV0O1SWPYMvJgA6z4ElzWByFkdolh4zwhuG9GBOb/sY+Rzy/huS61dOqGUqsKpAj7aGLPh5Afdj7X3SEWq4WnSBq56A25aDGEx8Olt8MaFsO9nAAIddv40pjufTh9KeJAfN7+dyD0fruVIbpGXC1fKt50q4JtUsU4vXVRltT0LblkCV8yEzBRrusBPboHMZAD6tGnCFzOGcdcFnfly/QEuev4HFqw/oMMRK+Uhpwr4RBGZdvKDInILsNozJakGzWaDvpOtbpvh98Hm+fCfBPj+KSjKI8DPzr0XdeGLGcNoGRHEHe+v4XfvriblqM4AqVRtO9VB1mjgU6CIE4GeAPgDVxpjUqt4bhusaf6isU6pfNUY839VFaMHWX3Q0b3wzV9h06cQ3houfAzirwIRSpwuXl/+G899s51ip4uz4ppyRd/WXNyrpQ5JrFQ11cZgY+cBvdw/bjLGLKnGc1oCLY0xa9yDla0GrnAPc1AhDXgftmcFLHwQDqyD2EFw8VPQegAA+w7nMW9NCp8npbDrUC7+dhvndm3O2L6tuaB7CwIdOm2gUpWp9cHGzrCIz4EXjTHfVNZGA97HuVyQ9J41mFnuQev8+Qv+CuGtAGukyo0pWXyWlMIX6/ZzMLuQ0AA/RvWMYWzfVgzpGIWfDmamVBleD3gRaQ8sA3qVmtO1HA34RqIwG3581rpQyuYHw+6FIXeC48Rxe6fL8NOuDD5PSuHrDalkF5bQLDSAS3u35Ip+rekTG4GIjpahlFcDXkRCgR+AJ40x8ypYfytwK0Dbtm0H7Nmzx6P1qHrk8G9W//yW+RDYxBr3pt91ENOrTLOCYiffbzvI50n7+W7rQYpKXLSPCubyvq0Z27cVHZuHVrIBpXyf1wJeRBzAl1gTgzx3qva6B99I7VkJP78KW78EZxG06g/9p0Kv8RAYXqZpZn4xizal8nlSCit2ZmAMxLeO4JLeLRnVM4a4ZrU6eoZS9Z5XAl6s789vA4eNMfdU5zka8I1c3mFYP8eaEPzgZnAEQ88rrbBvc5Y1HEIpaVkFfLFuP/PX7Wd9sjUOfZfoUEb1jGFUzxh6tgrXbhzl87wV8MOAH4ENgMv98J+NMV9V9hwNeAVYE4ukrLaCfuMnUJRjzS7V7zrrwGxo83JPST6Sx+JNaSzalMovuw/jMtZY9Rf1iGZUzxgGto/UA7TKJ3n9IGt1acCrcgpzYPNnVtjvW2UdlO06xtqr73i+NWb9SQ7nFvHtljQWb0pl2a+HKCpxERns4MLu0YzsGcPwzs301EvlMzTglW9I32YF/boPIC/DunCq37XQdwpEtqvwKbmFJfywPZ3Fm1L5butBsgtKCPa3c06X5ozqGcN53VoQEaQXVamGSwNe+ZaSItj2Fax9B3Z8Zz0WNxx6T7KmGgyMqPBpRSUuftqVwaJNqXyzOY2D2YX42YSzO0ZxTpfm9GsbSc9W4bp3rxoUDXjlu47usy6eWvchHPkN7AHQ9WLoPRE6XWTNRlUBl8uQlHzUCvtNaew6lAuAv91Gj1bh9GvbhH5tI+nftgmtmwTpwVpVb2nAK9937MDs+jmwcR7kHYKgSOhxhbVn3+YsayC0ShzMKmDtvqOs2XuEtXuPsj75KAXF1rkBzcMC6O8O/H5tmtA7tglB/rqXr+oHDXjVuDiLYedS2DAXtnwJJfnWdILxE6ywb971lC9R7HSxLTX7eOCv3XuE3Rl5ANhtQveWYfRvG2nt6beJpF1UsO7lK6/QgFeNV2EObF1g7dnvWgrGBTG9raDvNVd3xz4AAA+1SURBVB7CW1b7pTJyCkkqtZe/bt9RcoucADQJdtCzVTg9W0XQs1U4vVpHEBcVgs2moa88SwNeKYDsNNg0D9bPhf1rQGwQNwLiJ0Lr/uBygnG6710n/Vz+caezmANHctmdnsW2LH++z4xmVZqdIqfVtRPsb6dHy3Ar+Ftbwd+5RRj+fno+vqo9GvBKnezQr1bQb5gLR3bX2sua0Ghym3Ql2b8jm1xtWZEdzTfpEWQVWXvy/nYbXWPCyoR+95hw7dNXZ0wDXqnKHDs4e3SvddGU2Evd2076udTjNr9Sj9kgOxXSNkLaJkjdAOlbrXF1AGP3pyiyMweDOrGNdvyS35rFGVH8lh8MgE0grlkI3WLC6RIdRteYULpEh9EuKgS7dvGoU9CAV6quOYshYwekbnQH/0ZrOefEJGjOkGgOh3Vht609q0vi+DqnA+uP+HPsv2SAn41OLULpGh1Gl5gwusaE0TU6jJYRgXpAVx2nAa9UfZF76ETYp22CtA3WFbruvX1Xs64cbj6QX4P6ssrVnTWH/dmemk1qVsHxlwgL8KNLTJi1tx8dSpeYMNpHhdA0xF8v0mqENOCVqs+cxdZUhrt/hN3/g70rrQHWAKI6Q/th5LUczPbgPmzMCmZ7WjZbU7PZnpbN0bziMi8VGuBH0xB/okL9iQoJIOrYcmip5ZAAokL9aRrij0MHYGvwNOCVakicJZC6DnYvPxH4he6J0Jp2hPbDoP0wTLshpEsztqZms/9oPhm5RWTkFJGRW+i+LyIju4D8vGzCXVlESjZNJZsm5NBUsomUbGL8con2yyXAz0ZRUDSEReNo0orgprFEtGhDVKt2hEdElRuqWdUfGvBKNWQuJ6Sut8J+93LYuwIKrPHviYyD9kMhPNYagC0vA/IPu5fd9yUFFb6sQcjziyDbFo7L6STCeZgQ8su1KzAODtujyHY0oyiwOc7QGPzCYwhsGktY8zZERrfBP6od+OtkK96gAa+UL3E5rX78Y4G/539QcNQamiE4CoKaWvfBURBcevmkxwIjyg237MzP4nDaPg6n7SUnPZmio/sxWQew56YRVJhOeEkGzcxhQqXsHw0XwkF7DIeCO5EX2RWJ6UlYm97ExPWgSWhwXb47jY4GvFK+zOWyLsCy+9XJ5vKLnKSmp5ORuo/sQ/soyEiGI7tpkrWNloW7aGMOYBcrVwqNg50Sy4GADmSGdcHZvBsBrXoT3aot7ZqF0iIsQK/2rSENeKVUncnPzSFt13py9q7DlbqJ4KNbaZa7kyauw8fbZJgwtrnasEPacii4E4XhbbEHReAICiMgpAkBoRGEhoQREexPeJCDiCAH4YEOIoIdhAX46WmipWjAK6W8LzeDktSNZO1JojBlI36HthCRvQN/V/l+f4ASYyOXQLIJJscEkUMQuSaQHIIotIdQ4heC0xGK8Q/D7h9AgM0QYHdZ9zaDv1jLDpsLh7jwFxd+7mU/XDjEiR8u7Dixh7XA1v0yaDekwlnC6jMNeKVU/eRywdHdkJlsDQxXlIMpyKIoL5Oi3EyK8rJw5mfhKsiComxsRTnYi3PwK8klwJlLoCuvypcvMTac2Cnh5Hs7TmyUGOu+lWQQJEUcJpwfbGex0n8oW4P6EhAQQJC/H8EOO8H+dgL97ceXg/z93PfWzyEBfoQG+BHi774PsB4L8LN59BtHVQFfN512SilVEZsNmnawbm4CBLhvp+RyWdcMOIus4SPcN2OzU+gU8opd5Bc7yS8qIb/IRV5RCfnFTgqKneQVOd3rnBTl59Aq/Uc6HfqOMZnLubLgG3KLwvgl4GyWOYawingyi23kF514XnX52YRgf7s79P1O/CFw/wEI8fcjKtSfey7scrrv3qm3XeuvqJRSdcVmg8Dwcg8LEGiHwIon9KpEb+AOKM6HnUsI2Tyfc7d9xblZiyEg3JoprPvl0OkCXPZACkqcxwM/r8hJTmEJuYUl5BWVkFPoJLew5PhjuYUl5BZZjxXnZxOeu4vIw8k0K0ompmQ/gXYXXPh5Lb0pJ2jAK6VUaY4g6HaJdSsphF0/wJbPT8wr4AjB1mUkwT3GEtx5JFGhFZz/X5AFh3e5bzuh4DfIdv+ck1a2bUhzaNHdGviulrtytA9eKaWqw1lsDSexeT5s+cKaFtIvCDpdANG94OgeyNhphXjeobLPDWvp7oqKO9El1bSDdaFaBd9AToceZFVKqdrkcsKeFbBlvhX4OanW1cQnB/ixUPfgVb56kFUppWqTzQ5xw63b6H+Cqxj8qnVYuE5pwCulVE3YbGCrf+EOoGOFKqWUj9KAV0opH6UBr5RSPkoDXimlfJQGvFJK+SgNeKWU8lEa8Eop5aM8FvAiMktEDorIRk9tQymlVOU8uQf/FjDag6+vlFKqCh4LeGPMMuDwKRsqpZTyCK/3wYvIrSKSKCKJ6enp3i5HKaV8htcD3hjzqjEmwRiT0Lx5c2+Xo5RSPsPrAa+UUsozNOCVUspHefI0yQ+AlUBXEUkWkZs9tS2llFLleWw8eGPMZE+9tlJKqVPTLhqllPJRGvBKKeWjNOCVUspHacArpZSP0oBXSikfpQGvlFI+SgNeKaV8lAa8Ukr5KA14pZTyURrwSinlozTglVLKR2nAK6WUj9KAV0opH6UBr5RSPkoDXimlfJQGvFJK+SgNeKWU8lEa8Eop5aM04JVSykdpwCullI/SgFdKKR+lAa+UUj5KA14ppXyUBrxSSvkoDXillPJRGvBKKeWjNOCVUspHacArpZSP0oBXSikfpQGvlFI+SgNeKaV8lAa8Ukr5KA14pZTyUR4NeBEZLSLbRGSHiDzoyW0ppZQqy2MBLyJ24CXgYqAHMFlEenhqe0oppcry5B78IGCHMWaXMaYI+BAY68HtKaWUKsXPg6/dGthX6udk4KyTG4nIrcCt7h9zRGTbGW6vGXDoDJ9bF7S+mtH6akbrq5n6XF+7ylZ4MuCrxRjzKvBqTV9HRBKNMQm1UJJHaH01o/XVjNZXM/W9vsp4sosmBWhT6udY92NKKaXqgCcD/hegs4jEiYg/cDUw34PbU0opVYrHumiMMSUiciewCLADs4wxmzy1PWqhm8fDtL6a0fpqRuurmfpeX4XEGOPtGpRSSnmAXsmqlFI+SgNeKaV8VIML+FMNfyAiASIyx71+lYi0r8Pa2ojIUhHZLCKbROTuCtqcKyKZIpLkvv21rupzb3+3iGxwbzuxgvUiIv92v3/rRaR/HdbWtdT7kiQiWSJyz0lt6vT9E5FZInJQRDaWeqypiHwjIr+67yMree717ja/isj1dVjfMyKy1f3v96mINKnkuVV+FjxY36MiklLq33BMJc/1+FAnldQ3p1Rtu0UkqZLnevz9qzFjTIO5YR2s3Ql0APyBdUCPk9pMB2a6l68G5tRhfS2B/u7lMGB7BfWdC3zpxfdwN9CsivVjgK8BAQYDq7z4b50KtPPm+weMAPoDG0s99jTwoHv5QeCfFTyvKbDLfR/pXo6so/pGAn7u5X9WVF91PgserO9R4L5q/PtX+X/dU/WdtP5Z4K/eev9qemtoe/DVGf5gLPC2e/lj4AIRkboozhhzwBizxr2cDWzBuqK3IRkLzDaWn4AmItLSC3VcAOw0xuzxwraPM8YsAw6f9HDpz9jbwBUVPHUU8I0x5rAx5gjwDTC6Luozxiw2xpS4f/wJ6xoUr6jk/auOOhnqpKr63LkxEfigtrdbVxpawFc0/MHJAXq8jftDnglE1Ul1pbi7hvoBqypYfbaIrBORr0WkZ50WBgZYLCKr3cNEnKw673FduJrK/2N58/0DiDbGHHAvpwLRFbSpL+/jTVjfyCpyqs+CJ93p7kKaVUkXV314/4YDacaYXytZ7833r1oaWsA3CCISCnwC3GOMyTpp9Rqsboc+wH+Az+q4vGHGmP5Yo3zeISIj6nj7p+S+MO5y4KMKVnv7/SvDWN/V6+W5xiLyEFACvFdJE299Fl4GOgJ9gQNY3SD10WSq3nuv9/+XGlrAV2f4g+NtRMQPiAAy6qQ6a5sOrHB/zxgz7+T1xpgsY0yOe/krwCEizeqqPmNMivv+IPAp1lfh0urDEBMXA2uMMWknr/D2++eWdqzbyn1/sII2Xn0fReQG4FJgivuPUDnV+Cx4hDEmzRjjNMa4gNcq2a633z8/YBwwp7I23nr/TkdDC/jqDH8wHzh2xsJVwJLKPuC1zd1n9wawxRjzXCVtYo4dExCRQVj/BnXyB0hEQkQk7Ngy1sG4jSc1mw9MdZ9NMxjILNUdUVcq3XPy5vtXSunP2PXA5xW0WQSMFJFIdxfESPdjHicio4E/ApcbY/IqaVOdz4Kn6it9TOfKSrbr7aFOLgS2GmOSK1rpzffvtHj7KO/p3rDO8tiOdYT9Ifdjj2N9mAECsb7a7wB+BjrUYW3DsL6urweS3LcxwO+A37nb3Alswjor4CdgSB3W18G93XXuGo69f6XrE6yJWnYCG4CEOv73DcEK7IhSj3nt/cP6Q3MAKMbqB74Z65jOd8CvwLdAU3fbBOD1Us+9yf053AHcWIf17cDqvz72GTx2Vlkr4KuqPgt1VN877s/WeqzQbnlyfe6fy/1fr4v63I+/dewzV6ptnb9/Nb3pUAVKKeWjGloXjVJKqWrSgFdKKR+lAa+UUj5KA14ppXyUBrxSSvkoDXjlk0TEedLIlLU2GqGItC89+mA12oeIyLfu5eXui2iU8jj9oClflW+M6evtItzOBla6L3jKNScGAlPKo3QPXjUq7jG8n3aP4/2ziHRyP95eRJa4B8D6TkTauh+Pdo+pvs59G+J+KbuIvCbWuP+LRSSogm11dI8l/i5wDbAa6OP+RtGijn5l1YhpwCtfFXRSF82kUusyjTHxwIvAC+7H/gO8bYzpjTU417/dj/8b+MFYg5v1x7pqEaAz8JIxpidwFBh/cgHGmJ3ubxGrscYpeRvrSsm+xhq/RCmP0itZlU8SkRxjTGgFj+8GzjfG7HIPDJdqjIkSkUNYl8wXux8/YIxpJiLpQKwxprDUa7THGuu9s/vnBwCHMeaJSmr5xRgzUEQ+Ae42lYxvolRt0z141RiZSpZPR2GpZScVHM8SkZnug7Gd3V01o4EvReT3Z7hNpU6LBrxqjCaVul/pXl6BNWIhwBTgR/fyd8DtACJiF5GI6m7EGPM74DHgb1izPi1wd888X7PylaoePYtG+aogKTtZ8kJjzLFTJSNFZD3WXvhk92MzgDdF5H4gHbjR/fjdwKsicjPWnvrtWKMPVtc5wGys2YF+OKPfRKkzpH3wqlFx98EnGGMOebsWpTxNu2iUUspH6R68Ukr5KN2DV0opH6UBr5RSPkoDXimlfJQGvFJK+SgNeKWU8lH/DwgphZvJNKoAAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["plt.plot(history.history['masked_acc'], label='accuracy')\n","plt.plot(history.history['val_masked_acc'], label='val_accuracy')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569241850,"user_tz":-60,"elapsed":17,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"f697ed6e-f507-4e29-ae0b-6125041ca433","id":"pEVY2d6nf-ZB"},"execution_count":176,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb35fc7a3d0>"]},"metadata":{},"execution_count":176},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUZdbA8d9JIwkJIYTQUujSe0BRVxQbKoKiCKz6IiroKtZ1V191EZXdta5tXVbsbVXUF0VEkSKigkhAeieUBAIkJATSy5z3jzvgEBKSFSaTZM7385nP3PLMncNlcs/Mc+89j6gqxhhj/FeArwMwxhjjW5YIjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XNBvg7gv9W0aVNt06aNr8Mwxpg6Zfny5ZmqGlvRujqXCNq0aUNycrKvwzDGmDpFRHZWts66howxxs9ZIjDGGD9nicAYY/xcnTtHUJGSkhLS0tIoLCz0dSgGCA0NJT4+nuDgYF+HYoyphnqRCNLS0oiMjKRNmzaIiK/D8WuqyoEDB0hLS6Nt27a+DseY2qEwBzZ9BTEdoFVfCKhdnTH1IhEUFhZaEqglRISYmBgyMjJ8HYoxvpezG5ZOheS3oPiwsyyyJXS6FDpfBm1+B0EhPg0R6kkiACwJ1CL2f2H83r71sPglWDMd1AXdroT+4+HgTtg4C1Z9AMmvQ4NG0PEiJyl0uABCG/kk3HqTCIwxxqdUYcf38OOLsHUuBIdD0k0w8DaIbuO0aT0Qeo2GkgJIWegkhU1fwdpPIDAE2g5ykkKnSyGyeY2FbonAGGNORlkpbJgJi1+EPb9Aw1g472HofxOEN6n4NcFh0OkS5+Eqg9SlsPFL2PAFzJoLs+6BhAFOUug8FGLae/WfYImgjiktLSUoyP7bjPG54jz45X1Y8k+ny6dJexj6vPONPzis+tsJCITWZ0LrM9ELH6dozzpK139B8JbZNJg7CeZO4nCjDqQ1G0x4/2tp3an3Kf+n2BHlFLriiitITU2lsLCQu+66iwkTJvD111/z4IMPUlZWRtOmTZk/fz65ubnccccdJCcnIyI88sgjXHXVVURERJCbmwvAJ598wqxZs3jrrbe44YYbCA0N5ZdffuGss85i9OjR3HXXXRQWFhIWFsabb75Jp06dKCsr4/777+frr78mICCA8ePH061bN1588UU+++wzAObOncu//vUvZsyY4ctdZUzNyM+CtGXOY+9aaBABkS2cE7aezxEtICT8mJeWuZTDhSXkFDiPQwWllJS5CMg/QNyWd0nc9j4hxQfJiu7FpqR7SWt+LqWuAMqW76fMpZS5FJcqpe7pMpeSV1zK4cJScgtLyS1yng8VljjT7vlSlwK9gF7EkcGFgcu5KDuZATmv8XODVpYIquPRL9axfs+hU7rNrq0a8cjl3aps98Ybb9CkSRMKCgro378/w4cPZ/z48SxatIi2bduSlZUFwOOPP05UVBRr1qwBIDs7u8ptp6WlsXjxYgIDAzl06BDff/89QUFBzJs3jwcffJBPP/2UadOmsWPHDlauXElQUBBZWVlER0dz2223kZGRQWxsLG+++SY33njjye0QY2qjslLYvw5Sf4a0ZDRtGZK1DQCVAIqi2qOlRYTk7yPQVXTcy/MkgsyAJuzXaNJdUaSVNmafRrNPo9mvjSkimNGB3zIy8DtCpYS5Zf14pfQyktM7QboA66oMMSQogEahQUQ0CCIiNIjIBsEkNAkn8sh8aBARDYKJCA36tV2Dy4kIDWKfHqZ3VMSp3mtAPUwEvvTiiy8e/aadmprKtGnTOOecc45eT9+kidNfOG/ePD788MOjr4uOjq5y2yNHjiQwMBCAnJwcxo4dy5YtWxARSkpKjm731ltvPdp1dOT9rr/+et577z3GjRvHkiVLeOedd07Rv9gY7youdZGanc/2jDx2ZuVzuLCE/OIy8opKCczfT4tDa0jIW0e7ovW0L9lCKM4BPkOjWOHqyC+u0fzi6sgabUt+Qah7q0oj8mguB4kLPEjrkEMkBOXQKvAgzSSbWFcWHV2biJRMArXsmHhcASEc7DCC1F4TSIg5jScChAARAgM8HuXn3W2CAoSgwJO5fyDqJF57YvUuEVTnm7s3LFy4kHnz5rFkyRLCw8M599xz6d27Nxs3bqz2Njwvuyx/l3TDhg2PTv/lL3/hvPPOY8aMGezYsYNzzz33hNsdN24cl19+OaGhoYwcOdLOMZhaxeVS9h0uZHtGHimZeaRk5LE9M5ftmXmkZhdQ5lIaUkAnSaVXwDb6B26ld8BWWuHcq1JCEDtDOvBj1GXsbtid/VE9KImIJ7xBME0bBDI8JIjfNwgkPMT5lh0VHkxUmPMICw6s/HJnlwsKsuBwOhzeC/kHCGh3Lk0iW1DJKeA6y6tHBBEZArwABAKvqeoT5dY/B5znng0HmqlqY2/G5C05OTlER0cTHh7Oxo0b+emnnygsLGTRokVs3779aNdQkyZNuPDCC3n55Zd5/vnnAadrKDo6mubNm7NhwwY6derEjBkziIyMrPS94uLiAHjrrbeOLr/wwgt55ZVXOO+88452DTVp0oRWrVrRqlUrpkyZwrx587y+L4wpT1U5mF/C9gN5bM/IY3um80jJzGNHZh4FJc437zAK6RaczsDITH7fYA/tm6XRvGg7DQv2/LqxRvGQcDbE94f4/gS36EmH4FA6nOqgAwKgYVPn0aLHqd56reK1RCAigcDLwIVAGrBMRGaq6vojbVT1Ho/2dwB9vBWPtw0ZMoR///vfdOnShU6dOnHGGWcQGxvLtGnTGDFiBC6Xi2bNmjF37lwefvhhbr/9drp3705gYCCPPPIII0aM4IknnmDo0KHExsaSlJR09MRxeX/+858ZO3YsU6ZM4bLLLju6/Oabb2bz5s307NmT4OBgxo8fz8SJEwG49tprycjIoEuXLjWyP0wtUHDQuZxx93LnOWMjNGwGTdpCdNtjn8Oi4TfeCKiqZOeXkJ5TwN6cQtJzCknPKSA9p5C97kd6TuHRgz1AWEAJZ0VlcWXDfXSL203rsl00LdhOg9xUBIV8oCgEmp4G8WdCs87QrCu06gONWp2iHWSOEFX1zoZFBgKTVfVi9/z/Aqjq3ytpvxh4RFXnnmi7SUlJWn5gmg0bNtgBrgoTJ06kT58+3HTTTTXyfvZ/UsNKCmHvGvdBf4XzfGDrr+tjOkCzLpB3ALK3O90dnhpEQZM2RxODRrelMDKRQ2HxZAfFklNQxsGCEvYfKmTP0YN7gftAn09gaQENKSRCnOdGAYXEh5XSKryM5qElxAaXEBOUT6vSVBrnbiP40E5EXc57BwRBTEfnYB/bxYmzWRcnlkDrxjxVRGS5qiZVtM6bezkOSPWYTwNOr6ihiLQG2gILKlk/AZgAkJiYeGqj9AP9+vWjYcOGPPvss74OxZwKrjLI2HTsQX/fOnCVOusjW0JcP+g1BuL64WrZm/TiUHZm5nHQfSlkXu4hAg7uJOTQTsLzdtGoII0mmXtotncpLfQLgigjDAgDGmsQaRpLkTalPWX0kkKiAouIlALCtYAGQYVIUAVfKEuBQ+4HgAQ6N0a16gG9r3EO9rFdnGWBVqnWl2pLuh0NfKJa7hS9m6pOA6aB84ugJgOrD5YvX+7rEMxv5XI53+D3/OLxWAklec76BlEQ1wfOvJPC5r3Z0aAzm/Ij2JaRR0paLim/5JGS+ROFJa7jNh0UIESFdSQqrCuRYcFENXFOoDZuAPEB2cTpXpqV7SGmeA9RBWm0LEgnOCSCoLBWSGgkhERAgyPPEe7nRh7TERAS6Tw3iHRKLlgdqlrJm4lgN5DgMR/vXlaR0cDtXozFmNrP5YKsFEhf6Rzw01c5jyL3V+rABmiLnuR2HU1aeBfWS0dW5TVha2Y+KT/nsfdQIeB0BwUIxEeH0y62IWe0i6F9s4a0jWlIk4gQosKCaRQaTHjICa6YMX7Fm4lgGdBRRNriJIDRwO/LNxKRzkA0sMSLsRhTu7hckLXN+XafvtJ53rv62IN+8+5ktx/O5oAOLCmIZ/6BGLbuKqRw25Fv93lEhhbRLjaCMzvE0D42gnZNG9IuNoLWMeGEBgf67J9n6havJQJVLRWRicAcnMtH31DVdSLyGJCsqjPdTUcDH6q3zlob40uqkLsfDu5yf9tf5Rz401f/Wp8+KBRt3p28TiPYFtSBn4sSmZ8ZzcrUPApTnIN+4/BgesY35LoOzWkXG0H7WOeA3zQixL7Vm5Pm1XMEqjobmF1u2aRy85O9GYMxXuVyQe4+50Cfk+oUHzu4y+ORCmUe5QyCQqFFD4q6X8POkI4kF7fmu6xolqflkrmtGHDKEHRrFciYAYn0TmhM74TGJDYJtwO+8ZracrLYmNpLFbJ3OFfnZG93Du5HDvQ5qVBWfGz78KbQOBGad4dOl+KKSiBVm7LycBTfH4hiRVouKVvzjmyc9rFFDDqtGb0TouidEE2nFpGEBNWuoQxN/WaJwAc8q4yaWsjlgowNsHOx89i15Njr7hs2cw70LXtCl6HOdOPWznNUPAWE8ktqNsk7slm2I4tffjxIblEpUEjTCKV3QmOu6hdPr/jG9EyIolGoXTppfMsSgR+zsQ3cykqcvnvPA3/hQWddZCtofZZTLz7hdGjS7rhyxZm5RSTvyCZ5aRbLdq5k3e4cSl2KCHRqHskVfVrRv00T+rWOJq5xmHXxmFqn/h0FvnrAucPyVGrRAy55otLVDzzwAAkJCdx+u3MF7OTJkwkKCuLbb78lOzubkpISpkyZwvDhw6t8q9zcXIYPH17h69555x2eeeYZRISePXvy7rvvsm/fPm699VZSUlIAmDp1Kq1atWLo0KGsXbsWgGeeeYbc3FwmT558tBjeDz/8wJgxYzjttNOYMmUKxcXFxMTE8P7779O8efMKx0zIyclh9erVR2skvfrqq6xfv57nnnvupHZvjSvOh93Jvx7405ZBSb6zLqYDdLncffAf6HzT9zhwqyo7MvNYtiOL5B1ZJO/IJiXT6eYJCQqgd3xjJpzTjv5tmtA3MZqocPu2b2q/+pcIfGDUqFHcfffdRxPB9OnTmTNnDnfeeSeNGjUiMzOTM844g2HDhlX5bTA0NJQZM2Yc97r169czZcoUFi9eTNOmTY+ObXDnnXcyaNAgZsyYQVlZGbm5uVWOb1BcXMyRMh3Z2dn89NNPiAivvfYaTz31FM8++2yFYyYEBwfz17/+laeffprg4GDefPNNXnnllZPdfTUjcyusfA92/Ohco+8qAQRadIc+1zvf+BMHVjhObE5BCfPW72Pehn0s25FFZq5zTqBxeDBJraO5pn8C/dtE0z0uigZBdsmmqXvqXyI4wTd3b+nTpw/79+9nz549ZGRkEB0dTYsWLbjnnntYtGgRAQEB7N69m3379tGiRYsTbktVefDBB4973YIFCxg5ciRNmzYFfh1rYMGCBUfHFwgMDCQqKqrKRDBq1Kij02lpaYwaNYr09HSKi4uPjp1Q2ZgJgwcPZtasWXTp0oWSkhJ69KjlVRlTl8GPzzvjwQYEQVxfOHMiJJ7pjAkbVnGx2yMH/9lr0lm0JYOSMqVlVCjndIwlqU0T+reJpn1sBAEB1s1j6r76lwh8ZOTIkXzyySfs3buXUaNG8f7775ORkcHy5csJDg6mTZs2x40xUJHf+jpPQUFBuFy/lhQ40dgGd9xxB/feey/Dhg1j4cKFTJ48+YTbvvnmm/nb3/5G586dGTdu3H8VV41xuWDLHPjxBae/P7QxnHMfDJgAEc0qfVlFB/9WUaGMHdiGS3u2pE9CY+vfN/WSJYJTZNSoUYwfP57MzEy+++47pk+fTrNmzQgODubbb79l586d1dpOTk5Oha8bPHgwV155Jffeey8xMTFHxxo4//zzmTp1KnfffffRrqHmzZuzf/9+Dhw4QEREBLNmzWLIkCGVvt+RsQ3efvvto8srGzPh9NNPJzU1lRUrVrB69eqT2WWnXmkRrJ4Oi1+CzE0QlQhDnoQ+1zn1bipgB39jLBGcMt26dePw4cPExcXRsmVLrr32Wi6//HJ69OhBUlISnTt3rtZ2Kntdt27deOihhxg0aBCBgYH06dOHt956ixdeeIEJEybw+uuvExgYyNSpUxk4cCCTJk1iwIABxMXFnfC9J0+ezMiRI4mOjmbw4MFs374doNIxEwCuueYaVq5cWa0hNmtEYQ4kvwk/TYXcvc7J/ateh65XVFjGuKKDf1zjMMYObMNlPVvS2w7+xs94bTwCb7HxCHxv6NCh3HPPPZx//vmVtqmR/5Oc3bB0KiS/5ZRraHcenHWn81zuQH64sIRv1h1/8L+kews7+Bu/4KvxCEw9c/DgQQYMGECvXr1OmAS8bt96p/tnzcegLuh2pZMAWvY6pllRaRkLN2Uwc+Ue5m3YR1Gpy775G1MBSwQ+smbNGq6//vpjljVo0IClS5f6KKKqNW7cmM2bN/vmzVWda/5/fME5ERwcDv1vgjNug+jWR5u5XMrPO7L4fOVuZq/ZS05BCTENQxjdP4FhvePom2gHf2PKqzeJQFXr1B94jx49WLlypa/D8IpT3t24dy188zCkfOvU8TnvYScJhDc5+n4b0g/z+crdzFy1h/ScQsJDArm4WwuG9W7F2R2aEhxotXuMqUy9SAShoaEcOHCAmJiYOpUM6iNV5cCBA4SGhp78xg7vg2+nwC/vOSNfXfx3SBoHwWEApGblM3PVHj5fuZvN+3IJChDOOS2WBy7pzIVdmxMeUi8+3sZ4Xb34S4mPjyctLY2MjAxfh2JwEnN8fPxv30BxPiz5J/zwvFPZ8/Q/OPcBhDchK6+YL5fv5PNfdpO807lxLql1NI9f0Z3LerSkScOQU/SvMMZ/1ItEEBwcfPSOWFOHuVyw+iOY/xgc3gNdhsEFkyGmPWt35/Dc9GV8tzmDUpfSsVkEf7q4E8N6tSKhSXhVWzbGnEC9SASmHtjxA8x50KkC2qovXP06tD6TvKJSnpu1njd+3E50eAg3nd2WYb1b0bVlI+sGNOYUsURgfCtzK8ydBJu+hKgEGPEadL8KAgL4duN+Hv5sLbsPFjBmQCIPDOls1TyN8QJLBMY38rPguydh2WsQFAbnT3IuBQ0OY//hQh77Yj2zVqfToVkEH986kP5tmvg6YmPqLa8mAhEZAryAM3j9a6p6XGlQEbkGmAwosEpVf+/NmIyPlRbBz6/Coqeg6DD0HQvnPQgRzXC5lOk/7+JvszdQWOLingtO49Zz21lpZ2O8zGuJQEQCgZeBC4E0YJmIzFTV9R5tOgL/C5ylqtkiUnlpSFO3qcKGmU43UPYO6HAhXPQ4NHPKUGzdn8uD/7eGn3dkMaBtE/4+ogftYysuFGeMObW8+YtgALBVVVMARORDYDiw3qPNeOBlVc0GUNX9XozH+ErKQudKoN3LoVlXuO7/oINToqKotIx/fbuNqQu3ERocwJNX9WBkvwSr829MDfJmIogDUj3m04DTy7U5DUBEfsTpPpqsql+X35CITAAmACQmJnolWOMFqctgwWOwfRE0iodhL0HvayHA6epZmnKAB2esYVtGHsN6teIvQ7sSG9nAx0Eb4398fbI4COgInAvEA4tEpIeqHvRspKrTgGngVB+t6SDNf2nfOlgwBTbNhoaxzpgASeMgyDnI5+SX8MTXG/jg51Tio8N4a1x/zu1kvYLG+Io3E8FuIMFjPt69zFMasFRVS4DtIrIZJzEs82JcxlsObIOFf4c1nzglIQb/BU6/9eigMKrKrNXpPPrFerLyiphwTjvuvqCjlYIwxse8+Re4DOgoIm1xEsBooPwVQZ8BY4A3RaQpTldRihdjMt6Qs9u5CmjFu863/rPvccpCh/06cE1qVj6TPl/Lt5sy6BEXxVvj+tM9LsqHQRtjjvBaIlDVUhGZCMzB6f9/Q1XXichjQLKqznSvu0hE1gNlwJ9U9YC3YjKnWF4m/PCcczmoupyKoL+7DyKbH21SWFLG1IXbmPrdNoIChL8M7crYga0JsmqgxtQa9WKEMlPDCnNgycvOoyQfeo2BQfcfMy6AqvLN+n08Pms9adkFXN6rFQ9e2pmWUWE+DNwY/2UjlJlTozgflr3q/AooyIauw+G8hyC20zHNtmXk8ugX61m0OYPTmkfwwfgzGNg+xkdBG2OqYonAVM/G2TDrHmdw+A4XwOCHoVWfY5rkFZXy0oKtvP5DCqFBgUwa2pXrB7a2QWGMqeUsEZgTKy2CuY84g8S36AFXvwFtzjqmiaryxep0/vblBvYeKuTqfvHcP6Sz3RNgTB1hicBULisFPh4H6Sudy0AvfOzovQBHbNp7mEmfr2Xp9iy6xzXi5Wv70q91dCUbNMbURpYITMXWfAJf3O3cBTzqfegy9JjVOQUlPD9vM+8s2UlkaBB/vbI7o/snEmilIYypcywRmGMV58PXD8CKtyF+gNMV1PjX+wJdLuXTFWk8+fVGDuQVM2ZAIn+6qBPRNkSkMXWWJQLzq/0b4eMbIGODc1PYeQ9B4K8DwazdncOkz9eyYtdB+iQ25s0bBtAj3m4KM6aus0RgnBLRK9+HL++DkIZw3afOlUFuLpfywvwtvLhgCzENQ3j66p5c1TfeKoQaU09YIvB3RYdh1r2wZjq0+R1c9RpEtji6+mB+MXd/tJKFmzIY0TeORy7vRlSYDRdpTH1iicCfpa92uoKyt8O5D8I59x0tEQ1OV9Af3l/O3pxCplzRnWtPT7QB442phywR+CNVZ6zgOQ9CeAyM/QLanH1Mk0+Wp/HQjDVEh4cw/ZaB9Em0S0KNqa8sEfibgmz4fCJsnAUdL4IrpkLDpkdXF5WW8fis9bz30y4Gtovhpd/3oWmE3RhmTH1micCfpC6DT26Ew3vgoilwxu0Q8Gv5h/ScAv7w3gpWph7klkHt+NNFnaxKqDF+wBKBPyjOg0XPwOIXoVEruHEOxB9bhHDxtkzu+M8vTtnoa/tySY+WPgrWGFPTLBHUZ6qw/nOY8xAcSnPKRQ95AsIaezRRXv0+hSe/3kSbmHBeuf4MOjSL9GHQxpiaZomgvsrYBF/9GVIWQvMezmWhrQce0yS3qJQ/f7KK2Wv2ckn3Fjw9shcRDewjYYy/sb/6+qboMHz3FPz0LwhuCJc8DUk3QuCx/9Vb9+dyy7vJbM/M48FLOzP+d+3s0lBj/JQlgvpCFdZ+Ct88DIfToc91cP5kiIg9rulXa9K57+NVhAYH8t7Np3Nm+6bHb88Y4zcsEdQH+zfA7D/Bju+hZS+45l1I6H9cs9IyF0/P2cQri1LondCYqdf1taEjjTGWCOq0wkOw8AlY+m9oEAmX/QP63XDM3cFHHMgtYuJ/fmFJygGuPT2RSZd3pUHQ8e2MMf7Hq4lARIYALwCBwGuq+kS59TcATwO73Yv+qaqveTOmekEVVk+HuX+B3P3QbywMngQNKx4XeP2eQ4x/J5nM3CKevronI5MSKmxnjPFPXksEIhIIvAxcCKQBy0RkpqquL9f0I1Wd6K046p29a5xuoF1LIK4fjPnAea7EV2vSuXf6KqLCgvn41oH0jG9caVtjjH/y5i+CAcBWVU0BEJEPgeFA+URgqqMoF+Y/BstehbBoGPYS9L7umDuDPR0pHf3C/C30SWzMK9f1o1mj0BoO2hhTF3gzEcQBqR7zacDpFbS7SkTOATYD96hqavkGIjIBmACQmJjohVBruaJceP9qSF3qXAp63kMQ3qTS5nlFpfxx+iq+XreXq/vFM+WK7oQG2/kAY0zFfH2y+AvgA1UtEpFbgLeBweUbqeo0YBpAUlKS1myIPlacDx+MdpLAVa9D9xEnbJ6alc/4d5LZvO8wD1/WhZvObmv3BxhjTsibiWA34HlWMp5fTwoDoKoHPGZfA57yYjx1T0khfHQt7PgBRkyrMgn8lHKA295fQWmZizfHDWDQacffQ2CMMeV5s7TkMqCjiLQVkRBgNDDTs4GIeFY2GwZs8GI8dUtpMUz/H9i2AIb/E3pec8Lm7/20k+teW0rj8GA+u/0sSwLGmGrz2i8CVS0VkYnAHJzLR99Q1XUi8hiQrKozgTtFZBhQCmQBN3grnjqlrAQ+GQdb5sDQ55y7hCtRUubi0S/W8d5PuzivUywvjOlDo1AbStIYU32iWre63JOSkjQ5OdnXYXhPWSl8ehOs/wwueQpOv6XSpgdyi7jt/RUs3Z7FLYPa8eeLOxNoA8obYyogIstVNamidb4+WWw8ucrgsz84SeCiKSdMAhvSD3Hz28lk5Bbx3KheXNknvgYDNcbUJ5YIaguXC2beAWumw/mT4Mw7Km369VrnJrHI0CA+vmUgvRLsJjFjzG9niaA2UIUv74WV78OgB+B3f6ywmculvLhgC8/P20KvhMZMu74fze0mMWPMSbJE4Guq8NX9sPxNOPteOPeBCpvlFZVy38er+GrtXkb0jeNvV/awm8SMMaeEJQJfUnXGD/j5FRg40ekSquDmr2U7svjj9FWkZefbTWLGmFPOEoGvqDq1g5b8EwZMcE4Olzu4F5WW8Y+5m5m2KIX46DA+nDCQAW0rLy1hjDG/hSUCX/nuSfjBPX7AJU8dlwTW7cnh3o9WsWnfYcYMSOChy7raeMLGGK+wI4svfP8PWPh36H0tXPbcMUmgtMzFK4tSeH7eZhqHh/DGDUkM7tzch8EaY+q7aicCETkTaOP5GlV9xwsx1W+L/wnzH4UeI51S0h5lpLdn5vHH6StZsesgl/VoyZQruhPdMMSHwRpj/EG1EoGIvAu0B1YCZe7FClgi+G8snQbfPARdr4Ar/n10SElV5b2fdvK32RsJDhReGN2bYb1a2QlhY0yNqO4vgiSgq9a1ehS1hapzPmD+Y9DpMrjqNQh0dv3enEL+9Mkqvt+Sye86NuXpq3vRIsruDTDG1JzqJoK1QAsg3Yux1E+Fh5yyERtnQfer4IqpEBiMqjJz1R7+8tlaSsqUx6/oznWnJ9qvAGNMjatuImgKrBeRn4GiIwtVdZhXoqov9m+Ej66DrBS4+O9wxh9AhKy8Yv7y2Vq+XJNO38TG/OOa3rRp2tDX0Rpj/FR1E8FkbwZRL62bAZ/dDiENYewX0OYsABZs3Mf9n67hYNL9jf4AABV4SURBVH4xfx7SiVvOaW8VQ40xPlWtRKCq34lIa6Cjqs4TkXCcMQZMeWWlMH8yLH4J4gfANW9Do1bkF5fy+Kz1fPBzKp1bRPL2uAF0bdXI19EaY0y1rxoajzN4fBOcq4figH8D53svtDooN8MZUGbH99D/Zqc7KCiE4lIXt7y7nB+2ZnLroPbcc2FHGgRZHjXG1A7V7Rq6HRgALAVQ1S0i0sxrUdVFacnO0JL5B5xLQ3uPAZyKofd/uprvt2Ty1NU9uSYpoYoNGWNMzapuIihS1eIjV7SISBDOfQRG1akc+tX9ENkCbvoGWvY6uvrJORuZ8ctu/njhaZYEjDG1UnUTwXci8iAQJiIXArcBX3gvrDqipAC+vA9WvgcdLoARr0L4r0Xh3vxxO698l8J1ZyQycXAHHwZqjDGVC6i6CQAPABnAGuAWYLaqPlTVi0RkiIhsEpGtIlJxoX2n3VUioiJS4XiatVL2TnjjYicJnPNn+P30Y5LArNV7eGzWei7u1pxHh3W3+wOMMbVWtS8fVdVJwKsAIhIoIu+r6rWVvUBEAoGXgQuBNGCZiMxU1fXl2kUCd+E+/1AnbJ3vDDDvcsGYD6HTJcesXrLtAPd+tIp+idG8MLqPXR5qjKnVqvuLIEFE/hdAREKAT4EtVbxmALBVVVNUtRj4EBheQbvHgSeBwmrG4jsuFyx6Bt67CiJbwoRvj0sCG/ceYsK7ySTGhPPa2CQbRcwYU+tVNxHcCPRwJ4NZwHeqOrmK18QBqR7zae5lR4lIXyBBVb880YZEZIKIJItIckZGRjVDPsWKDsP062HB406piJvnQUz7Y5rsPljA2Dd+pmFIEG/fOIDG4VY51BhT+50wEYhIX/fBug/wAjAK55fAd+7lv5mIBAD/ACoeqd2Dqk5T1SRVTYqNjT2Zt/1tXC74dDxs+sq5N+Cq15w7hj0czC9m7Bs/k19cxls39ieucVjNx2mMMb9BVecIni03nw10dS9XYPAJXrsb8LxeMt697IhIoDuw0H0itQUwU0SGqWpy1aHXoB+fh81fOSOJnX7LcasLS8q4+e1kdh3I5+0bB9C5hd0xbIypO06YCFT1vJPY9jKgo4i0xUkAo4Hfe2w7B6eYHQAishC4r9Ylge2Lfu0OGjDhuNVlLuXOD35h+a5s/jmmLwPbx/ggSGOM+e2qdY5ARKJE5B9H+ulF5FkRiTrRa1S1FJgIzAE2ANNVdZ2IPCYidaNq6aF0+ORGiOkAl7943LjCqsojM9fyzfp9TBralct6tvRRoMYY89tV9/LRN3DGJLjGPX898CYw4kQvUtXZwOxyyyZV0vbcasZSM8pKnLpBxfkwdhY0iDiuycvfbuW9n3Zxy6B2jDurrQ+CNMaYk1fdRNBeVa/ymH9URFZ6I6BaY/6jsGsJXPU6NOt83Orpyak8881mRvSJ4/6Lj19vjDF1RXUvHy0QkbOPzIjIWUCBd0KqBTZ84ZSR7j8eelx93OoFG/fxv/+3ht91bMqTV/ckwG4YM8bUYdX9RXAr8I7HeYFsYKx3QvKxA9vgs9sgrh9c/NfjVq9MPcjt7/9C15aNmHpdP4IDq5tLjTGmdqpuIjikqr1EpBGAqh5yXw1UvxTnO6WkAwJh5FsQ1OCY1SkZudz41jJiIxvwxg39iWhQ3d1njDG1V3W/zn4KTgJQ1UPuZZ94JyQfmv0n2LcORrwGjROPWZVxuIixb/4MwNs3DiA2skFFWzDGmDrnhF9pRaQz0A2IEhHPK4QaAaHeDKzGrXjHqSQ66H7oeMFxq5/4aiP7DhUx/ZaBtLWB5o0x9UhVfRudgKFAY+Byj+WHgfHeCqrGpa9yxhVod56TCMrZuv8wM35J46az29I7obEPAjTGGO+pKhGEA/cB01R1SQ3EU/MKDjrnBRo2dWoIBRxfLfS5eVsICw7k1kHtK9iAMcbUbVUlgkTgYyBYROYDXwE/q2r9GKbS5YLP/gA5aTDuKycZlLNuTw5frk7njsEdiImw8wLGmPrnhCeLVfVJVR0MXAqswilHvUJE/iMi/yMizWsiSK9Z/AJsmg0X/RUSBlTY5B/fbKZRaBA3/65dDQdnjDE1o1rXP6rqYWCG+4GIdAUuAd4BLvZadN60/XuY/xh0u7LCiqIAK3ZlM3/jfv50cSeiwoJrOEBjjKkZVY1HcJ3H9FlHpt3DTRapat1MAof3OsXkmrSHYS8dV0zuiGe/2UTTiBBuOLNNzcZnjDE1qKr7CO71mH6p3LobT3EsNaOs1EkCxbkw6l1oEFlhs8XbMvlx6wH+cG4HGtqNY8aYeqyqI5xUMl3RfN0w/1HY+SOMeBWadamwiary7DebadEolGtPT6ywjTHG1BdV/SLQSqYrmq/9NsyCxS9C0k3Q85pKmy3clMHyndnccX4HG3zeGFPvVfWLoLOIrMb59t/ePY17vm5dRnNgm3OpaKu+MOTvlTZzuZRnvtlEQpMwRvZLqLSdMcbUF1Ulgl5AcyC13PIEYK9XIvKWDV84N4td8/ZxxeQ8zVm3l3V7DvHsyF6EBFllUWNM/VfVke45IEdVd3o+gBz3urrj7Lvhtp+OKybnqcylPDt3M+1jG3JFn7gaDM4YY3ynqkTQXFXXlF/oXtbGKxF5U2SLE67+fOVutu7P5d4LOxFog80YY/xEVYngRBXWwqrauIgMEZFNIrJVRB6oYP2tIrJGRFaKyA/uG9V8oqTMxfPzttC1ZSMu6X7ihGGMMfVJVYkgWUSOqzIqIjcDy0/0QhEJBF7GuQO5KzCmggP9f1S1h6r2Bp4C/lHtyE+xj5PT2JWVz30Xn2ZDTxpj/EpVJ4vvBmaIyLX8euBPAkKAK6t47QBgq6qmAIjIh8BwYP2RBh6D3AA0xEeXpBaWlPHSgi30SWzMeZ2a+SIEY4zxmRMmAlXdB5wpIucB3d2Lv1TVBdXYdhzHXm2UBpxevpGI3I5zB3MIMLg6QZ9q/1m6i/ScQp4d2QuppNyEMcbUV9UtOvct8K03AlDVl4GXReT3wMPA2PJtRGQCMAEgMfHU3umbX1zKvxZu5cz2MZzZ4fgy1MYYU99580L53Tj3GxwR715WmQ+BKypaoarTVDVJVZNiY2NPYYjw1uIdZOYW88eLOp3S7RpjTF3hzUSwDOgoIm1FJAQYDcz0bCAiHT1mLwO2eDGe4+QUlPDKdykM7tyMfq2ja/KtjTGm1vBaWU1VLRWRicAcIBB4Q1XXichjQLKqzgQmisgFQAmQTQXdQt70+vcp5BSUcO+Fp9Xk2xpjTK3i1frKqjobmF1u2SSP6bu8+f4nkpVXzOs/bOfSHi3oHhflqzCMMcbn/LaYzr+/20ZBSZn9GjDG+D2/TAT7DhXy9uIdXNEnjg7NKh6Yxhhj/IVfJoKXv91KmUu5+3z7NWCMMX6XCFKz8vng511c0z+BxJhwX4djjDE+53eJ4KUFWxAR7hjcwdehGGNMreBXiSAlI5dPV+zmutNb0zKqyuKpxhjjF/wqETw3bwsNggK47bz2vg7FGGNqDb9JBBvSD/HFqj2MO6sNTSMqH6rSGGP8jd8kgkWbM2gUGsSE39mvAWOM8eTVO4trk1sGtefqfvFEhQf7OhRjjKlV/OYXAUCMdQkZY8xx/CoRGGOMOZ4lAmOM8XOWCIwxxs9ZIjDGGD9nicAYY/ycJQJjjPFzlgiMMcbPWSIwxhg/Z4nAGGP8nFcTgYgMEZFNIrJVRB6oYP29IrJeRFaLyHwRae3NeIwxxhzPa4lARAKBl4FLgK7AGBHpWq7ZL0CSqvYEPgGe8lY8xhhjKubNXwQDgK2qmqKqxcCHwHDPBqr6rarmu2d/AuK9GI8xxpgKeDMRxAGpHvNp7mWVuQn4qqIVIjJBRJJFJDkjI+MUhmiMMaZWnCwWkeuAJODpitar6jRVTVLVpNjY2JoNzhhj6jlvjkewG0jwmI93LzuGiFwAPAQMUtUiL8ZjjDGmAt78RbAM6CgibUUkBBgNzPRsICJ9gFeAYaq634uxGGOMqYTXEoGqlgITgTnABmC6qq4TkcdEZJi72dNABPCxiKwUkZmVbM4YY4yXeHWoSlWdDcwut2ySx/QF3nx/Y4wxVasVJ4uNMcb4jiUCY4zxc5YIjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XOWCIwxxs9ZIjDGGD9nicAYY/ycJQJjjPFzlgiMMcbPWSIwxhg/Z4nAGGP8nCUCY4zxc5YIjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XOWCIwxxs95NRGIyBAR2SQiW0XkgQrWnyMiK0SkVESu9mYsxhhjKua1RCAigcDLwCVAV2CMiHQt12wXcAPwH2/FYYwx5sSCvLjtAcBWVU0BEJEPgeHA+iMNVHWHe53Li3EYY4w5AW92DcUBqR7zae5l/zURmSAiySKSnJGRcUqCM8YY46gTJ4tVdZqqJqlqUmxsrK/DMcaYesWbiWA3kOAxH+9eZowxphbxZiJYBnQUkbYiEgKMBmZ68f2MMcb8Bl5LBKpaCkwE5gAbgOmquk5EHhORYQAi0l9E0oCRwCsiss5b8RhjjKmYN68aQlVnA7PLLZvkMb0Mp8vIGGOMj9SJk8XGGGO8xxKBMcb4OUsExhjj5ywRGGOMn7NEYIwxfs4SgTHG+DlLBMYY4+csERhjjJ+zRGCMMX7OEoExxvg5SwTGGOPnLBEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGGOMn7NEYIwxfs4SgTHG+DlLBMYY4+csERhjjJ/zaiIQkSEisklEtorIAxWsbyAiH7nXLxWRNt6MxxhjzPG8lghEJBB4GbgE6AqMEZGu5ZrdBGSragfgOeBJb8VjjDGmYt78RTAA2KqqKapaDHwIDC/XZjjwtnv6E+B8EREvxmSMMaacIC9uOw5I9ZhPA06vrI2qlopIDhADZHo2EpEJwAT3bK6IbPqNMTUtv+1axuI7ORbfyavtMVp8v13rylZ4MxGcMqo6DZh2stsRkWRVTToFIXmFxXdyLL6TV9tjtPi8w5tdQ7uBBI/5ePeyCtuISBAQBRzwYkzGGGPK8WYiWAZ0FJG2IhICjAZmlmszExjrnr4aWKCq6sWYjDHGlOO1riF3n/9EYA4QCLyhqutE5DEgWVVnAq8D74rIViALJ1l400l3L3mZxXdyLL6TV9tjtPi8QOwLuDHG+De7s9gYY/ycJQJjjPFz9TIR1ObSFiKSICLfish6EVknIndV0OZcEckRkZXux6Sais/9/jtEZI37vZMrWC8i8qJ7/60Wkb41GFsnj/2yUkQOicjd5drU+P4TkTdEZL+IrPVY1kRE5orIFvdzdCWvHetus0VExlbUxguxPS0iG93/fzNEpHElrz3hZ8HLMU4Wkd0e/4+XVvLaE/69ezG+jzxi2yEiKyt5bY3sw5OiqvXqgXNiehvQDggBVgFdy7W5Dfi3e3o08FENxtcS6OuejgQ2VxDfucAsH+7DHUDTE6y/FPgKEOAMYKkP/6/3Aq19vf+Ac4C+wFqPZU8BD7inHwCerOB1TYAU93O0ezq6BmK7CAhyTz9ZUWzV+Sx4OcbJwH3V+Ayc8O/dW/GVW/8sMMmX+/BkHvXxF0GtLm2hqumqusI9fRjYgHOHdV0yHHhHHT8BjUWkpQ/iOB/Ypqo7ffDex1DVRThXvnny/Jy9DVxRwUsvBuaqapaqZgNzgSHejk1Vv1HVUvfsTzj3+fhMJfuvOqrz937SThSf+9hxDfDBqX7fmlIfE0FFpS3KH2iPKW0BHCltUaPcXVJ9gKUVrB4oIqtE5CsR6VajgYEC34jIcnd5j/Kqs49rwmgq/+Pz5f47ormqprun9wLNK2hTG/bljTi/8CpS1WfB2ya6u6/eqKRrrTbsv98B+1R1SyXrfb0Pq1QfE0GdICIRwKfA3ap6qNzqFTjdHb2Al4DPaji8s1W1L07l2NtF5Jwafv8quW9SHAZ8XMFqX++/46jTR1DrrtUWkYeAUuD9Spr48rMwFWgP9AbScbpfaqMxnPjXQK3/e6qPiaDWl7YQkWCcJPC+qv5f+fWqekhVc93Ts4FgEWlaU/Gp6m73835gBs7Pb0/V2cfedgmwQlX3lV/h6/3nYd+RLjP38/4K2vhsX4rIDcBQ4Fp3ojpONT4LXqOq+1S1TFVdwKuVvLdPP4vu48cI4KPK2vhyH1ZXfUwEtbq0hbs/8XVgg6r+o5I2LY6csxCRATj/TzWSqESkoYhEHpnGOam4tlyzmcD/uK8eOgPI8egCqSmVfgvz5f4rx/NzNhb4vII2c4CLRCTa3fVxkXuZV4nIEODPwDBVza+kTXU+C96M0fO805WVvHd1/t696QJgo6qmVbTS1/uw2nx9ttobD5yrWjbjXE3wkHvZYzgfeoBQnC6FrcDPQLsajO1snC6C1cBK9+NS4FbgVnebicA6nCsgfgLOrMH42rnfd5U7hiP7zzM+wRl0aBuwBkiq4f/fhjgH9iiPZT7dfzhJKR0owemnvgnnvNN8YAswD2jibpsEvObx2hvdn8WtwLgaim0rTt/6kc/gkavoWgGzT/RZqMH9967787Ua5+DesnyM7vnj/t5rIj738reOfO482vpkH57Mw0pMGGOMn6uPXUPGGGP+C5YIjDHGz1kiMMYYP2eJwBhj/JwlAmOM8XOWCIxfE5GyctVMT1n1ShFp41mtshrtG4rIPPf0D+6blYzxOvugGX9XoKq9fR2E20BgifvGsjz9tSicMV5lvwiMqYC7hvxT7jryP4tIB/fyNiKywF0Ibb6IJLqXN3fX9V/lfpzp3lSgiLwqztgT34hIWAXv1d5dy/494PfAcqCX+xdKsxr6Jxs/ZonA+Luwcl1DozzW5ahqD+CfwPPuZS8Bb6tqT5xCbS+6l78IfKdOobu+OHeRAnQEXlbVbsBB4KryAajqNvevkuU4dWjexrlztbc69WmM8Sq7s9j4NRHJVdWICpbvAAaraoq7SOBeVY0RkUycUgcl7uXpqtpURDKAeFUt8thGG5yxBjq65+8HglV1SiWxLFPV/iLyKXCXVlK/xphTzX4RGFM5rWT6v1HkMV1GBeflROTf7pPKHd1dREOAWSJyz298T2P+K5YIjKncKI/nJe7pxTgVLgGuBb53T88H/gAgIoEiElXdN1HVW4FHgcdxRjH70t0t9NzJhW9M9dhVQ8bfhcmxg45/rapHLiGNFpHVON/qx7iX3QG8KSJ/AjKAce7ldwHTROQmnG/+f8CpVlldg4B3cEa7+u43/UuM+Y3sHIExFXCfI0hS1Uxfx2KMt1nXkDHG+Dn7RWCMMX7OfhEYY4yfs0RgjDF+zhKBMcb4OUsExhjj5ywRGGOMn/t/PtOb7SLHlKcAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### Translate"],"metadata":{"id":"lIPGj1Tzf-ZC"}},{"cell_type":"code","source":["result = model.translate(['Ich gehe heute nach Hause.'])\n","result[0].numpy().decode()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1669569242429,"user_tz":-60,"elapsed":588,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"66d86ddf-e795-4224-9512-0c02b6a0612f","id":"r7AcJek3f-ZD"},"execution_count":177,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'im going home today . '"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":177}]},{"cell_type":"markdown","source":["#### Attention Plot"],"metadata":{"id":"JlwYzLKlf-ZE"}},{"cell_type":"code","source":["model.plot_attention('Ich gehe heute nach Hause.')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":525},"executionInfo":{"status":"ok","timestamp":1669569242430,"user_tz":-60,"elapsed":26,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"e94db024-bb87-4440-88b7-356df8487160","id":"NfSMRT5ff-ZF"},"execution_count":178,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 720x720 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAnoAAAH8CAYAAABGqwJUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwldX3v/9d7ZtgXN1DRJCq4AqIgIrhHEve4BI254oZGQsT8jASNS7x69bpj4hKNEEVwjTfu159LEhUXgiKuKIsisqkoiwLDzszn/lHV4dCcnmlmpruqv/16Ph7nwTnfqlP16eJMn3d/q77fSlUhSZKk9qwYugBJkiQtDIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjVg1dgDpJ/nQD3vb5qrpykxcjSZKaEO91Ow5J1t7EtxRwl6o6cyHqkSRJS589euNy26r6zXxWTHLZQhcjSZKWNq/RG49jgZtyGvaDwKULVIskSWqAp24lSZIaZY/eiCRZk+TWQ9chSZLaYNAblwxdgCRJaodBT5IkqVGOuh2fP0uyzkEWVfX+xSpGkiQtXQ7GGJF+Lr0r6ObIm0tV1faLVJIkSVrCDHoj0ge9ec+lJ0mStC5eozcupm5JkrTJGPTGxVG3krTIktwzyT8l+XySnfq2JyTZc+japI1l0BuXdd4dI8neSb6wiPVIUtOSPBz4NnB74GHAVv2iXYBXDlWXxiPJLW/qY+iaJxn0RqSqDgL2TfLmJK9LsjNAkrsm+TTwrWErlJaOJI9K8tkkpyT5/b7tL5LsP3RtGpXXAIdV1ROBaybajwP2GaQijc2FwAU34fGbme/vMXB6lRFJ8kzgfcDFwC2B5yR5AXAk8Ang3lV18oAlSktCkgOBdwPvAfYHNusXrQReDHxpoNI0PrsDn5vSPvN7WAJ4Et1nYn3C9M/TYAx64/JC4GVV9YYkfwb8K/AiYK+q+tmwpUlLyouB51bVvyb5i4n2bwKvHqgmjdPFdKdtz5rVvhdw3qJXozE6G/haVV00n5WTnAlcu7AlzZ+nbsdlF+Cj/fOPAWvoTikY8qSb5i7ACVPaVwPOQ6lJHwbenOT36GY+WJXkIcARgJPTi6q603xDXr/+7lV17kLWdFMY9MZlG+BygKpaC1wFjObDovFKskOS+yXZYuhaRuKXwF2ntD8Y8A8nTfp74Od0vTbbAqcAXwa+Abx2wLqkTcJTt+PzmCSX9M9XAI9I8uvJFarqE4tflsYoyXbAe+muHym6nqwzk7wbOL+qXjVgeUM6Cnj7xGnb30/yIOBNwKsGq0qjU1XXAgcmeQXd6doVwPeq6qfDVqaxSRLgGcABwM50v3PPBP4N+FCN9A4U3hljRPo7Y6xPVdXKBS9GS0KSdwH3Ag6l64HYo6rOTPJY4LVVda9BCxxQktfSXfe6Zd90NXBEVb1iuKq0FCS5M3BeVV01dC0ajySfAJ4AnEzX8xtgV7oBPZ+sqgMGLG9OBj1pCUtyHvDEqvp2ksuAe/VBbxfg+1W13cAlDirJ1nS/iFcAp1TV6oFLGlSSLYEX0I1EvjWzLt+pqj2GqGtISV4HnF5Vx/Y9Nv9Od3wuAR5ZVU5rpZmR/EcCf1pV/z5r2SOAjwMHV9WHh6hvXbxGb4lJ8kdD16BRuQUw7SLh7egG8yxLSY5Osl1VXVFVJ1XViVW1Osk2SY4eur4BvQt4Cd0I00/RfTlNPpajA4HT++ePAu4N7Es3EOMNQxWl0Xka8MbZIQ+gqr4IvLlfZ3Ts0VsCktweOAh4NnAHT91qRpLjgE9V1Vv7Hr09qurnSf6Z7rPy6GErHEaSNcBOVfWbWe070F27uCyvT05yMfBnVfWfQ9cyFkmuAu5cVecl+Se678VD+9O3J1XVzQcuUSOQ5JfAn1TVd+ZYvjfwmaq63eJWtn7L8pfdUpBkJfB44DnAw4Ef0k0A+29D1qXReRnwxSS70f17Pqx/vg/dCNNlpb/1UPrHLZJcN7F4JfAY4NfT3rtMXIEj+We7CLgD3Zx5D6fr8YTu35P3H9eMWwG/WsfyXzHSCbYNeiOT5G7AX9CN7Lmcbo6nhwNPr6pThqxN41NV/5Xk/sDhdNOG7A98F9hvmd5F5UK6kXBFd7H0bMXyvn/pm+j+GDhkrCMEB/Bx4MNJfkL3Rf3Fvv3ewBmDVaWx2Yx1T4J8HdffgWdUPHU7Ikm+Tjd65+PAB6rqq337tXQX2Rv0pHXoJ7oN3TxoB3DDWxZdA5xdVb8corahJPnMrKYH0w00OIVZX1xV9bjFqmsskqyiG6DyB8AxVfW9vv2FwGVV9Z4h69M49LNiHE3XKz7N1sBBY7y0yqA3Iv1ppncCR1XVjyfaDXpapyS3Y/ooyu8OU9GwktwBOMdeK0jyvvmuW1UHLWQt0lLVXw+93t8nVfWHC1/NTWPQG5Eke9Kdtn0q3ai49wMfobumxqCnG+k/Mx8E7s6NrydatnMuJtlrXcuXawDWjflZUesMeiPUz3X1ZLpRtg+k66V5CfCeqvrtkLVpXJJ8m+5i8lfT3fbrBv+gq+rsIeoaWn+apbhh+P3vY7OMA/BuwMqq+uGs9j2A65bjH5N+VtQ6g96IJPkD4NzJ0039EP+ZwRm3Ar5cVY8aqESNTJLLgT2r6idD1zIm/anbSZsBewIvB15aVZ9f/KqGl+R44J2zJ3VN8ufA86vqgcNUNhw/K5qPJIfNZ72q+oeFruWmMuiNyFxzf/XLVgKPBZ5dVY9f9OI0Skm+Cby4qr42dC1LQZKHA6+sqgcMXcsQ+rkW96yqM2a17wJ8t6puNkxl47PcPyu6oSQ/X8fiAm4LbDHGHmCnVxmXOedsqqo1wKf7h5axfq64GS8D3pTk7+nuvzh7FOXkqFPBz+mmzViu1gDTwtwtcM642Zb7Z0UTqupO09qT7Ay8lu5yq1HOc2vQk5aembniZszcn3N2W9FNErzszArD0B2PnYBXcf3trpajrwIvT/Lk/o/HmelFXg4sy15hPyvaEEluBbwCOAQ4Hti3qk4atqrpDHrjc3iSdd54vapevVjFjEl/g/p7M30akU8MUtQwRjd8f4Rmh2HovsDPBZ6y+OWMxouBbwBnJPlG3/ZAYFuW4Z1Uen5WNG9JtgIOo/u3dBbwxLFfx+k1eiPSj/46nW6G7blUVe2xSCWNRpI/optq5lZTFi/baUQ0XT9x8qS1wAXAGVW1rn9fzUuyE/B8rj8t+T3gXcttIukZflY0H0lW0N2S9H/RXSLzCrobG4w+RBn0RqQPeredNhhjuUvyY+DbwMuW6xfSXJLcE/hLYBe6wTq/SvIEurtAfG/Y6iRp6UtyCt09kd8OvAO4atp6Y7wu2qA3Iusadbvc9dOI7FFVPxu6ljHpRwZ+Bvg88GjgHlV1ZpK/BR5UVU8YtMABGYDn1t9J5Q+AzSfbl/PobY+J1qXviJkxLTiFkZ5d8hq9cXHU29yOB+4GGPRu6DXAYVX1rn7qjBnHAX87TEnDmxWAHwZs1S/aBXgWsCwDcB9mPkx3Pd7MJMGTX1qj+5JaaB4TzdOSvTbaoDcu/wtY50CM5WTWrYneDRzR/1KeNo3Icr1N0e7A56a0XwzMHk24nBiAp3sr3RQru9JdCvFI4DZ0d1Z54YB1DcljovWqqq8OXcOGMuiNy1voeh6umGlIcg/gRXSj4j5RVf86UG1DOIkb35roqCnrLdtpROgC3e3pRn9N2gs4b9GrGQ8D8HQPAR5TVaclKeCCqjo+ydV04fg/hi1vEB4TrVeSg4Fjq+rq/vVuwOkzA3aSbAP8XVX9zwHLnGrF+lfRIvpnul49AJLsAHyd7o4YdwM+lOSpA9U2hDsBO/f/Xddj56EKHIEPA29O8nt0gXdVP4rwCOD9g1Y2rJkAPNtyD8Bb0U0nAt0xunX//BRg2Y3m73lMNB//zA0nGz+B7prOGdvSzUc5Ovbojct+dBePz3g6cA3dBfaXJHkj3bQIH5725tZU1dlD17AE/D1wDHA2Xc/nKXR/wH2Ibrb25WomAP8ZNw7A7xu0smGdBtydrgf4+8AhSc4FDgV+MWBdQ/KYaD5mX0O/ZK6pd9TtiPQjS3erqrP615+hGyH41/3rXYGvVtWOw1U5jCSvBc6tqnfPaj8EuH1VvWKYysahvw3PXnQh73tV9dOBSxpUks3oAvCf0/1CXsv1Afig5To/WpIDgc2q6pj+GtgvADsAVwPPrKr/M2iBA/CYaD5mT3/WX/t7r6o6s399G+CXYxx1a9AbkSQXAA+tqh/3r88H/raqPtS/3hk4uaq2GbDMQSQ5B3hyVX1rVvt9gY9V1R2GqWxYSY6eY1HRzfN0BvDR5Tr3oAF43fq7zdwdOKeqLlzf+suBx0TTLOWg56nbcfkBcBDdbdAeCuwIfHli+S7AsvzCprtu5oIp7RfRjZBbrnYEHkTXY/Wjvm13ul6s7wB/Crw6yYOq6vvDlDiMJE8B9uf6W+Y9LenOtlTV4wYsbVBTjstM+7I9Lh4TzdNjklzSP18BPCLJr/vXNx+opvUy6I3La4DP99cV7QgcU1W/mlj+RLr7VC5H59AFmjNntT+Y5X1x/fF0U/I8p6qugP/ukfgXuj8cHk03KOMtdF9ky0KSNwN/A3yF7o8jT13gcZnGY6Kb4L2zXr9z1utRfnY8dTsy/XQqDwfOB/6tqtZOLDsYOHG59cwA9Hd6eDnwd1zfy7k/8HrgjVX1pqFqG1KSXwEPq6pTZ7XvCnypqnZKsifwn1U17T7BTer/yj60qj42dC1j4nG5MY+JWmeP3kgk2Qf4Tv+Ffeq0darqqIn17wP8sKqunbZua6rqLf10M2+nu0VR6C6Wfhvw5iFrG9i2wE7c+DNz234ZwKUsv3/rK+hGUOqGPC435jHROk18P6+Z5/qj+n52Hr3xOIGbNpHrV4DfX6BaRqmqXko3Gm5f4H7AjlX1klre3dKfBN6b5MlJ7tg/nkx3iuET/Tr7AD8ZrMJhHAU8begiRsjjcmMeE63Pkv5+Xm5/5Y9ZgNcnuWK9a3Y2X/8qS1s/vczTqurS/vm0dYBlfXH9IcA/AB/k+n/P1wFHA4f3r08Fnrv4pS2uJG+feLkCODDJHwM/5Ma3zPv/FrO2Ebk58NTlflz8rGy4JKcCd6mq5ZQflvT383L6HzV2X6MbVTtfJwBXLlAtY3ER11/cetGQhYxVPwDjkP4axpnPz8+q6vKJdZbLaal7zno983PffVb7cu4B3hWPC/hZ2RjvBJbN9b69Jf397GAMSZKkRnmNniRJUqMMeiPXT6miWTwu03lcpvO43JjHZDqPy3Qel+mWwnEx6I3f6D9EA/G4TOdxmc7jcmMek+k8LtN5XKYb/XEx6EmSJDXKwRhTbL5iq9pq5XZDlwHANWuvZPMVWw1dBgBX3W48I8bXrL6cldtuM3QZrFqdoUu4geuuvpxVW4zguFx61dAl3MA1a69i8xVbDl0G3SwN4zCm3y2sXbv+dRbJNXUVm2f4z8rETZFG4dq6ms2yxdBlkJUrhy7hBsbyu+XS6y68sKp2nLbM6VWm2Grlduy3w5OHLmN0Tn35HYcuYXRu83U7xae55ReW2/zM87RiXF9SY1FXjesPgzGoK0czO8eorLj5zYYuYZS+eMGRZ8+1zG8pSZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVFLNuglOSbJZ4euQ5IkaaxWDV3ARngBkKGLkCRJGqslG/Sq6pKha5AkSRqzJk7dJjkuyT8neUuSi5NckOQFSbZI8s4kv0tyTpKnD123JEnSYlmyQW+KA4HLgPsBbwDeCnwK+AmwN3As8J4kOw1WoSRJ0iJqKej9uKpeVVU/Bf4BuBC4tqreVlVnAK+mu6bvAdPenOTgJCclOematVcuXtWSJEkLpKWg98OZJ1VVwG+AkyfargV+C9x62pur6qiq2ruq9t58xVYLXaskSdKCaynoXTvrdc3R1tLPLEmSNCdDjyRJUqMMepIkSY0y6EmSJDVqKU+Y/KyJ5w+dsnz3KW23XdiqJEmSxsMePUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRq4YuYIxqzXWsvejiocsYnbv99e+GLmF0Tn/bvYcuYZRu8amrhy5hlNbsceehSxilS+6y9dAljM4t/8/3hi5hlGr15UOXsOTYoydJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUqCUR9JI8NEkl2WHoWiRJkpaKJRH0gP8CdgIuGroQSZKkpWLV0AXMR1VdA5w/dB2SJElLyaL06CXZJsn7k6xO8uskL03y2STH9MtvkeTYJL9NcmWS/0yy28T7b3DqNsmz+m3tn+RHSS5P8pUkd5q135f2+1vd7/+VSc5ajJ9ZkiRpaIt16vYtwEOAJwIPA+4FPGhi+THA/YDHA/sAVwBfSLLVOra5BfBS4NnAfsDNgXfPLEzy58ArgZcDewGnAodtkp9GkiRpCVjwU7dJtqULY8+oqv/o254DnNc/vwvwOOAhVfW1vu3pwDnAgcB75tj0KuDQqjq9f88RwNFJUlUFvAA4pqpm3v/6JH8I3HWOOg8GDgbYkq037oeWJEkagcXo0dsF2Aw4caahqi4HftS/vAewFjhhYvklwMnAruvY7tUzIa/3S2Bz4Bb967tP7rP3rbk2VlVHVdXeVbX3ZtlinT+QJEnSUjD2Ube1jmXXzbHu2H8mSZKkRbEYoehnwLXAfWcakmwN7N6/PLWvY7+J5dsD9wRO2Yj9nja5z94+G7E9SZKkJWXBr9GrqtVJjgbemORC4FfA39OFu6qqnyb5NHBkf53c74DXApcCH96IXb8NeF+SbwNfpxsIcj/gtxuxTUmSpCVjsebROxzYBvgMsBr4R+A2wFX98oOAt/bLtwSOBx5ZVVdu6A6r6l+T7Ay8Adga+ATdqNzHb+g2JUmSlpJFCXpVtRp4ev8gyRbA3wCf65f/FnjmOt5/HJCJ18fQTcky5zp92+uA1828TvJJ4IwN/0kkSZKWjkUJekn2pBtdeyKwHfB3/X8/uoD73Br4K+ALdAM3DqDrzTtgofYpSZI0Jot5C7TDgLvRha7vAw+uqvMWcH8FPAp4GbAV8FPgaVX1yQXcpyRJ0mgs1qnb7wF7L8a+JvZ5JfBHi7lPSZKkMXHOOUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEatGrqAUdpmK9bee/ehqxidnHDy0CWMzq6vOWfoEkZp7Qr/hpzmum38lTvNb/ZdM3QJo7PDl281dAmjdN15vxi6hCXH38aSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjVrwoJfkuCT/tND7kSRJ0g3ZoydJktQog54kSVKjFivorUjyuiQXJvlNkiOSrABIcoskxyb5bZIrk/xnkt1m3pjkWUlWJ3lUktOSXJHkM0luluRJSX6a5JIkH0iy1cT7kuTFSX7Wb/fkJE9bpJ9XkiRpcIsV9A4ErgPuDzwf+BvgKf2yY4D7AY8H9gGuAL4wGdqALYC/7bezP7A38HHgmcABwBOAxwLPm3jP/waeAxwK7Aq8HjgyyWM2+U8nSZI0QqsWaT+nVNX/7J//JMlzgf2TnAQ8DnhIVX0NIMnTgXPoQt17Juo8tKpO79f5MPBC4DZVdWHf9mngD4G3JNkGOAx4eFV9vd/Gz5PsQxf8/v/ZBSY5GDgYYMstbrZJf3hJkqQhLFbQ++Gs178Ebg3cA1gLnDCzoKouSXIyXS/cjKtnQl7v18D5MyFvom3mPbsCW9L1DNbEOpsBZ00rsKqOAo4C2H6729e0dSRJkpaSxQp61856Xaz/tPFk2LpuyrJ1bXPmv39C1zu4rlokSZKatFhBby6n0oWy/YCZU7fbA/cE3rcR2z0FuBq4Q1V9eWOLlCRJWooGDXpV9dP+2roj+2vkfge8FrgU+PBGbPeyJEcARyQJXYjcFtgXWNufppUkSWraGObROwg4EfhM/9+tgUdW1ZUbud1XAK8CDgd+DPwH3Qjdn2/kdiVJkpaEBe/Rq6qHTml71sTz39JNkzLX+4+hm4Jlsu0I4IhZbS+Z9bqAd/QPSZKkZWe9PXpJtphPmyRJksZlPqduT5hnmyRJkkZkzlO3SW4L3B7YKsmeQPpF29NdRydJkqQRW9c1eo8AngX8HvAWrg96lwIvW9iyJEmStLHmDHpVdSxwbJIDqurji1iTJEmSNoH5XKP3hCT/ffPXJHdI8qUFrEmSJEmbwHyC3jeAbyV5dJLn0s1H99aFLUuSJEkba73z6FXVkUl+DHwFuBDYs6rOX/DKJEmStFHmM4/e04GjgWfQTVz8uST3WuC6JEmStJHmc2eMA4AHVtVvgI8k+SRwLHDvBa1MkiRJG2U+p26fAJBk66q6oqpOTLLPwpcmSZKkjTGfU7f7JTkFOK1/fS8cjCFJkjR68xl1+1a6yZMvAqiqHwAPXsiiJEmStPHmE/SoqnNnNa1ZgFokSZK0Cc1nMMa5Se4PVJLNgBcApy5sWZIkSdpY8+nROwQ4FLg98Au60bbPW8iiJEmStPHm06N3t6o6cLIhyQOA4xemJEmSJG0K8+nRe8c82yRJkjQic/boJdkPuD+wY5LDJhZtD6xc6MIkSZK0cdZ16nZzYNt+ne0m2i8FnrSQRUmSJGnjzRn0quqrwFeTHFNVZy9iTZIkSdoE1nuNniFPkiRpaZrXhMmSJElaeuZzr9sHzKdNkiRJ4+L0KpIkSY1yehVJkqRGOb2KJElSo5xeZYpccx2bnXfR0GWMzpoVGbqE0Vl7yaVDlzBKFz7lXkOXMEqbr147dAmjdPfDfjB0CaPzi4P3GrqEUbrtu349dAnjdO3ci+Zzr9tjktTsxqp62EaUJEmSpAU2n6B3+MTzLYEDgOsWphxJkiRtKusNelX1nVlNxyc5cYHqkSRJ0iay3qCX5JYTL1cA9wFutmAVSZIkaZOYz6nb7wAFhO6U7c+B5yxkUZIkSdp48zl1e6fFKESSJEmb1nxO3W4JPA94IF3P3teBd1fVVQtcmyRJkjbCfE7dvh+4jOtve/ZU4APAkxeqKEmSJG28+QS93atq14nXX0lyykIVJEmSpE1jxTzW+W6SfWdeJLkfcNLClSRJkqRNYT49evcB/ivJOf3rPwBOT3IyUFW1x4JVJ0mSpA02n6D3yAWvQpIkSZvcfILe/66qp082JPnA7DZJkiSNy3yu0dtt8kWSVXSncyVJkjRicwa9JC9NchmwR5JLk1zWv/418OlFq1CSJEkbZM6gV1Wvr6rtgDdX1fZVtV3/uFVVvXQRa5QkSdIGmM81ep9P8uDZjVX1tQWoR5IkSZvIfILeiyaebwnsA3wHeNiCVCRJkqRNYr1Br6r+ZPJ1kt8H3rpgFUmSJGmTmM+o29nOA+6xqQuRJEnSprXeHr0k7wCqf7kCuDfw3YUsSpIkSRtvPtfoTd7X9jrgI1V1/ALVI0mSpE1kPkHvo8Cd++dnVNVVC1iPJEmSNpF1TZi8Ksmb6K7JOxZ4P3Bukjcl2WyxCpQkSdKGWddgjDcDtwTuVFX3qaq9gF2AmwNHLEZxkiRJ2nDrCnqPBZ5bVZfNNFTVpcBfAY9e6MIkSZK0cdYV9KqqakrjGq4fhStJkqSRWlfQOyXJM2Y3JnkacNrClSRJkqRNYV2jbg8FPpHk2XS3PAPYG9gKeOJCFyZJkqSNM2fQq6pfAPdL8jBgt775c1X1pUWpbJYknwUurKpnDbF/SZKkpWY+97r9MvDlDd1BkuOAH1XV8zd0G5IkSbrpNuRet5IkSVoCFjToJTkGeAhwaJLqH3dM8uAk30pyVZJfJ/nHJJtPvG/rJMckWd0vf9mUbT8tybeTXJbkN0n+Lcnt+2VJckaSw2e95y59DXst5M8tSZI0Bgvdo/cC4ATgfcBO/eNa4PPA94A9gecA/wN4/cT7jgD+GDgA2L9f78Gztr058ErgXnRz/u0AfAS6eWGA9wIHzXrPs4HvV9V3N8lPJ0mSNGILGvSq6hLgGuCKqjq/qs4Hngf8EnheVZ1aVZ8FXgI8v+/J25Yu/L24qr5YVT+iC2xrZ2376Kr6XFWdWVUn0k3k/KAkv9ev8j7grkn2BUiyEngGXQC8kSQHJzkpyXhTzpkAAAsDSURBVEnXrLlyEx8JSZKkxTfENXr3AL5ZVZPB7Rt0PXR3prvN2uZ0PYEAVNVq4OTJjSTZK8mnk5yd5DLgpH7RH/TvOR/4LF0vHsAj6W7p9qFpRVXVUVW1d1XtvfnKrTbyR5QkSRre2AZjzOuOG0m2Ab4IXAE8HbgvXZCDLiTOeA/wlCRb0wW+T1bVbzdduZIkSeO1GEHvGmDlxOtTgX2TTO77gf16P+sf1wL7zizsg93uE+vfne6avJdV1deq6jTg1lP2/QXgUuAQ4E+Aozf6p5EkSVoiFiPonQXs04+23QF4F3A74F1J7pHkMcAbgH+qqiv607TvBd6Y5I+T7EYX0CbD4jnA1XTX9e3cb+M1s3fc35f3aLqBHr8ABpnsWZIkaQiLEfSOoOutOwW4ANgMeBTdSNrv0wWxjwCTU6gcDnwF+GT/3x8BX5tZWFUXAM8EntBv95XAYXPs/2i607nv60fjSpIkLQvrvTPGxqqqnwD7zWo+C7jfOt5zOd0I2WesY52PAh+d1Zwpq94WWAMcs/5qJUmS2rHgQW8oSbYAdqQ7pfvJqjpn4JIkSZIW1dhG3W5K/wM4m27QxlyndSVJkprVbNCrqmOqamVV7VVV5w5djyRJ0mJrNuhJkiQtdwY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUauGLmCM1m6xGVftcuuhyxidVef+cugSRqeuvHLoEkZpx8+cPnQJo5SbbT90CaN03dVXD13C6Oz8pJ8OXcIoXXHkyqFLGKdr515kj54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6PWSHJzkpCQnXXvt5UOXI0mStNEMer2qOqqq9q6qvTfbbJuhy5EkSdpoBj1JkqRGGfQkSZIatayCXpLnJzlt6DokSZIWw7IKesAOwN2GLkKSJGkxLKugV1WvqqoMXYckSdJiWFZBT5IkaTkx6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjVg1dwBjd7U4X8qUPvnfoMkbn0XvsP3QJo7Pm4t8NXcIorbn4t0OXME4XXTx0BVoiLn/wBUOXMEort99+6BLG6aq5F9mjJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktSoBQt6SY5LUv1j34XazzxrOWuilh2GrEWSJGmxLHSP3vuAnYDvAEyErdmPQ/rlD+1fn5Zk1eSG+rB2+MTrySB5TZJfJflCkqclyaw67gscsLA/qiRJ0rgsdNC7oqrOr6prJ9qeSxf+Jh/HznrfHYDnzGP7M0FyZ+BxwAnAkcAnk6ycWamqLgAu3tAfQpIkaSlatf5VNrnfVdX561nn7cCrknywqi5fx3pXTGzrPODbSb4JfAF4Bl0QlCRJWpbGOhjjHcC1wGE39Y1V9UXgZDxVK0mSlrkhgt4Hkqye9bjnrHWuAl4BvCjJjhuwj1PoTufOW5KDk5yU5KQLLlqzAbuUJEkalyGC3ouAe896nD5lvQ8AZ9EFvpsqQN2UN1TVUVW1d1XtveOtVq7/DZIkSSM3xDV651fVGetbqarWJnkJ8Kkkb7uJ+9gVOHODqpMkSWrEWK/RA6CqPgccD7x2vu9J8ghgd+BjC1WXJEnSUjBEj97Nk9x2Vtvqqlo9x/ovBr5JNzhjtq37ba2im2bl0f36nwY+uInqlSRJWpKG6NH7F+BXsx4vmWvlqvo2Xe/cFlMWH9S//0zg/wL7AYcAT6wqR1RIkqRlbVF79Kpq9h0rZi8/jm4gxez2pwBPmdX20E1ZmyRJUmsWukfv4H76lPsu8H7WKcmPgc8PWYMkSdJiW8gevQOBrfrn5y7gfubj0cBm/XNvhSZJkpaFBQt6VfWLhdr2TVVVZw9dgyRJ0mIb9fQqkiRJ2nAGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVGpqqFrGJ0kFwBnD11HbwfgwqGLGCGPy3Qel+k8LjfmMZnO4zKdx2W6sRyXO1TVjtMWGPRGLslJVbX30HWMjcdlOo/LdB6XG/OYTOdxmc7jMt1SOC6eupUkSWqUQU+SJKlRBr3xO2roAkbK4zKdx2W6eR+XJKs39c6T3DHJU2/qsnlu+6FJ7r8Bb/WzMp3HZTqPy3SjPy5eoydJE5KsrqptN/E2HwocXlWPvSnL5rntVwGrq+qIjalRUpvs0ZOkKfqesuOSfCzJaUk+lCT9srOSvCnJyUlOTHLnvv2YJE+a2MZM7+AbgAcl+X6SF87a1Q2WJVmZ5M1Jvp3kh0n+st/WC5Mc3T+/Z5IfJdkVOAR4Yf/+By3sUZG01KwaugBJGrE9gd2AXwLHAw8AvtEvu6Sq7pnkGcBbgXX1yL2EuXvtbrAsycH9tu+bZAvg+CT/DrwNOC7JE4GXA39ZVackeTf26Emagz16kjS3E6vqvKpaC3wfuOPEso9M/He/TbjPhwPPSPJ94FvArYC79DU8C/gA8NWqOn4T7lNSo+zRk6S5XT3xfA03/J1ZU55fR/8HdJIVwOYbsM8Af11VX5yy7C7AauB2G7BdScuQPXqStGGeMvHfE/rnZwH36Z8/Dtisf34ZsN0c25m97IvAXyXZDCDJXZNsk+RmwNuBBwO3mrgWcF3blrTMGfQkacPcIskPgRcAMwMs/gV4SJIf0J3Ovbxv/yGwJskPpgzGmL3sPcApwHeT/Ag4kq4n8R+Bd1bVT4DnAG9Icmvg/wJPdDCGpGmcXkWSbqIkZwF7V9UY7nEpSXOyR0+SJKlR9uhJkiQ1yh49SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhr1/wBdHyLy+fbbBQAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["### Using tf.keras.layers.AdditiveAttention"],"metadata":{"id":"SEMnhraogA78"}},{"cell_type":"code","source":["model = Translator(UNITS, context_text_processor, target_text_processor, attention_mechanism=tf.keras.layers.AdditiveAttention)\n","\n","logits = model((ex_context_tok, ex_tar_in))\n","\n","print(f'Context tokens, shape: (batch, s, units) {ex_context_tok.shape}')\n","print(f'Target tokens, shape: (batch, t) {ex_tar_in.shape}')\n","print(f'logits, shape: (batch, t, target_vocabulary_size) {logits.shape}')"],"metadata":{"id":"MWMYU-mMgA7_","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569242430,"user_tz":-60,"elapsed":21,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"41b29b38-a78b-4f27-d3ff-88d0f47c5a0d"},"execution_count":179,"outputs":[{"output_type":"stream","name":"stdout","text":["Context tokens, shape: (batch, s, units) (64, 20)\n","Target tokens, shape: (batch, t) (64, 18)\n","logits, shape: (batch, t, target_vocabulary_size) (64, 18, 5000)\n"]}]},{"cell_type":"code","source":["model.compile(optimizer='adam',\n","              loss=masked_loss,\n","              metrics=[masked_acc, masked_loss])"],"metadata":{"id":"4y3J13t0gA8B","executionInfo":{"status":"ok","timestamp":1669569242431,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":180,"outputs":[]},{"cell_type":"markdown","source":["#### Model Training"],"metadata":{"id":"dtQSBMntgA8D"}},{"cell_type":"code","source":["history = model.fit(\n","    train_ds.repeat(), \n","    epochs=20,\n","    steps_per_epoch = 100,\n","    validation_data=val_ds,\n","    validation_steps = 20,\n","    callbacks=[\n","        tf.keras.callbacks.EarlyStopping(patience=3)])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569514564,"user_tz":-60,"elapsed":272146,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"a6b366c7-3ed1-4a7b-c845-bf485225e699","id":"IDR21KLzgA8F"},"execution_count":181,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/20\n","120/120 [==============================] - 33s 176ms/step - loss: 5.2227 - masked_acc: 0.2071 - masked_loss: 5.2227 - val_loss: 4.3257 - val_masked_acc: 0.3166 - val_masked_loss: 4.3257\n","Epoch 2/20\n","120/120 [==============================] - 12s 98ms/step - loss: 4.0838 - masked_acc: 0.3470 - masked_loss: 4.0838 - val_loss: 3.8200 - val_masked_acc: 0.3893 - val_masked_loss: 3.8200\n","Epoch 3/20\n","120/120 [==============================] - 11s 87ms/step - loss: 3.5960 - masked_acc: 0.4078 - masked_loss: 3.5960 - val_loss: 3.3118 - val_masked_acc: 0.4399 - val_masked_loss: 3.3118\n","Epoch 4/20\n","120/120 [==============================] - 10s 80ms/step - loss: 3.1968 - masked_acc: 0.4540 - masked_loss: 3.1968 - val_loss: 3.0362 - val_masked_acc: 0.4713 - val_masked_loss: 3.0362\n","Epoch 5/20\n","120/120 [==============================] - 9s 77ms/step - loss: 2.8930 - masked_acc: 0.4955 - masked_loss: 2.8930 - val_loss: 2.7854 - val_masked_acc: 0.5107 - val_masked_loss: 2.7854\n","Epoch 6/20\n","120/120 [==============================] - 9s 77ms/step - loss: 2.6500 - masked_acc: 0.5253 - masked_loss: 2.6500 - val_loss: 2.5329 - val_masked_acc: 0.5395 - val_masked_loss: 2.5329\n","Epoch 7/20\n","120/120 [==============================] - 10s 81ms/step - loss: 2.4613 - masked_acc: 0.5527 - masked_loss: 2.4613 - val_loss: 2.3700 - val_masked_acc: 0.5672 - val_masked_loss: 2.3700\n","Epoch 8/20\n","120/120 [==============================] - 9s 76ms/step - loss: 2.2690 - masked_acc: 0.5793 - masked_loss: 2.2690 - val_loss: 2.2216 - val_masked_acc: 0.5821 - val_masked_loss: 2.2216\n","Epoch 9/20\n","120/120 [==============================] - 10s 80ms/step - loss: 2.1472 - masked_acc: 0.5991 - masked_loss: 2.1472 - val_loss: 2.0994 - val_masked_acc: 0.6018 - val_masked_loss: 2.0994\n","Epoch 10/20\n","120/120 [==============================] - 9s 75ms/step - loss: 2.0575 - masked_acc: 0.6097 - masked_loss: 2.0575 - val_loss: 2.0163 - val_masked_acc: 0.6107 - val_masked_loss: 2.0163\n","Epoch 11/20\n","120/120 [==============================] - 10s 81ms/step - loss: 1.9687 - masked_acc: 0.6238 - masked_loss: 1.9687 - val_loss: 1.9694 - val_masked_acc: 0.6184 - val_masked_loss: 1.9694\n","Epoch 12/20\n","120/120 [==============================] - 9s 76ms/step - loss: 1.9246 - masked_acc: 0.6299 - masked_loss: 1.9246 - val_loss: 1.9105 - val_masked_acc: 0.6234 - val_masked_loss: 1.9105\n","Epoch 13/20\n","120/120 [==============================] - 9s 79ms/step - loss: 1.8531 - masked_acc: 0.6389 - masked_loss: 1.8531 - val_loss: 1.8022 - val_masked_acc: 0.6458 - val_masked_loss: 1.8022\n","Epoch 14/20\n","120/120 [==============================] - 9s 76ms/step - loss: 1.8041 - masked_acc: 0.6474 - masked_loss: 1.8041 - val_loss: 1.7259 - val_masked_acc: 0.6517 - val_masked_loss: 1.7259\n","Epoch 15/20\n","120/120 [==============================] - 9s 76ms/step - loss: 1.7054 - masked_acc: 0.6599 - masked_loss: 1.7054 - val_loss: 1.6923 - val_masked_acc: 0.6613 - val_masked_loss: 1.6923\n","Epoch 16/20\n","120/120 [==============================] - 10s 80ms/step - loss: 1.6878 - masked_acc: 0.6634 - masked_loss: 1.6878 - val_loss: 1.6588 - val_masked_acc: 0.6651 - val_masked_loss: 1.6588\n","Epoch 17/20\n","120/120 [==============================] - 9s 76ms/step - loss: 1.6386 - masked_acc: 0.6715 - masked_loss: 1.6386 - val_loss: 1.6237 - val_masked_acc: 0.6711 - val_masked_loss: 1.6237\n","Epoch 18/20\n","120/120 [==============================] - 9s 77ms/step - loss: 1.6227 - masked_acc: 0.6732 - masked_loss: 1.6227 - val_loss: 1.6559 - val_masked_acc: 0.6678 - val_masked_loss: 1.6559\n","Epoch 19/20\n","120/120 [==============================] - 9s 77ms/step - loss: 1.5885 - masked_acc: 0.6797 - masked_loss: 1.5885 - val_loss: 1.6633 - val_masked_acc: 0.6618 - val_masked_loss: 1.6633\n","Epoch 20/20\n","120/120 [==============================] - 11s 92ms/step - loss: 1.5755 - masked_acc: 0.6807 - masked_loss: 1.5755 - val_loss: 1.5855 - val_masked_acc: 0.6809 - val_masked_loss: 1.5855\n"]}]},{"cell_type":"markdown","source":["#### Graphs:"],"metadata":{"id":"X4hCMFpFgA8G"}},{"cell_type":"code","source":["plt.plot(history.history['loss'], label='loss')\n","plt.plot(history.history['val_loss'], label='val_loss')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569514565,"user_tz":-60,"elapsed":23,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"87a19604-e68f-4b7e-eafa-81c9753f182c","id":"PSfLy78FgA8H"},"execution_count":182,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb4445941d0>"]},"metadata":{},"execution_count":182},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1d3H8c/Jvu8hCwHCvoZNwKKAokWtVVBbRWqtu617q7W1tU8fa7V9umkX21ptVVBUUGtFUdQKgqggi2ET2QJIEpYsJJB9mfP8cScYIAmBZDLJzff9es1r7tx7Zu4vw/DNzZlzzzXWWkRExH0C/F2AiIj4hgJeRMSlFPAiIi6lgBcRcSkFvIiISwX5u4DGkpKSbGZmpr/LEBHpMtasWVNorU1ualunCvjMzExWr17t7zJERLoMY8zu5rapi0ZExKUU8CIiLqWAFxFxqU7VBy8i3U9tbS25ublUVVX5u5ROLSwsjIyMDIKDg1v9HFcEfEVNHbV1ltiI1v/gItI55ObmEh0dTWZmJsYYf5fTKVlrKSoqIjc3l759+7b6eV2+i6a8uo7TH36PJz7Y4e9SROQUVFVVkZiYqHBvgTGGxMTEk/4rp8sHfGRoEKN7x7FgXT6aGVOka1K4n9ipvEddPuABpo9KZ09xJZ/uKfF3KSIinYYrAv78EamEBAWwIDvf36WISBcUFRXl7xJ8whUBHxMWzDmDe/DG+r3U1Xv8XY6ISKfgioAHmDE6ncKyalbkFPu7FBHpoqy13HvvvYwYMYKsrCzmzZsHwN69e5kyZQqjR49mxIgRfPDBB9TX13Pttdceafvoo4/6ufrj+XSYpDFmF3AYqAfqrLXjfLWvqUN6EBUaxGvZeUwamOSr3YiID/3i9U18ln+oXV9zWHoM/3vx8Fa1/fe//012djbr1q2jsLCQ8ePHM2XKFJ5//nnOP/987r//furr66moqCA7O5u8vDw2btwIQElJ5/sOsCOO4Kdaa0f7MtwBwoIDOX94Kos27aOqtt6XuxIRl1q+fDmzZs0iMDCQlJQUzjrrLFatWsX48eN5+umneeCBB9iwYQPR0dH069ePnJwc7rjjDhYtWkRMTIy/yz+OK050ajB9dDqvrM3l/S0FXDAi1d/liMhJau2RdkebMmUKy5YtY+HChVx77bXcfffdfOc732HdunW8/fbbPP7448yfP5+nnnrK36UexddH8BZ4xxizxhhzs4/3xZn9E0mMDOH1dRpNIyInb/LkycybN4/6+noKCgpYtmwZEyZMYPfu3aSkpHDTTTdx4403snbtWgoLC/F4PHzjG9/goYceYu3atf4u/zi+PoKfZK3NM8b0AN41xnxurV3WuIE3+G8G6N27d5t2FhQYwNdHpjFv1R4OV9USHaapC0Sk9S699FI+/vhjRo0ahTGG3/72t6SmpjJ79mx+97vfERwcTFRUFHPmzCEvL4/rrrsOj8cZuffrX//az9Ufz3TU2Z/GmAeAMmvt75trM27cONvWC36s2V3MN/7+MY9cMYrLxma06bVExPc2b97M0KFD/V1Gl9DUe2WMWdPcd5w+66IxxkQaY6IbloHzgI2+2l+Dsb3j6RkXzms66UlEujlf9sGnAMuNMeuAT4CF1tpFPtwf4MzXMH10Osu3F1JUVu3r3YmIdFo+C3hrbY61dpT3Ntxa+7Cv9nWs6aPSqfdY3tywt6N2KSLS6bjmTNbGhqRGMygligUaTSMi3ZgrA94Yw/RR6azadZC8kkp/lyMi4heuDHiA6aN6AmhMvIh0W64N+N6JEYzuFafRNCLSbbk24MGZYXLz3kNs23/Y36WIiEu0NHf8rl27GDFiRAdW0zJXB/zXR6YRYNCXrSLSLblqsrFj9YgO44z+SSxYl8/d0wbpuo8ind1b98G+De37mqlZ8LX/a3bzfffdR69evbjtttsAeOCBBwgKCmLJkiUcPHiQ2tpaHnroIWbMmHFSu62qquKWW25h9erVBAUF8cgjjzB16lQ2bdrEddddR01NDR6Ph1deeYX09HSuuOIKcnNzqa+v53/+53+YOXNmm35scPkRPDgzTO4uqmBdbqm/SxGRTmjmzJnMnz//yOP58+dzzTXX8Oqrr7J27VqWLFnCPffcw8lO6/LXv/4VYwwbNmzghRde4JprrqGqqorHH3+cu+66i+zsbFavXk1GRgaLFi0iPT2ddevWsXHjRi644IJ2+dlcfQQPcP7wVH726kYWZOczulecv8sRkZa0cKTtK2PGjOHAgQPk5+dTUFBAfHw8qamp/OAHP2DZsmUEBASQl5fH/v37SU1t/TTky5cv54477gBgyJAh9OnTh61btzJx4kQefvhhcnNzueyyyxg4cCBZWVncc889/PjHP+aiiy5i8uTJ7fKzuf4IPjY8mKlDknl9fT71no6ZWE1EupbLL7+cl19+mXnz5jFz5kzmzp1LQUEBa9asITs7m5SUFKqqqtplX9/61rdYsGAB4eHhXHjhhSxevJhBgwaxdu1asrKy+NnPfsaDDz7YLvtyfcCDMya+4HA1K3OK/F2KiHRCM2fO5MUXX+Tll1/m8ssvp7S0lB49ehAcHMySJUvYvXv3Sb/m5MmTmTt3LgBbt27liy++YPDgweTk5NCvXz/uvPNOZsyYwfr168nPzyciIoJvf/vb3Hvvve02t7zru2gAzh3ag8iQQF7LzueMAbpeq4gcbfjw4Rw+fJiePXuSlpbGVVddxcUXX0xWVhbjxo1jyJAhJ/2at956K7fccgtZWVkEBQXxzDPPEBoayvz583n22WcJDg4mNTWVn/70p6xatYp7772XgIAAgoOD+fvf/94uP1eHzQffGu0xH3xz7p6XzX8372fVz75KaFCgT/YhIidP88G3XqeZD76zmT46nUNVdSzdUuDvUkREOkS36KIBOHNAEgmRIby2Lp/zhuuC3CJy6jZs2MDVV1991LrQ0FBWrlzpp4qa1m0CPjgwgK9npfHSmj2UVdcRFdptfnSRTs9a26VORMzKyiI7O7tD93kq3endposGnG6aqloP7362z9+liIhXWFgYRUVFpxRg3YW1lqKiIsLCwk7qed3qMPY07/VaF2Tnc+kYXZBbpDPIyMggNzeXggJ9P9aSsLAwMjJOLre6VcAHBBguGpXGvz7YSXF5DQmRIf4uSaTbCw4Opm/fvv4uw5W6VRcNwIxRPanT9VpFpBtwR8BXHgRPfauaDk2LZkCPKBboQiAi4nJdP+AriuHxKbD4l61qboxhxqh0PtlVTL6u1yoiLtb1Az4iAQacA8sfhc8WtOopF49KB3S9VhFxt64f8ABf+y30PA3+cwsUbD1h88ykSEb1itOVnkTE1dwR8EGhcMWzEBQG866C6hNfg3XGqHQ25R9i+4GyDihQRKTjuSPgAWJ7wuVPQ9EO50j+BCdNXKTrtYqIy7kn4AH6ToFpv4DNr8OHf2yxaY+YMCb2T2RBdp7OoBMRV3JXwANMvB2GXwrvPQg7lrTYdPqodHYVVbAhT9drFRH3cV/AGwPTH4OkwfDy9VDyRbNNLxieRkhgAK9pTLyIuJD7Ah4gNApmPgeeOph3NdQ2fS3F2IhgzhqczBu6XquIuJA7Ax4gaQBc+g/Ymw0L72n2S9cZo9PZf6ialTt1vVYRcRf3BjzAkAthyr2Q/RysebrJJucOSSEqNIg/vruNmjpPBxcoIuI7Pg94Y0ygMeZTY8wbvt5Xk87+CQz4Krz5I9iz6rjN4SGBPHzpCD7ZVcwvXt/khwJFRHyjI47g7wI2d8B+mhYQCJc9CTHpMP87UHbguCYzRvfku2f1Y+7KL3huxW4/FCki0v58GvDGmAzg68A/fbmfE4pIcL50rSyGl66D+rrjmvzo/CFMHZzMAws2sSJH/fEi0vX5+gj+j8CPgGY7t40xNxtjVhtjVvv0ii5pI+HiP8Hu5fDf/z1uc2CA4U+zxtA7MYJb565lT3GF72oREekAPgt4Y8xFwAFr7ZqW2llrn7DWjrPWjktOTvZVOY5RV8KEm+Hjx2DjK8dtjgkL5snvjKO23sNNc1ZTUXP8kb6ISFfhyyP4M4HpxphdwIvAOcaY53y4v9Y572HodTq8djvs/+y4zf2To/jzrDFs2X+YH760TtMYiEiX5bOAt9b+xFqbYa3NBK4EFltrv+2r/bVaUAhcPhtCo52ZJytLjmsydXAP7rtgCG9u2MdfFm/3Q5EiIm3n7nHwzYlJc0K+5At49XvgOf4rgpun9OPSMT155N2tvL1pnx+KFBFpmw4JeGvt+9baizpiX63WZyKc/yvY+hZ88PvjNhtj+PVlWYzMiOXuedls2XfiOeZFRDqT7nkE32DCzTByJiz5FWx797jNYcGBPHH1OCJCg7hxzioOltf4oUgRkVPTvQPeGLjoj5AyAl65AYp3HtckNTaMf1x9GvtLq7nt+bXU1ms6AxHpGrp3wAOERMDMOc7y/Kuh5vjx72N7x/PwpSP4aEcRDy/030m5IiInQwEPkNDPmc5g3wZYeHeTM09ePq4X15/Zl2c+2sX8VXv8UKSIyMlRwDcYdD6cdR+sewFW/6vJJj+9cAiTByZx/382sGZ3cQcXKCJychTwjZ31Yxh4Hrx1X5MzTwYFBvCXWWNIjwvnu8+uJb+k0g9Fioi0jgK+sYAA5yIhLcw8GRcRwj+/M46q2nq+++waqmrr/VCoiMiJKeCP1XjmyZevb3LmyYEp0fxx5mg25pfyo5fXazoDEemUFPBNSRvpDJ/c9QG890CTTb46LIUfnjeYBevyeXxpTsfWJyLSCgr45oyeBeNvhI/+AptebbLJrWf35+sj0/jt25/z7mf7O7hAEZGWKeBbcv6vIWM8/Oc2KNhy3GZjDL/75kiyesZy29y1LNlyfJ+9iIi/KOBbEhQCV8xxToZ68SqoOnRck4iQIOZcP4GBKVF8d84ahbyIdBoK+BOJSYfLn4HiHHjt1iZPgoqLCGHujac7If/sGt5XyItIJ6CAb43MSTDtQdj8Onz4pyabNIT8gOQoblbIi0gnoIBvrYm3wfBL4b1fQM7SJpscG/JLt/rwGrMiIieggG8tY2D6Y5A4EF6+Dkpzm2wWH/llyN80Z7VCXkT8RgF/MkKj4Mq5UFfjnOlaV91ks4aQ76+QFxE/UsCfrKSBcOnfIW8NvPXjZpvFR4bwfKOQX6aQF5EOpoA/FUMvhkk/gDVPw6fPNdus4Ui+X1KkQl5EOpwC/lRN/Rn0PQveuBvys5ttlhAZwvM3fYW+3pD/YJtCXkQ6hgL+VAUGwTefgshkmHc1VDQ/P3zjkL9x9mqWbyvswEJFpLtSwLdFZJJzub+yffCv85x++WY0DvkbZq9SyIuIzyng26rnafDtV6C2Av45Dd7/P6ivbbJpgrdPXiEvIh1BAd8e+k6BWz6CrG/C+792juYLtzXZNDEq9KiQ/3C7Ql5EfEMB317C4+CyJ+Dy2XBwJzw+GT55ssm5axpCPjMxkuufUciLiG8o4Nvb8Evg1hXO/DVv/hCeuwwO5R/XLDEqlOdvckL+htmrNHeNiLQ7BbwvRKfCVS/B1x+BL1bA3ybChpePa5YYFcpcb8hf98wq/vTfbXg8uvyfiLQP09rriRpjzgAygaCGddbaOe1ZzLhx4+zq1avb8yX9r2gHvPpdyF0FI74BF/7eue5rIxU1ddz/6kZe/TSPyQOT+OPM0SRGhfqpYBHpSowxa6y145ra1qojeGPMs8DvgUnAeO+tyReUYyT2h+sWwTk/g89eg7+fAdvfO6pJREgQj1wxil9dmsXKnGK+/uflrNnd/Lh6EZHWaNURvDFmMzDMtvZw/xS58gi+sfxs52i+4HMYf5Mzx3xIxFFNNuaVcsvcNewtqeK+rw3hhkl9Mcb4qWAR6ezafAQPbARS26+kbip9NNz8PnzlNlj1JPxj8nEnR43oGcsbd0zmnCE9eGjhZm55bi2HqpoeVy8i0pLWBnwS8Jkx5m1jzIKGW0tPMMaEGWM+McasM8ZsMsb8ou3lukBwOFzwK/jOAqitck6OWvLro06Oig0P5h9Xn8b9Fw7l3c37mf6X5WzKL/Vj0SLSFbW2i+asptZba5u+tJHzHANEWmvLjDHBwHLgLmvtiuae4/oummNVljhTDq9/EVKz4OI/OWfGNrJqVzG3P7+WgxW1/HLGcK4Y10tdNiJyRJu7aLxBvgsI9i6vAtae4DnWWlvmfRjsvWkMYGPhcXDZP2Dmc1BeCP/8Krx1H1QfPtJkfGYCC++czITMBH78ygZ++NJ6Kmvq/Vi0iHQVrR1FcxPwMvAP76qewH9a8bxAY0w2cAB411q7sok2NxtjVhtjVhcUdNOpdIdeDLethHHXw8rH4a+nw+dvHtmcFBXK7OsncOe5A/n3p7lc8tcP2VFQ1sILioi0vg/+NuBM4BCAtXYb0ONET7LW1ltrRwMZwARjzIgm2jxhrR1nrR2XnJzc+srdJiwWvv4HuOEdZ/nFWTDv20fOgg0MMNw9bRCzr5tAQVk10/+ynDfWH3+GrIhIg9YGfLW1tqbhgTEmiJPobrHWlgBLgAtOrrxuqNcE+O4yOPfnsO1deGyCM6eNxwPAlEHJLLxzEoNTo7n9+U/539c2Ul2nLhsROV5rA36pMeanQLgxZhrwEvB6S08wxiQbY+K8y+HANODzthTbbQQGw+R74NaPIeM0Z06bp86D/ZsASIsNZ953J3LDpL7M/ng3V/xjBXuKK/xctIh0Nq0dRRMA3ACcBxjgbWvtkyd4zkhgNhCI84tkvrX2wZae0+1G0bSGtbB+Prz9E6gqhTPuhLN+5Ay3BBZt3Mu9L62n1uPhznMHcuOkfoQEaYohke6ipVE0rQ34B621P2/0OBCYY629qv3KVMC3qKIY3vkZZM+F+L5w0aPQfyoA+SWVPPj6ZyzatI8BPaL45YwRTOyf6OeCRaQjtMeZrL2MMT/xvlgI8ArQ9BUtxDciEuCSv8E1r4MJgGcvgX/fDOWFpMeF8/jVp/H0teOprqtn1pMr+MG8bAoOV/u7ahHxo9YewRtgLrABmAq8Za19tL2L0RF8K9VWwQd/gOWPQmgUnPcQjL4KjKGqtp6/LdnO40tzCA0O4N7zB3PV6X0IDNDJUSJudMpdNMaYsY0eBuOMg/8Q+BeAtbbFk51OlgL+JB34HF6/C/asgMzJzpmwif0ByCko4+evbWL59kKyesby0CUjGNUrzs8Fi0h7a0vAL2nhda219py2FteYAv4UeDyw9hl49wGoq3K+gD3jTggKwVrLG+v38ss3PqOgrJqrTu/NvecNITYi2N9Vi0g7afOXrB1FAd8Gh/fBWz9y5pzvMQwu/jP0Gu9sqqrl0Xe38cxHO4mPCOGnFw7lsrE9NaeNiAu0xwU/Yo0xjzRMKWCM+YMxJrZ9y5Q2iU6FK+bAlS84wyn/NQ3evBeqDxMdFszPLx7G63dMok9iBPe8tI6ZT6xg6/7DJ35dEemyWjuK5ingMHCF93YIeNpXRUkbDLnQuej3hJudM2AbzWszPD2Wl793Bv93WRZb9x/mwj99wK/f2kx5dZ2fixYRX2jtKJps75wyLa5rK3XRtLPc1bDgTjiwCYZOhwt/5xzpA8XlNfzmrc+Zt3oP6bFh/OiCIVw0Mo2gQJ0kJdKVtMc4+EpjzKRGL3gmUNkexYkPZYyD7y515rXZ+rYzr83qp8DjISEyhN98cySv3DKR2IgQvj8vm3P+sJS5K3dTVau5bUTcoLVH8KOAOUBDv/tB4Bpr7fr2LEZH8D5UtAPe+D7sXAa9JzpDKpMHA+DxWN7dvJ+/LdnOutxSkqNDuWlyX751eh+iQoP8XLiItKQ9piroa63daYyJAbDWHmpY156FKuB9zFrIfh7euR9qyp0JzSb9AIJCvZstH+0o4m/vb+fD7UXEhgdzzcQ+XHtmXxIiQ/xcvIg0pT0Cfq21duwx69ZYa09r7jmnQgHfQcoKnMnLNrwESYOcIZV9Jh7VJHtPCX9bsp13PttPeHAgsyb05qYpfUmLDfdT0SLSlLac6DQEGA78Fri30aYY4F5r7fD2LFQB38G2/RcW/gBKvoDhl8L4G6HPmdBofPy2/Yf5+9IdvJadT4CBy8Zk8L2z+9M3KdKPhYtIg7YE/AzgEmA6sKDRpsPAi9baj9qzUAW8H9SUw7Lfw+p/OePnkwY7lw4cdaVzzVivPcUVPPlBDvNW7aG23sPXstK49ez+DE/X6RAi/tSWgJ8FvAMMstZ+7KP6jlDA+1FNBWx61Qn6vDUQFA5Z34BxN0DPL3vnCg5X8/SHO3n2490crq7j7MHJ3Hr2ACb0TfBj8SLdV1sC/sfA+TgTjb0HvAV8Yn00v4ECvpPIz3aGU254CWorIH2Mc1Q/4psQEgHAoapanv14N08t30lReQ3jM+O5beoAzhqUrCkQRDpQe3zJGg18FeeaqhOAzcAinCs77W+vQhXwnUxVKayb5xzVF3wOobEwepYT9t4hlpU19cxfvYd/LN1BfmkVWT1juf2cAUwbmkKApigW8bl2n2zMGDMM+BpwnrX2/DbWd4QCvpOyFr74GFb9y5nMzFMLfSbB+OthyMUQFEJNnYdXP83lb+/vYHdRBYNSorht6gAuGpmuuehFfKgtXTTfttY+510+01r7YaNtt1trH2vPQhXwXUBZAWQ/B6ufhpLdEJkMY66G066B+Ezq6j0s3LCXxxZvZ9uBMjITI7j17AFcMqanrhUr4gNtCfgj49+PHQvf1Nj4tlLAdyEeD+xY7HTfbF0E1uMMsRz9LRg2A09wFO98tp/HlmxjY94h0mPD+N7Z/bliXC/CggP9Xb2Ia7Ql4D+11o45drmpx+1BAd9FlebCuheds2SLd0BwBAy9GEZ/C5s5maXbinhs8XZW7z54ZBqEq07vQ6SmQRBpMx3BS8ewFnJXQfZc2PgqVJdCTAaMuhI7ahYrD8Xz2OLtLN9eSFxEMNef2ZdrzsgkNlxXmBI5VW0J+ApgO2CA/t5lvI/7WWvb9XRGBbyL1FbCljedo/odi50unIwJMPpbrI+dyp8/KuC/mw8QHRrE1RP7cM0ZmaTEhPm7apEupy0BPxBIAfYcs6kXsM9au/34Z506BbxLHdoL6+fBuhec4ZaBoTD0Inb3msHvtqWzcNMBrIVRveKYNrQH04alMiglSuPpRVqhLQH/BvATa+2GY9ZnAb+y1l7cnoUq4F3OWsj/1Dmq3/gyVB6EqFRKBl7GInMmL3wRx7rcUgB6JYTz1aEpTBuWwvjMBIJ1IRKRJrUl4FdZa8c3s22DtTarnWoEFPDdSl21cxGS7Odh2ztg6yG2NxV9p/Fx8Om8eKA3S3eUUFPnISYsiKlDejBtWApnDUomOkx99iIN2hLw26y1A5vZtt1aO6CdagQU8N1WWQFsfQu2vAU7lkBdJYTGUtfvXDbGTOLl0iG8ua2C4vIaggMNX+mXyLRhKZw7NIWecZq+WLq3tgT8C8Bia+2Tx6y/EZhmrZ3ZnoUq4IWaCsh5H7YshC2LoKIQAoKwfSbxRY+zeaNqNK/sMOQUlgMwLC2GacNSmDIoieHpsRpjL91OWwI+BXgVqAHWeFePA0KAS621+9qzUAW8HMVT71w4fMtC5+i+cKuzPjWL4l7TWGJP44Uv4lmzpwRrISQwgBE9YxjbO57T+sQztk+8RuaI67XHZGNTgRHeh5ustYvbsb4jFPDSosLtztDLLW/CnpXO0MuYDKr6ncdnEafxYUVvlu4NYn1eKTV1HgB6xoVzWh9v4PeOZ0hatL6wFVdp98nGfEUBL61WXuh8SbvlTWecfW2Fsz46nfr0MeyPGka2px/vlfZkeW4t+w9VAxAeHMioXrFHjvLH9I7X9WalS/NLwBtjegFzcMbRW+AJa+2fWnqOAl5OSW0V7F0H+Wudi5XkrXWmTPCyiQOoTB7FjpDBrKzqw6KiZD7dW029x/ns90uKZEzveMb2ieO0PvEM7BGtGTCly/BXwKcBadbatd755NcAl1hrP2vuOQp4aTeVB50x93lrnVv+Wji819kWEIQneRgFMcP5LGAAS8t68ebeGA5UON06UaFBjO4Vx9jecYzpE8/YXvHERmhopnROnaKLxhjzGvCYtfbd5too4MWnDuV/GfZ5ayDvU2e+HMBiqI9MpTQkhTybyJbKODaWRZNrE8m3SYQk9mZwnwzG9klgbJ94BiRH6YIm0in4PeCNMZnAMmCEtfbQMdtuBm4G6N2792m7d+/2eT0igDPlcXGOE/hF251ZMUu+cO4P5UF9zVHNywkjz+ME/oHAZIjtRXSPTNL6DCSj7xASUjMJCNQwTelYfg14Y0wUsBR42Fr775ba6gheOg2PB8oLnLAv3QOludjSPVQU7KamaDchZflE1pcc9ZRKG0JuQDqFIRkcisykJrYvJrE/4WmDSUxOIy0unKSoUPXvS7tqKeB9OiG3MSYYeAWYe6JwF+lUAgIgOsW5ZZwGOFOoRnpvANRUUFawi107tlC2bxuBxTlElO2iT+VOehR/TFBxPex0mpbYSHbaND6yaRSE9uJwZB9qYvsRmNSPxIRE+iREMCQtmp5x4ZpkTdqNL79kNcBsoNha+/3WPEdH8OIa9bXYg7spy/+c8vwt1BVsJ7BkB5GHdxFTc/R16vfZeDZ5MvnAk8Xa4DGEpgxmSFosQ9KiGZIaw+DUaKJ0cRRphr9G0UwCPgA2AB7v6p9aa99s7jkKeOkWaiqcvv+i7diiHdTs34Ld8wlhh5zD/cKAJJbVZ7GkdjjLPSM4SAy9EyIYkhrNkLQYhnrv+yRE6Ite8f+XrK2lgJdu7eBuyFkCOxZjc5ZiqkqwGAqihvBp8BjeqRrKwpLeVFlnyGZ4cCCDUqMZmhrNgB5R9IwLJyM+gp7x4cRHBKurp5tQwIt0NZ56yM92ztLNWeJMzeCpwwZHUJZyOttjJrCCkXxQksjmfYc5WFF71NMjQgK9gR9Oz3hv8Dd6nBwVql8ALqGAF+nqqg/Drg+dwN+xGIq2Oeuj06DfVCrSv8Le8IHkmAz2lNaTV1JJ7sEK730lJcf8AggNCqBnXEP4hzOwRzQT+iYwNC1Go3y6GAW8iNuU7PF25yxx7isPOusDQ6DHUEgdCVrm/aoAAA2USURBVGmjnPuU4ZQRRt7BSvJKKsg9WEneQSf4c0sqyS2uoKjcGfMfExbEhL4JfKVfIqf3TWRYugK/s1PAi7iZx+PMvbN3HexbD3vXO/cVRd4GBhIHQNpIb/CPhNRREJl45CX2llayMqeYFTlFrNxZzE7vfPvRYUGMz0zgK/0SOL1vIsPTYwjSbJydigJepLux1pmaoSHwG8K/dM+XbWJ6fhn4SYMgMgkikiAymf31kazYVcqKnGJW7iwip8AJ/KjQIMZlxnuP8BMY0TNW0y/7mQJeRBwVxUcf5e9d7/TnW8/xbcPjITIZIpKoCk1gX10kORURbCoN5vNDYRQRQ0VQHL169SEjPZ3w0BAiQgKJCAny3gcSHhJEZEgg4d71jZfV9dM+FPAi0ryacijNc6ZmqCh07ssLvTfvcsP6imKc2b+PVm8NB4mm2EZTZGMpIoYi73Ix0RTaWGcbMRTZGEqJJDjICfyIkCDiIoKPDPP8cuSP8zg2XDN5tsRvUxWISBcQEgnJg5zbiXjqnZAvL2j0C6GQwPICEsoLiTt8gMzyQkz5AQIqNxPona3zuJchkIrgWMoC4zkcEEtRVQz5udF8sT2Sz+ujWW5jKbIxFNpYqkMTSYqPPRL+GfHhR/0yiGvtmP+aCud7iYpCKC9qtFzojFLy1IKnDurrnOX6WufnPbJc591e621b/+VySJTT1ZU22vlyO2UEhESc5D9E+1PAi0jrBQRCVLJzO3aT93aU+lonSBv/VVBRSEB5AVHlBUSVF5FaXsDA8t1QVwgBh5t4EagsjeDgoVj258RwwOP8RbCVWAptDIcD44kID6NHwGESAw6TaA4TxyFiPYeI8RwiylNCZF0JIZ6qJn8kawKpD4nCBIYQEBiECQyGgGAIDIaAIOfWeF1w+PHbK4ud6wZ/+pzzoibA+V4jbdSXt9QsCItt09t/shTwIuI7gcEQnercWqOmwjmqLvP+hVB+AMoLCC8rILz8AGnlBXgOH8CW7SSwqhjT0F1U/eVLVBFKaUAsB4kh10ZTaHtQ4ImmsD6KImK83UgxHCSaIhvNISKh8su/AKJDg0iMCiExKpTESOc+KSrkyHJiVAhJ3m1xESFffpfQ8MX23nVf3nZ+AOvnfVlcQr9jQv/o0UztTX3wItI11dc5R85lB5xukogkiEhstmuk3mOpqq2nsraeypp6qmrrqahxHh+qrKW4vIai8hoKy6opKquhqNy5Lyyrobi8Gk8TURlgID4ihNjwYEKDAwkLDiA0KIDQoIblQBIpIbN2B72rt9KzchspFVuIrco78hoV4WkcThxFyg0vwimcXaw+eBFxn8AgiOrh3FrTPMAQGRpE5CnMzOnxWEoqaykqq6awUfgXlVVTVF5DSWUt1bUequvqqa7zUFJRQ3Wdh6pa53F1XR+qajOoqj0Lj4VYyhgWsJsRZicj6ncRX1VIig+mjlDAi4icQECAISEyhITIEAamtO21aus9TujX1lPlvff4qCdFAS8i0oGCAwMIDgzokDn+dQqaiIhLKeBFRFxKAS8i4lIKeBERl1LAi4i4lAJeRMSlFPAiIi6lgBcRcSkFvIiISyngRURcSgEvIuJSCngREZdSwIuIuJQCXkTEpRTwIiIupYAXEXEpBbyIiEv5LOCNMU8ZYw4YYzb6ah8iItI8Xx7BPwNc4MPXFxGRFvgs4K21y4BiX72+iIi0TH3wIiIu5feAN8bcbIxZbYxZXVBQ4O9yRERcw+8Bb619wlo7zlo7Ljk52d/liIi4ht8DXkREfMOXwyRfAD4GBhtjco0xN/hqXyIicrwgX72wtXaWr15bREROTF00IiIupYAXEXEpBbyIiEsp4EVEXEoBLyLiUgp4ERGXUsCLiLiUAl5ExKUU8CIiLqWAFxFxKQW8iIhLKeBFRFxKAS8i4lIKeBERl1LAi4i4lAJeRMSlFPAiIi6lgBcRcSkFvIiISyngRURcSgEvIuJSCngREZdSwIuIuJQCXkTEpRTwIiIupYAXEXEpBbyIiEsp4EVEXEoBLyLiUgp4ERGXUsCLiLiUAl5ExKV8GvDGmAuMMVuMMduNMff5cl8iInI0nwW8MSYQ+CvwNWAYMMsYM8xX+xMRkaP58gh+ArDdWptjra0BXgRm+HB/IiLSSJAPX7snsKfR41zg9GMbGWNuBm72Piwzxmw5xf0lAYWn+NyOoPraRvW1jeprm85cX5/mNvgy4FvFWvsE8ERbX8cYs9paO64dSvIJ1dc2qq9tVF/bdPb6muPLLpo8oFejxxnedSIi0gF8GfCrgIHGmL7GmBDgSmCBD/cnIiKN+KyLxlpbZ4y5HXgbCASestZu8tX+aIduHh9TfW2j+tpG9bVNZ6+vScZa6+8aRETEB3Qmq4iISyngRURcqssF/ImmPzDGhBpj5nm3rzTGZHZgbb2MMUuMMZ8ZYzYZY+5qos3ZxphSY0y29/bzjqrPu/9dxpgN3n2vbmK7Mcb82fv+rTfGjO3A2gY3el+yjTGHjDHfP6ZNh75/xpinjDEHjDEbG61LMMa8a4zZ5r2Pb+a513jbbDPGXNOB9f3OGPO599/vVWNMXDPPbfGz4MP6HjDG5DX6N7ywmef6fKqTZuqb16i2XcaY7Gae6/P3r82stV3mhvNl7Q6gHxACrAOGHdPmVuBx7/KVwLwOrC8NGOtdjga2NlHf2cAbfnwPdwFJLWy/EHgLMMBXgJV+/LfeB/Tx5/sHTAHGAhsbrfstcJ93+T7gN008LwHI8d7He5fjO6i+84Ag7/JvmqqvNZ8FH9b3APDDVvz7t/h/3Vf1HbP9D8DP/fX+tfXW1Y7gWzP9wQxgtnf5ZeBcY4zpiOKstXuttWu9y4eBzThn9HYlM4A51rECiDPGpPmhjnOBHdba3X7Y9xHW2mVA8TGrG3/GZgOXNPHU84F3rbXF1tqDwLvABR1Rn7X2HWttnffhCpxzUPyimfevNTpkqpOW6vPmxhXAC+29347S1QK+qekPjg3QI228H/JSILFDqmvE2zU0BljZxOaJxph1xpi3jDHDO7QwsMA7xpg13mkijtWa97gjXEnz/7H8+f4BpFhr93qX9wEpTbTpLO/j9Th/kTXlRJ8FX7rd24X0VDNdXJ3h/ZsM7LfWbmtmuz/fv1bpagHfJRhjooBXgO9baw8ds3ktTrfDKOAvwH86uLxJ1tqxOLN83maMmdLB+z8h74lx04GXmtjs7/fvKNb5W71TjjU2xtwP1AFzm2nir8/C34H+wGhgL043SGc0i5aP3jv9/6WuFvCtmf7gSBtjTBAQCxR1SHXOPoNxwn2utfbfx2631h6y1pZ5l98Ego0xSR1Vn7U2z3t/AHgV50/hxjrDFBNfA9Zaa/cfu8Hf75/X/oZuK+/9gSba+PV9NMZcC1wEXOX9JXScVnwWfMJau99aW2+t9QBPNrNff79/QcBlwLzm2vjr/TsZXS3gWzP9wQKgYcTCN4HFzX3A25u3z+5fwGZr7SPNtElt+E7AGDMB59+gQ34BGWMijTHRDcs4X8ZtPKbZAuA73tE0XwFKG3VHdJRmj5z8+f410vgzdg3wWhNt3gbOM8bEe7sgzvOu8zljzAXAj4Dp1tqKZtq05rPgq/oaf6dzaTP79fdUJ18FPrfW5ja10Z/v30nx97e8J3vDGeWxFecb9vu96x7E+TADhOH8ab8d+ATo14G1TcL5c309kO29XQh8D/iet83twCacUQErgDM6sL5+3v2u89bQ8P41rs/gXKhlB7ABGNfB/76ROIEd22id394/nF80e4FanH7gG3C+03kP2Ab8F0jwth0H/LPRc6/3fg63A9d1YH3bcfqvGz6DDaPK0oE3W/osdFB9z3o/W+txQjvt2Pq8j4/7v94R9XnXP9PwmWvUtsPfv7beNFWBiIhLdbUuGhERaSUFvIiISyngRURcSgEvIuJSCngREZdSwIsrGWPqj5mZst1mIzTGZDaefbAV7SONMf/1Li/3nkQj4nP6oIlbVVprR/u7CK+JwMfeE57K7ZcTgYn4lI7gpVvxzuH9W+883p8YYwZ412caYxZ7J8B6zxjT27s+xTun+jrv7QzvSwUaY540zrz/7xhjwpvYV3/vXOLPAd8C1gCjvH9R9OigH1m6MQW8uFX4MV00MxttK7XWZgGPAX/0rvsLMNtaOxJncq4/e9f/GVhqncnNxuKctQgwEPirtXY4UAJ849gCrLU7vH9FrMGZp2Q2zpmSo60zf4mIT+lMVnElY0yZtTaqifW7gHOstTneieH2WWsTjTGFOKfM13rX77XWJhljCoAMa211o9fIxJnrfaD38Y+BYGvtQ83UsspaO94Y8wpwl21mfhOR9qYjeOmObDPLJ6O60XI9TXyfZYx53Ptl7EBvV80FwBvGmB+c4j5FTooCXrqjmY3uP/Yuf4QzYyHAVcAH3uX3gFsAjDGBxpjY1u7EWvs94BfAL3Gu+rTQ2z3zaNvKF2kdjaIRtwo3R18seZG1tmGoZLwxZj3OUfgs77o7gKeNMfcCBcB13vV3AU8YY27AOVK/BWf2wdY6C5iDc3Wgpaf0k4icIvXBS7fi7YMfZ60t9HctIr6mLhoREZfSEbyIiEvpCF5ExKUU8CIiLqWAFxFxKQW8iIhLKeBFRFzq/wGolVxvpIjsbAAAAABJRU5ErkJggg==\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["plt.plot(history.history['masked_acc'], label='accuracy')\n","plt.plot(history.history['val_masked_acc'], label='val_accuracy')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":299},"executionInfo":{"status":"ok","timestamp":1669569514566,"user_tz":-60,"elapsed":16,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"7404deec-0d5f-479a-bec6-5929071760fd","id":"7Juf7oSxgA8I"},"execution_count":183,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb444723810>"]},"metadata":{},"execution_count":183},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEJCAYAAACZjSCSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhU1fnA8e+bSSAr2dkSwr7vEBBrRcSluIEbAlrrCmq11fpTa7UqtbS1LrXaWhV3xRYRxFJEURR3QMK+72ASIASyQ/Z5f3/MgDEkJEDuTMi8n+eZJ3c5c+fNzeS+M+ece46oKsYYYwJXkL8DMMYY41+WCIwxJsBZIjDGmABnicAYYwKcJQJjjAlwlgiMMSbAOZoIRGSUiGwSka0icn8N+58WkZXex2YRyXMyHmOMMUcTp+4jEBEXsBk4D8gAlgITVHV9LeV/BQxU1RsdCcgYY0yNgh089lBgq6puBxCR6cAYoMZEAEwAHqnroAkJCdqhQ4eGitEYYwLCsmXL9qtqYk37nEwESUB6lfUM4LSaCopIe6Aj8FldB+3QoQNpaWkNEqAxxgQKEdlV277G0lg8HpipqpU17RSRSSKSJiJp2dnZPg7NGGOaNicTQSbQrsp6sndbTcYD/6ntQKo6VVVTVTU1MbHGbzbGGGNOkJOJYCnQVUQ6ikgzPBf7OdULiUgPIBZY5GAsxhhjauFYG4GqVojIHcB8wAW8qqrrRORRIE1VDyeF8cB0PYnuS+Xl5WRkZFBSUnLygZuTFhoaSnJyMiEhIf4OxRhTD451H3VKamqqVm8s3rFjB1FRUcTHxyMiforMAKgqBw4coLCwkI4dO/o7HGOaDC3MgmbhSPOoE3q+iCxT1dSa9jnZa8hnSkpK6NChgyWBRkBEiI+Pxxr1TSArKCknI6eYjNxDZOQWk5FbzO68YorLK6lwuymvVCoq3VS61bPsdlNRqZR7f1a4PfuDKks5U5cyhi84U1aT1vchhl15d4PH2yQSAWBJoBGxv4Vp6mq60P+wfIiCkooflQ9v5qJtTBgRzVwEu4IIDhLCmwXjChJCXEJwUBDBLiHEFUSwQOfStQzO+5i+eZ8SWllEYbNElif+nOieIx35fZpMIjDGmNqUVbjJyM7hYIWLkkqltNxNaUUlJd6fpRVuSso9P0vL3ZRUVB5Vpriskj35JTVe6MNCXLSLCyM5NpzUDrEkx3qWD/+MDQ+p+wNS7k5Y9Q6s+g/k7oCQcOh9CfSfQFTH4QwNcjl2fiwRGGNOfZXlULiHwqwd7M/cRmHWLipyv8dVmElkaRYJldl0koO4VSgijHyNoIBwCrw/KzWcSiI45F0/KBEcckVR4oqkLLgFZcGRlDWLplV0DIPbV7/QhxEX0ezEvgmXFMD6/3ou/ru+8WzrcCYMvxd6jYYTbA84XpYITjEVFRUEB9ufzQQIVSgrgkM5cOgAFOzGnZdO0b5dFO/fBfnphBbvJap8P0EoUcDhS2eeRnDA1ZKDYW0pjEolJKYtYUGVhFYWElZRSHR5IcFlBQSXF+Aq3U1QaT5SfvDHr+8GyryPQ0BxCyhOhoPJUJAEecmQ0w6ikz2PFm3BVUdvOXclbF8Iq6bDhrlQUQxxnWHk76HfOIhJaeizWCe7ojSgSy+9lPT0dEpKSrjzzjuZNGkSH330EQ888ACVlZUkJCTw6aefUlRUxK9+9SvS0tIQER555BGuuOIKIiMjKSoqAmDmzJnMnTuX119/neuvv57Q0FBWrFjBGWecwfjx47nzzjspKSkhLCyM1157je7du1NZWclvf/tbPvroI4KCgpg4cSK9e/fm2Wef5f333wfgk08+4V//+hezZ8/256kygaiiDIpzPBf1Iz9z0UM5lBXtp7zwAO6DOWhxDkHFuQSX5tKsPB+X/rgaJghoriEc0Dh2awIHgvtQFtEaiWlHeEJ74tp2om1KV9q2SiAm6Dg/pVeWez6ll+RBSX6VRx4U50LBHsjPgPx0yFzmSU4/IhDV5ofEEJ0E0d5EERoDmz+C1TOgaK9nfcDV0H8CJKeCH9vWmlwi+MP/1rF+d0GDHrNX2xY8cknvOsu9+uqrxMXFUVxczJAhQxgzZgwTJ07kyy+/pGPHjuTk5ADwxz/+kejoaNasWQNAbm5uncfOyMjg22+/xeVyUVBQwFdffUVwcDALFizggQceYNasWUydOpWdO3eycuVKgoODycnJITY2ll/+8pdkZ2eTmJjIa6+9xo032gCvxkGqUJQFe9fA3tUUp6+iYvdqIop2EYT7qOICoMEcJJJcjSKPSHI1ljxtRx6RFEgUpSHRVDSLITg2mRatO9KmTTKdW0bRMzGSuIhmDRe7KwQi4j2P+ig7BAWZnsSQnwH5mT8kij2rYOMHUFn6Q/mgYOh6PvQfD91GQXDzhov9JDS5ROBPzz777JFP2unp6UydOpXhw4cf6U8fFxcHwIIFC5g+ffqR58XGxtZ57LFjx+JyeRqL8vPzue6669iyZQsiQnl5+ZHj3nrrrUeqjg6/3rXXXsu0adO44YYbWLRoEW+++WYD/cYm4FWWw/4tkLUW9q5G967FvWc1ruIfPinvdyeyQVPYGjSIkuYJVDSPRcNiISyOoIhYmkUmEh4RRXR4M1qEhRATHkKHsBCivcthIa7G2xOtWTgkdPU8aqLq+daQnw6FWZA0GCIb3zA5TS4R1OeTuxM+//xzFixYwKJFiwgPD2fEiBEMGDCAjRs31vsYVd/s1e+SjoiIOLL80EMPcfbZZzN79mx27tzJiBEjjnncG264gUsuuYTQ0FDGjh1rbQzmxBTnQda6Ixd99q5F921AvJ94yyWErdqO1RV92aApZDTvQov2A+jTOYWhHeM4p00LXMdbVXOqE4GIBM+jEbMrQgPJz88nNjaW8PBwNm7cyOLFiykpKeHLL79kx44dR6qG4uLiOO+883juuef4+9//DniqhmJjY2nVqhUbNmyge/fuzJ49m6iomnsM5Ofnk5SUBMDrr79+ZPt5553Hiy++yNlnn32kaiguLo62bdvStm1bpkyZwoIFCxw/F+YUVHYICnZDgbd6o8BbxVGw27ucCaX5R4oXh8Sw3dWJ7yrPZ2VZOzZoe0padGJwp5YM6RDHzzvG0TkxovF+kjc/YomggYwaNYoXXniBnj170r17d4YNG0ZiYiJTp07l8ssvx+1207JlSz755BN+//vfc/vtt9OnTx9cLhePPPIIl19+OY899hgXX3wxiYmJpKamHmk4ru6+++7juuuuY8qUKVx00UVHtt98881s3ryZfv36ERISwsSJE7njjjsAuOaaa8jOzqZnz54+OR+mkXFXQsZST1/1wxf2Iz8zPA2h1Z8SnkBxaGtygluxu0UvtpRG80VuAqvKU9hXEkOnxEhO6xbHiI5x3NshjuTYcN//XqZBNImxhjZs2GAXuDrccccdDBw4kJtuusknr2d/k0aiJB9WvA3fvehJAoeFxkB0MtoiieLw1mRLAumVsWw6FM3qwgiWHghld5WelOHNXHRtGcnAlFhO6xhHaoc4EqMaR0OnqZ8mP9aQObbBgwcTERHBU0895e9QjK/s3wJLXoSV/4byg2i7YeQNu4+tri6sPxjFxgMVbMkqYsvWIvKLy488LSo0mK4tIzmzZxRdW0XSpWUkXVtF0aZFKEGBVr8fQCwRBIBly5b5OwTjC243bPuU8m//RciOz6iUEFbFnMs7QRfwYXprCrZUAPuB/cSEh9CtZRQX9WtD15aRdG3pufC3jGpu9foByBKBMaeo4rJKtuwrZFvGXsLXv0O/3e/QpiKTHI1hWsWV/KfyHEo1nh6to7ikfxTdW0cdueDHn+iQCKZJskRgTCPmdiv7D5ayO6+E9JxDbMkqZFNWIZv2FkLOdn7h+pixri+IkmI2BndnQdJDlHS5mMFt47i6dRStW4TaBd/UyRKBMX5UVFrBnrxiMvOK2Z1Xwu68Ynbne8au351Xwt78Esoqf7gbN0iUK2O28A/5iD7Nl6ASzMGul1D501/SI2UIPfz4u5hTlyUCYxxWUelmdWY+S3fkHJmgxHPhLz5qOGNXkNC6RShtokMZkBxNx55RdAotJDmkgKTynbTcNI2g/ZshIhHOug9JvZGoqNZ++s1MU2GJwBgHpOcc4sst2Xy9ZT/fbN1/5IIfHRZCUnRzerUoZXSrg7RvXkRbVz6JkkdMZQ7hpfsJOpjlGatne5ZnZMqq2gyAy16E3pc1mnFqzKnPEoEfVB1l1DQNBSXlLNp2gK+8F/+cA9l0lD0MijjA3xLz6Nl8H63KdxN8aB/k74O8yqMP0jwaolpBZCtIHuL5GdkKolp7frZIgvjOfh2l0jRNlggCmM1tcOIqKt2s3pnFurUr2Lt9LRzYSgf2cLkri/tcWbQIzfMWBPYHQUx7iOsEbftCZGvvxb2ld9l7wQ8J8+vvZAJX07sKfHi/Z/jbhtS6L1zwWK2777//ftq1a8ftt98OwOTJkwkODmbhwoXk5uZSXl7OlClTGDNmTJ0vVVRUxJgxY2p83ptvvsmTTz6JiNCvXz/eeustsrKyuPXWW9m+fTsAzz//PG3btuXiiy9m7dq1ADz55JMUFRUxefLkI4Phff3110yYMIFu3boxZcoUysrKiI+P5+2336ZVq1Y1zpmQn5/P6tWrj4yR9NJLL7F+/Xqefvrpkzq9jZoqFO1D960nd+dqsnetozJ7CzHF3zOI/Qw6XC4YysJaEpzYlaCEYRDf5YdHbAcIbsChko1pYI4mAhEZBTwDuICXVfWoq6mIXAVMBhRYpapXOxmTE8aNG8ddd911JBHMmDGD+fPn8+tf/5oWLVqwf/9+hg0bxujRo+vsyhcaGsrs2bOPet769euZMmUK3377LQkJCUfmNvj1r3/NWWedxezZs6msrKSoqKjO+Q3Kyso4PExHbm4uixcvRkR4+eWXefzxx3nqqadqnDMhJCSEP/3pTzzxxBOEhITw2muv8eKLL57s6Ws8ivMgeyPsW8+hjDWUZK4jLHcjYRX5CBAHBGsYGUFJZMUMpLBND5K79CWiTXeI70wzH00raExDcywRiIgLeA44D8gAlorIHFVdX6VMV+B3wBmqmisiLU/6hY/xyd0pAwcOZN++fezevZvs7GxiY2Np3bo1v/nNb/jyyy8JCgoiMzOTrKwsWrc+dg8PVeWBBx446nmfffYZY8eOJSHBM5zt4bkGPvvssyPzC7hcLqKjo+tMBOPGjTuynJGRwbhx49izZw9lZWVH5k6obc6EkSNHMnfuXHr27El5eTl9+/Y9zrPVCJQXw/7NkLUe9q2nbM863FnrCT2050gRt4ayU5PZrIPIjeyCq1Uv4jr2Z1DPbvRMjLS++aZJcfIbwVBgq6puBxCR6cAYYH2VMhOB51Q1F0BV9zkYj6PGjh3LzJkz2bt3L+PGjePtt98mOzubZcuWERISQocOHY6aY6AmJ/q8qoKDg3G7f+h7fqy5DX71q19x9913M3r0aD7//HMmT558zGPffPPN/PnPf6ZHjx7ccMMNxxWX35QXe2aK2jCHyr1rCcrdgajn/JQRzDZ3WzZqJza7zyK/RRdC2/YhuUM3+qfEMqZNNGHNXH7+BYxxlpOJIAlIr7KeAZxWrUw3ABH5Bk/10WRV/cjBmBwzbtw4Jk6cyP79+/niiy+YMWMGLVu2JCQkhIULF7Jr1656HSc/P7/G540cOZLLLruMu+++m/j4+CNzDZxzzjk8//zz3HXXXUeqhlq1asW+ffs4cOAAkZGRzJ07l1GjRtX6eofnNnjjjTeObK9tzoTTTjuN9PR0li9fzurVq0/mlDlLFTKXw8ppuFfPJKisgGyJZ1lFJzZpPza521HYoisx7XrQJzmefskxnJvUgqjQOiYeN6YJ8ndjcTDQFRgBJANfikhfVc2rWkhEJgGTAFJSUnwdY7307t2bwsJCkpKSaNOmDddccw2XXHIJffv2JTU1lR496nfPZ23P6927Nw8++CBnnXUWLpeLgQMH8vrrr/PMM88wadIkXnnlFVwuF88//zynn346Dz/8MEOHDiUpKemYrz158mTGjh1LbGwsI0eOZMeOHQC1zpkAcNVVV7Fy5cp6TbHpc0X7YNV03CumEbR/E2U044PKIcyqPIugTsM5rXMiA5Oi+UVSNLENOdetMacwx+YjEJHT8XzC/5l3/XcAqvqXKmVeAJao6mve9U+B+1V1aW3HtfkI/O/iiy/mN7/5Deecc06tZXz6N6kogy3z0RXTYMsniFayUrvyTsVw1sScw4VDunP5wGRaR4f6Jh5jGiF/zUewFOgqIh2BTGA8UL1H0PvABOA1EUnAU1W03cGYzEnIy8tj6NCh9O/f/5hJwGey1sGKt6lcNR1X8QFyJJZ3yy/kg6AR9Ow3hKtS2/Hn9rHWsGtMHRxLBKpaISJ3APPx1P+/qqrrRORRIE1V53j3nS8i64FK4F5VPeBUTI3JmjVruPbaa3+0rXnz5ixZssRPEdUtJiaGzZs3+zeIQzmwdhbu5dMI2ruSCoL5pHIQMypvpDhlBFcO6cD0Pq2JaO7vWk9jTh2O/reo6jxgXrVtD1dZVuBu7+NkX+uU+uTXt29fVq5c6e8wHNHg1Y2HcmDbZ+jGueiGDwhyl7GZDkwv/wWLI0ZyXmovJg9Opn18RN3HMsYcpUl8bAoNDeXAgQPEx8efUsmgKVJVDhw4QGjoSdTHu92wZyVsXQBbPkEz0xB1UyAteK/8bN5nBCm9T2fs4GQe6pKAy6ZQNOakNIlEkJycTEZGBtnZ2f4OxeBJzMnJycf3JO+nfrZ8Ats+hYPZKEJ+bB/+1+wq3ivoSXFiP645vRNv9k8iOty6eRrTUJpEIggJCTlyR6w5RVT71E9mGqgbwmLRzuewIWIoUza15ds9QXRKjOA347txUd82NoG6MQ5oEonAnCJq+NQPAm0HwvB70S7n8s2h9jy5YCsr0/NoFxfGU2O7MWZAW4JdQf6O3pgmyxKBcV5eOnz+GKz6D2glhMVCl3Ohy3nQ5RyISOC7HTk8NW8TS3ak0SY6lL9c3pcrBycTYgnAGMdZIjDOOZQDXz0F370EKAydBH2ugKRBEOQZv2dleh5PTV/CV1v2kxjVnMmX9GL80BRCQ2x8H2N8xRKBaXhlB2Hxv+CbZ6GsCPpfDSPuh5h2R4qs253P059sZsGGfcRFNOOBC3tw7bAONsCbMX5gicA0nMpyWPY6fPE4HNwH3S+Ccx6Clj8MNbElq5CnF2xm3pq9tAgN5p7zu3H9GR2JtBvAjPEb++8zJ8/thnXvwWdTIHcHpPwExk2DlB8Gm9114CB/X7CF91dmEh7i4tcju3DTmZ2IDrNuoMb4myUCc+JUPb2AFkyGvauhVR+4+l3oet6RCdb35pfw7GdbmLE0nWCXMGl4J24Z3pk4G/nTmEbDEoE5MRnLYMEjsPMriEmBy6ZC37EQ5Onlk3uwjOe/2MYb3+7ErcrVp6Vwx9ldaNnCRgA1prGxRGCOz/4t8OmjsGEOhCfABY/D4OshuDkARaUVvPr1Dl76cjtFZRVcNiCJu87tRkp8uH/jNsbUyhKBqZ+CPfD5X2DFNAgJgxG/g9NvB++E7SXllby95Hv+tXArBw6WcX6vVtzzs+50a2UTuhvT2FkiMMdWWgTf/gO+fdbTK2joRDjzHohMBKCi0s2s5Rk8s2ALu/NL+EnneO79WXcGpjTC2cuMMTWyRGBq5q70fPpf+CcoyoLel8M5D0OcZ0wnt1v5cO1envpkE9uzD9K/XQxPjO3PGV0S/By4MeZ4WSIwR9uyAD55CPath3anwbi3od0QwDPM9Bebs3li/ibW7S6ga8tIXrx2MOf3amVDgBtzirJEYH6wd60nAWz7DGI7wlVvQs/RR7qCpu3M4fH5m/huRw7JsWE8NbY/lw5MsvkAjDnFWSIwnobghVNgxdsQFgOjHoPUmyDY09f/+wOH+PO8DXy0bi8Jkc15dExvxg9JoVmwDQhnTFNgiSCQlRZ5GoG//Qe4Kzy9gIbf4xkdFE9X0OcWbuWVr3bgChL+77xu3HRmR8Kb2dvGmKbE/qMDkbsSVrwFC/9ca0PwzOUZPDF/E9mFpVw+MIn7RvWgdbTdDGZMU2SJIJCoemYE++ThGhuCwdMO8If/rWdNZj4DU2KYeu1g6wpqTBNniSBQ5GyHuXfD9oU1NgRn5hXz2Icb+d+q3bRuEcrfxw1gdP+2NjWkMQHA0UQgIqOAZwAX8LKqPlZt//XAE0Cmd9M/VfVlJ2MKSOvnwH9v91z0qzUEHyqr4IUvtjP1y22owq9HduHWEZ2tHcCYAOLYf7uIuIDngPOADGCpiMxR1fXVir6jqnc4FUdAqyjzDAy3+F+QNBjGvu4ZIA7P/QBzVu3msQ83sie/hIv7teH+C3qQHGtjAhkTaJz82DcU2Kqq2wFEZDowBqieCIwT8tJh5g2QsRROuxXO++ORbwGr0vP4w//Wsfz7PPokteCZ8QMZ2jHOzwEbY/zFyUSQBKRXWc8ATquh3BUiMhzYDPxGVdNrKGOOx+aPYfYkqKyAsW9A70sByCoo4a8fbeS95ZkkRDbn8Sv6ceXgZGsHMCbA+bsi+H/Af1S1VERuAd4ARlYvJCKTgEkAKSkpvo3wVFJZ4Rkb6Ou/Qau+cNUbEN8ZgJnLMnj4v2upqFRuPaszt5/dmahQmx3MGONsIsgE2lVZT+aHRmEAVPVAldWXgcdrOpCqTgWmAqSmpmrDhtlEFO6FmTfBrq9h0C888wSEhFFe6WbK3PW8sWgXp3eK57Er+tI+PsLf0RpjGhEnE8FSoKuIdMSTAMYDV1ctICJtVHWPd3U0sMHBeJquHV96kkBZEVz6AgyYAEB2YSm3v72c73bmMPHMjvx2VA+CXTYshDHmxxxLBKpaISJ3APPxdB99VVXXicijQJqqzgF+LSKjgQogB7jeqXiaJLcbvnoKPv8zxHeF6+ZAy54ArEzP49a3lpFXXMYz4wcwZkCSn4M1xjRWonpq1bSkpqZqWlqav8Pwv4MH4L2JsO1T6HsVXPw0NI8EYEZaOr9/fy0to5rz4rWD6d022s/BGmP8TUSWqWpqTfv83VhsTsT3SzxdQw/uh4v/7pkzWISyCjd/nLuetxbv4owu8fxzwiBiI5r5O1pjTCNnieBUogqL/gkLJkN0Mtz0MbQdAHjaA3759jKW7sxl0vBO3Pez7tYeYIypF0sEp4pDOTDnV7BxLvS4GC79F4R6qnxWfJ/LbdOWW3uAMeaEWCJo7Nxuz5DRCyZDaQH87C8w7LYjg8XNWOptD2jRnPduO4NebVv4N15jzCnHEkFjtnslfPB/kJkGKT+Bi56EVr0BKKtw8+jcdUxb/D0/7ZLAPyYMtPYAY8wJsUTQGBXnwmd/grRXIDweLnsR+o078i1gX2EJv5y2nLRdudwyvBP3WnuAMeYkWCJoTFRh1X/g44egOAeG3AxnP+iZR9hrxfe53DptGfnF5Tw7YSCj+7f1Y8DGmKbAEkFjsXctzLsHvl8EyUPgovegTf8fFXln6fc89P46WkVbe4AxpuFYIvC3kgL4/C+w5EXPJ//R/4QB10DQD1U9brfyp3kbeOXrHZzZNYFnx1t7gDGm4Vgi8BdVWDMTPn4QivZB6g0w8iEI//G8AKUVldw9YxUfrN7D9T/pwO8v6mntAcaYBmWJwB/2bfRUA+38CtoOhAn/8cwgVk1+cTm3vJXG4u05/O6CHkwa3gkRmzvAGNOwLBH4UmkRfPFXz9SRzSI9w0MM+gUEuY4quie/mOtfXcr2/UX8fdwALh1oN4kZY5xhicBXdi2CmTdC4W4YeC2c+weIiK+x6OasQq579TsKSyp47fqh/LRrgo+DNcYEEksEvrD9c/jPBGjRFm5aAO2G1Fp0yfYDTHwzjeYhLt65ZZiNHGqMcZwlAqdtXQDTr4G4TvCL/0Jky1qLzluzh7umr6RdXBiv3zCUdnHhPgzUGBOoLBE4adNHMONaSOwO1/631qoggNe+2cGjc9czKCWWV65LJSbcuocaY3zDEoFTNvwP3r0BWveBn793VLfQw9xu5a8fbeTFL7dzfq9WPDthIKEhRzceG2OMUywROGHtLJg10dMl9OczjwwXXV1ZhZv7Zq7i/ZW7uXZYeyaP7o0ryLqHGmN8yxJBQ1s1Hd6/DVJOh6vfgeZRNRYrLCnn1mnL+GbrAe79WXd+OaKz3SNgjPELSwQNaflbnsljOp4JE6ZDs4gai2UVlHD9a0vZklXIk2P7c+XgZB8HaowxP7BE0FCWvgIf3A2dz4Hxb0NIWI3Ftu4r4rpXvyP3UBmvXD+Es7ol+jhQY4z5sXonAhH5CdCh6nNU9U0HYjr1LH4BPvotdBsFY9+AkNAai6XtzOHmN9MIDgrinUmn0zfZ7hEwxvhfvUYvE5G3gCeBnwJDvI/UejxvlIhsEpGtInL/McpdISIqInUes9H55hlPEuh5CVz1Vq1J4Kst2Vzz8hJiw5vx3m0/sSRgjGk06vuNIBXopapa3wOLiAt4DjgPyACWisgcVV1frVwUcCewpL7HbjS+eAIWToE+V3hmEXOF1Fhs095Cbpu2nI4JEfx74jDibAhpY0wjUt/xjNcCrY/z2EOBraq6XVXLgOnAmBrK/RH4K1BynMf3H1XPVJILp0C/8XDZ1FqTwL7CEm58fSnhzVy8dsMQSwLGmEanvt8IEoD1IvIdUHp4o6qOPsZzkoD0KusZwGlVC4jIIKCdqn4gIvfWdiARmQRMAkhJSalnyA5RhQWT4Zu/w8CfwyXP1jh6KEBxWSUT31xGzsEyZtxyOm2ia25ANsYYf6pvIpjc0C8sIkHA34Dr6yqrqlOBqQCpqan1rp5qcKow/wHPMNKpN8GFT/5oJrGq3G7l7hkrWZ2Rx4s/H2xtAsaYRqteiUBVvxCR9kBXVV0gIuFAXeMgZALtqqwne7cdFgX0AT733kjVGpgjIqNVNa2+v4DPuN3w4b2w9GU47TYY9Rc4xg1gj8/fxIdr9/L7i3pyfu/jrVUzxhjfqW+voYnATOBF76Yk4P06nrYU6CoiHUWkGTAemHN4p6rmq2qCqnZQ1Q7AYqBxJgHwTCiz9GU44846k/ualZsAABPUSURBVMD0777nhS+28fNhKdz0044+DNIYY45ffRuLbwfOAAoAVHULUPt4yp4yFcAdwHxgAzBDVdeJyKMicqy2hcZn1yL48nHoP8EzocwxksA3W/fz+/fXMrxbIpMv6W3DRhhjGr36thGUqmrZ4YuaiAQDddbVq+o8YF61bQ/XUnZEPWPxrZJ8eG8SxKTAhU8cMwls3VfIrdOW0TkxkueuHmiTzBtjTgn1vVJ9ISIPAGEich7wLvA/58JqRD74PyjIhCteqXUAOYD9RaXc8PpSmge7eOX6VKJCa+5OaowxjU19E8H9QDawBrgFmKeqDzoWVWOx6h1Y8y6M+B0k137Tc0l5JZPeTCO7sJRXrkslOdZmFjPGnDrq3X3UW6XzEnjuGhaRt1X1GudC87PcnZ5vAymnw5l311rM7VbueXcVy7/P4/lrBtG/XYzvYjTGmAZQ328E7UTkdwDeHkCzgC2OReVvlRWeiWUkCC6fWusNYwBPL9jM3NV7uP+CHlzQt40PgzTGmIZR30RwI9DXmwzmAl+o6mTHovK3L5+AjO/g4r95GolrMXNZBv/4bCvjh7TjluGdfBigMcY0nGNWDXmHgDjsGTz3EXyDp/F4kKoudzI4v/h+saeraL/x0PfKWost2naA3723mjO6xPPHS/tYN1FjzCmrrjaCp6qt5wK9vNsVGOlEUH5Tkg/vTfyhq2gttmUXceu0ZbSPj+Bf1wwmxLqJGmNOYcdMBKp6tq8CaRQ+uAfyM+HGjyC0RY1Fcg6WcePrSwkOEl67fgjRYdZN1BhzaqvvEBPRIvI3EUnzPp4SkaY1itrqGbBmBpz1W2g3tMYipRWV3PJWGnvyS5j6i1TaxVk3UWPMqa++dRqvAoXAVd5HAfCaU0H53OGuou2GwZn/V2MRVeX+WWtYujOXp8b2Z3D7WN/GaIwxDqnvfQSdVfWKKut/EJGVTgTkc5UVniEkwNNV1FXzKXk3LYPZKzK55/xuXNK/rQ8DNMYYZ9X3G0GxiPz08IqInAEUOxOSj331JKQvgYv+BrHtay327+++p0frKG4/u4sPgzPGGOfV9xvBrcCbVdoFcoHrnAnJh75f4hleut846De21mLbsotYmZ7Hgxf2tG6ixpgmp76JoEBV+4tICwBVLRCRU3ug/ZICeO9miG7nmWnsGGYty8AVJIwZaFVCxpimp75VQ7PAkwBUtcC7baYzIfnIPG9X0ctfqrWrKEClW5m9IpPhXRNoGRXqwwCNMcY36rqzuAfQG4gWkcur7GoBnLpXxdXvwup3PKOKppx2zKKLth1gT34JD17U00fBGWOMb9VVNdQduBiIAS6psr0QmOhUUI7K3QUf3A3tToMz76mz+KzlGbQIDebcnq18EJwxxvheXYkgHLgHmKqqi3wQj7Pq2VX0sKLSCj5au5fLBiURGlL7CKTGGHMqqysRpOCZjSxERD4FPgS+U9U6p6lslL56CtIXe9oFYjvUWXzemj0Ul1dyxaBk52Mzxhg/OWZjsar+VVVHAhcCq/AMR71cRP4tIr8QkVOnviT9O09X0b5XQb+r6vWUWcsy6JgQwaAUm2zGGNN01avXkKoWqupsVb1FVQcCU4BE4E1Ho2tI2ZsgriNcdOyuooel5xxiyY4crhiUZPcOGGOatGMmAhH5eZXlMw4vq+p6oFRVf+ZgbA1r0LVw2yIIrd9Yee8tz0QELrNqIWNME1fXN4Kqk/X+o9q+G+s6uIiMEpFNIrJVRO6vYf+tIrJGRFaKyNci0qseMZ+44Gb1KqaqvLcig9M7xZMUE+ZoSMYY4291JQKpZbmm9R/vFHEBzwEX4JnMZkINF/p/q2pfVR0APA78re6QnZe2K5ddBw5ZI7ExJiDUlQi0luWa1qsbCmxV1e2qWgZMB8b86AA/3KUMEFGPY/rErGUZhDdzMapPa3+HYowxjqur+2gPEVmN59N/Z+8y3vW6ZmtPAtKrrGcAR93GKyK346mCakYtU1+KyCRgEkBKSu2TyTeEkvJKPli9hwv6tCGieX2HYjLGmFNXXVe6/kArfnxBB2gH7G2IAFT1OeA5Ebka+D01jGqqqlOBqQCpqamOfmuYv24vhaUVXDE4ycmXMcaYRqOuqqGngXxV3VX1AeR79x1LJp6EcViyd1ttpgOX1hWw02YtzyQpJoxhHeP9HYoxxvhEXYmglaquqb7Ru61DHc9dCnQVkY4i0gwYD8ypWkBEulZZvQjYUmfEDsoqKOHrLdlcNjCJoCC7d8AYExjqqho61i21x+xXqaoVInIHMB9wAa+q6joReRRIU9U5wB0ici5QTiOY7Gb2ikzcCpcPsmohY0zgqCsRpInIRFV9qepGEbkZWFbXwVV1HjCv2raHqyzfeRyxOkpVmbUsg0EpMXRKjPR3OMYY4zN1JYK7gNkicg0/XPhT8fTwuczJwHxtTWY+W/YV8afL+vg7FGOM8aljJgJVzQJ+IiJnA4evkB+o6meOR+Zjs5Zl0Cw4iIv72XSUxpjAUq+O8qq6EFjocCx+U1bhZs6q3ZzXqxXRYSH+DscYY3yqvnMWN2mfbdxH7qFyrrQhJYwxAcgSAZ7pKBOjmnNm1wR/h2KMMT4X8Ikg52AZCzfu49IBbQl2BfzpMMYEoIC/8s1ZmUmFW7lisFULGWMCU8AnglnLM+ndtgU9WrfwdyjGGOMXAZ0INmcVsiYz3+YdMMYEtIBOBLOWZRAcJIwZYPcOGGMCV8AmgopKN7NXZDKie0viI5v7OxxjjPGbgE0EX2/dz77CUq60eQeMMQEuYBPBrOWZxISHcHaPlv4OxRhj/CogE0FBSTkfr9vL6P5taR7s8nc4xhjjVwGZCD5YvYfSCrf1FjLGGAI0EcxalkGXlpH0S472dyjGGON3AZcIdu4/SNquXC4flISITUdpjDEBlwjeW56BCFw20HoLGWMMBFgicLuVWcsz+WmXBNpEH3PKZWOMCRgBlQiW7MghM6/YGomNMaaKgEoEs5ZnENk8mJ/1bu3vUIwxptFwNBGIyCgR2SQiW0Xk/hr23y0i60VktYh8KiLtnYrlUFkFH67Zw4V9WxPWzO4dMMaYwxxLBCLiAp4DLgB6ARNEpFe1YiuAVFXtB8wEHncqno/W7uVgWaVVCxljTDVOfiMYCmxV1e2qWgZMB8ZULaCqC1X1kHd1MeDYVTo6LIQL+rRmSIc4p17CGGNOScEOHjsJSK+yngGcdozyNwEfOhXMOT1bcU7PVk4d3hhjTllOJoJ6E5GfA6nAWbXsnwRMAkhJSfFhZMYY0/Q5WTWUCbSrsp7s3fYjInIu8CAwWlVLazqQqk5V1VRVTU1MTHQkWGOMCVROJoKlQFcR6SgizYDxwJyqBURkIPAiniSwz8FYjDHG1MKxRKCqFcAdwHxgAzBDVdeJyKMiMtpb7AkgEnhXRFaKyJxaDmeMMcYhjrYRqOo8YF61bQ9XWT7Xydc3xhhTt4C6s9gYY8zRLBEYY0yAs0RgjDEBzhKBMcYEOEsExhgT4CwRGGNMgLNEYIwxAc4SgTHGBDhLBMYYE+AsERhjTICzRGCMMQHOEoExxgQ4SwTGGBPgLBEYY0yAs0RgjDEBzhKBMcYEOEsExhgT4CwRGGNMgLNEYIwxAc4SgTHGBDhLBMYYE+AsERhjTIBzNBGIyCgR2SQiW0Xk/hr2DxeR5SJSISJXOhmLMcaYmjmWCETEBTwHXAD0AiaISK9qxb4Hrgf+7VQcxhhjji3YwWMPBbaq6nYAEZkOjAHWHy6gqju9+9wOxmGMMeYYnKwaSgLSq6xneLcZY4xpRE6JxmIRmSQiaSKSlp2d7e9wjDGmSXEyEWQC7aqsJ3u3HTdVnaqqqaqampiY2CDBGWOM8XAyESwFuopIRxFpBowH5jj4esYYY06AY4lAVSuAO4D5wAZghqquE5FHRWQ0gIgMEZEMYCzwooiscyoeY4wxNXOy1xCqOg+YV23bw1WWl+KpMjLGGOMnp0RjsTHGGOdYIjDGmABnicAYYwKcJQJjjAlwlgiMMSbAWSIwxpgAZ4nAGGMCnCUCY4wJcJYIjDEmwFkiMMaYAGeJwBhjApwlAmOMCXCWCIwxJsBZIjDGmABnicAYYwKcJQJjjAlwlgiMMSbAWSIwxpgAZ4nAGGMCnCUCY4wJcJYIjDEmwFkiMMaYAOdoIhCRUSKySUS2isj9NexvLiLvePcvEZEOTsZjjDHmaI4lAhFxAc8BFwC9gAki0qtasZuAXFXtAjwN/NWpeIwxxtTMyW8EQ4GtqrpdVcuA6cCYamXGAG94l2cC54iIOBiTMcaYapxMBElAepX1DO+2GsuoagWQD8Q7GJMxxphqgv0dQH2IyCRgkne1SEQ2neChEoD9DROVIyy+k2PxnbzGHqPFd+La17bDyUSQCbSrsp7s3VZTmQwRCQaigQPVD6SqU4GpJxuQiKSpaurJHscpFt/JsfhOXmOP0eJzhpNVQ0uBriLSUUSaAeOBOdXKzAGu8y5fCXymqupgTMYYY6px7BuBqlaIyB3AfMAFvKqq60TkUSBNVecArwBvichWIAdPsjDGGONDjrYRqOo8YF61bQ9XWS4BxjoZQzUnXb3kMIvv5Fh8J6+xx2jxOUCsJsYYYwKbDTFhjDEBrkkmgsY8tIWItBORhSKyXkTWicidNZQZISL5IrLS+3i4pmM5GONOEVnjfe20GvaLiDzrPX+rRWSQD2PrXuW8rBSRAhG5q1oZn58/EXlVRPaJyNoq2+JE5BMR2eL9GVvLc6/zltkiItfVVMaB2J4QkY3ev99sEYmp5bnHfC84HONkEcms8ne8sJbnHvP/3cH43qkS204RWVnLc31yDk+KqjapB56G6W1AJ6AZsAroVa3ML4EXvMvjgXd8GF8bYJB3OQrYXEN8I4C5fjyHO4GEY+y/EPgQEGAYsMSPf+u9QHt/nz9gODAIWFtl2+PA/d7l+4G/1vC8OGC792esdznWB7GdDwR7l/9aU2z1eS84HONk4J56vAeO+f/uVHzV9j8FPOzPc3gyj6b4jaBRD22hqntUdbl3uRDYwNF3XDd2Y4A31WMxECMibfwQxznANlXd5YfX/hFV/RJPz7eqqr7P3gAureGpPwM+UdUcVc0FPgFGOR2bqn6snrv5ARbjuc/Hb2o5f/VRn//3k3as+LzXjquA/zT06/pKU0wEp8zQFt4qqYHAkhp2ny4iq0TkQxHp7dPAQIGPRWSZ967u6upzjn1hPLX/8/nz/B3WSlX3eJf3Aq1qKNMYzuWNeL7h1aSu94LT7vBWX71aS9VaYzh/ZwJZqrqllv3+Pod1aoqJ4JQgIpHALOAuVS2otns5nuqO/sA/gPd9HN5PVXUQnpFjbxeR4T5+/Tp5b1IcDbxbw25/n7+jqKeOoNF10RORB4EK4O1aivjzvfA80BkYAOzBU/3SGE3g2N8GGv3/U1NMBMcztAVyjKEtnCIiIXiSwNuq+l71/apaoKpF3uV5QIiIJPgqPlXN9P7cB8zG8/W7qvqcY6ddACxX1azqO/x9/qrIOlxl5v25r4YyfjuXInI9cDFwjTdRHaUe7wXHqGqWqlaqqht4qZbX9ut70Xv9uBx4p7Yy/jyH9dUUE0GjHtrCW5/4CrBBVf9WS5nWh9ssRGQonr+TTxKViESISNThZTyNimurFZsD/MLbe2gYkF+lCsRXav0U5s/zV03V99l1wH9rKDMfOF9EYr1VH+d7tzlKREYB9wGjVfVQLWXq815wMsaq7U6X1fLa9fl/d9K5wEZVzahpp7/PYb35u7XaiQeeXi2b8fQmeNC77VE8b3qAUDxVCluB74BOPoztp3iqCFYDK72PC4FbgVu9Ze4A1uHpAbEY+IkP4+vkfd1V3hgOn7+q8QmeSYe2AWuAVB//fSPwXNijq2zz6/nDk5T2AOV46qlvwtPu9CmwBVgAxHnLpgIvV3nujd734lbgBh/FthVP3frh9+DhXnRtgXnHei/48Py95X1/rcZzcW9TPUbv+lH/776Iz7v99cPvuypl/XIOT+ZhdxYbY0yAa4pVQ8YYY46DJQJjjAlwlgiMMSbAWSIwxpgAZ4nAGGMCnCUCE9BEpLLaaKYNNnqliHSoOlplPcpHiMgC7/LX3puVjHGcvdFMoCtW1QH+DsLrdGCR98ayg/rDoHDGOMq+ERhTA+8Y8o97x5H/TkS6eLd3EJHPvAOhfSoiKd7trbzj+q/yPn7iPZRLRF4Sz9wTH4tIWA2v1dk7lv004GpgGdDf+w2lpY9+ZRPALBGYQBdWrWpoXJV9+araF/gn8Hfvtn8Ab6hqPzwDtT3r3f4s8IV6BrobhOcuUoCuwHOq2hvIA66oHoCqbvN+K1mGZxyaN/DcuTpAPePTGOMou7PYBDQRKVLVyBq27wRGqup27yCBe1U1XkT24xnqoNy7fY+qJohINpCsqqVVjtEBz1wDXb3rvwVCVHVKLbEsVdUhIjILuFNrGb/GmIZm3wiMqZ3Wsnw8SqssV1JDu5yIvOBtVO7qrSIaBcwVkd+c4Gsac1wsERhTu3FVfi7yLn+LZ4RLgGuAr7zLnwK3AYiIS0Si6/siqnor8Afgj3hmMfvAWy309MmFb0z9WK8hE+jC5MeTjn+kqoe7kMaKyGo8n+oneLf9CnhNRO4FsoEbvNvvBKaKyE14Pvnfhme0yvo6C3gTz2xXX5zQb2LMCbI2AmNq4G0jSFXV/f6OxRinWdWQMcYEOPtGYIwxAc6+ERhjTICzRGCMMQHOEoExxgQ4SwTGGBPgLBEYY0yAs0RgjDEB7v8BF1gxlH6BZDAAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["#### Translate"],"metadata":{"id":"niUBA7kSgA8J"}},{"cell_type":"code","source":["result = model.translate(['Ich gehe heute nach Hause.'])\n","result[0].numpy().decode()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1669569515258,"user_tz":-60,"elapsed":701,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"881a6c03-bc6f-446d-db9e-42133f64d61c","id":"0vPBMoD1gA8K"},"execution_count":184,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'im going home today . '"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":184}]},{"cell_type":"markdown","source":["#### Attention Plot"],"metadata":{"id":"_bC7LtgAgA8L"}},{"cell_type":"code","source":["model.plot_attention('Ich gehe heute nach Hause.')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":525},"executionInfo":{"status":"ok","timestamp":1669569515259,"user_tz":-60,"elapsed":7,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"a8c3f7d2-7c71-4c12-e9b7-04207b93e148","id":"kgLCv5eDgA8L"},"execution_count":185,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 720x720 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAnoAAAH8CAYAAABGqwJUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwldX3v/9d7NpYBlQgKrnFXUBQcEdyjiXtcgsZcURSNhIi5RkTjEq9eE3dMXOJGlMU1/uISvV6XLIoLQRFXFFAR2RQQUNkGGBg+vz+q+tI0Z3p6lu6q/vbr+Xicx5xaTtXn1Jzu8+5v1fdbqSokSZLUnmVDFyBJkqT5YdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWrUiqELUCfJn2zGy75QVVdu9WIkSVIT4r1uxyHJdZv4kgLuUlVnzEc9kiRp8bNFb1x2rapfz2XFJJfNdzGSJGlx8xq98TgW2JTTsB8GLp2nWiRJUgM8dStJktQoW/RGJMn6JLcYug5JktQGg964ZOgCJElSOwx6kiRJjbLX7fj8aZJZO1lU1QcXqhhJkrR42RljRPqx9NbSjZG3IVVVN1mgkiRJ0iJm0BuRPujNeSw9SZKk2XiN3riYuiVJ0lZj0BsXe91K0gJLcq8k/5TkC0l26+c9KcleQ9cmbSmD3rjMeneMJGuSfHEB65GkpiV5JPBt4NbAw4Ht+kV3Al49VF0ajyS/t6mPoWuezqA3IlV1ELBvkrckeX2SOwIkuWuSzwDfGrZCafFI8pgkn0tySpLb9vP+PMkjhq5No/J3wGFV9WRg3bT5xwH7DFKRxuYi4MJNePx66vt7DBxeZUSSPAs4GvgN8HvAc5O8EHgf8CngPlV18oAlSotCkgOA9wLvBx4BrOwXLQdeCvzXQKVpfO4JfH7C/KnfwxLAU+g+ExsTJn+eBmPQG5cXAa+oqjcm+VPgX4CXAHtX1c+HLU1aVF4KPK+q/iXJn0+b/03gtQPVpHH6Dd1p2zNnzN8bOHfBq9EYnQV8raounsvKSc4ArpnfkubOU7fjcifg4/3zTwDr6U4pGPKkTXMX4IQJ8y8HHIdS030UeEuS29CNfLAiyUOBIwAHpxdVdYe5hrx+/XtW1TnzWdOmMOiNy2rgCoCqug64ChjNh0XjlWTnJPdPss3QtYzEr4C7Tpj/EMA/nDTd3wK/oGu12QE4Bfgy8A3gdQPWJW0Vnrodn8cluaR/vgx4VJILpq9QVZ9a+LI0Rkl2BD5Ad/1I0bVknZHkvcD5VfWaAcsb0pHAO6adtr1tkgcDbwZeM1hVGp2qugY4IMmr6E7XLgO+V1U/G7YyjU2SAAcC+wN3pPudewbwr8BHaqR3oPDOGCPS3xljY6qqls97MVoUkrwbuDdwKF0LxJ5VdUaSxwOvq6p7D1rggJK8ju661237WVcDR1TVq4arSotBkjsD51bVVUPXovFI8ingScDJdC2/AXan69Dz6araf8DyNsigJy1iSc4FnlxV305yGXDvPujdCfh+Ve04cImDSrI93S/iZcApVXX5wCUNKsm2wAvpeiLfghmX71TVnkPUNaQkrwd+UlXH9i02/053fC4BHl1VDmulqZ787wP+pKr+fcayRwGfBA6uqo8OUd9svEZvkUnyh0PXoFHZCZh0kfCOdJ15lqQkRyXZsarWVtVJVXViVV2eZHWSo4aub0DvBl5G18P03+i+nKY/lqIDgJ/0zx8D3AfYl64jxhuHKkqj8wzgTTNDHkBVfQl4S7/O6NiitwgkuTVwEPAc4PaeutWUJMcB/1ZVb+tb9Pasql8keQ/dZ+Wxw1Y4jCTrgd2q6tcz5u9Md+3ikrw+OclvgD+tqv8cupaxSHIVcOeqOjfJP9F9Lx7an749qapuNnCJGoEkvwL+uKq+s4Hla4DPVtWtFrayjVuSv+wWgyTLgScCzwUeCfyQbgDYfx2yLo3OK4AvJdmD7uf5sP75PnQ9TJeU/tZD6R87Jbl22uLlwOOACya9dolYiz35Z7oYuD3dmHmPpGvxhO7nyfuPa8rNgfNmWX4eIx1g26A3MknuBvw5Xc+eK+jGeHok8MyqOmXI2jQ+VfXfSR4AHE43bMgjgO8C+y3Ru6hcRNcTrugulp6pWNr3L30z3R8Dh4y1h+AAPgl8NMlP6b6ov9TPvw9w+mBVaWxWMvsgyNdy/R14RsVTtyOS5Ot0vXc+CXyoqr7az7+G7iJ7g540i36g29CNg7Y/N7xl0TrgrKr61RC1DSXJZ2fMeghdR4NTmPHFVVVPWKi6xiLJCroOKrcDjqmq7/XzXwRcVlXvH7I+jUM/KsZRdK3ik2wPHDTGS6sMeiPSn2Z6F3BkVf142nyDnmaV5FZM7kX53WEqGlaS2wNn22oFSY6e67pVddB81iItVv310Bv9fVJVfzD/1Wwag96IJNmL7rTt0+l6xX0Q+BjdNTUGPd1I/5n5MHB3bnw90ZIdczHJ3rMtX6oBWDfmZ0WtM+iNUD/W1VPpetk+iK6V5mXA+6vqt0PWpnFJ8m26i8lfS3fbrxv8QFfVWUPUNbT+NEtxw/D7/47NEg7AewDLq+qHM+bvCVy7FP+Y9LOi1hn0RiTJ7YBzpp9u6rv4T3XOuDnw5ap6zEAlamSSXAHsVVU/HbqWMelP3U63EtgLeCXw8qr6wsJXNbwkxwPvmjmoa5I/A15QVQ8aprLh+FnRXCQ5bC7rVdU/zHctm8qgNyIbGvurX7YceDzwnKp64oIXp1FK8k3gpVX1taFrWQySPBJ4dVU9cOhahtCPtbhXVZ0+Y/6dgO9W1U2HqWx8lvpnRTeU5BezLC5gV2CbMbYAO7zKuGxwzKaqWg98pn9oCevHipvyCuDNSf6W7v6LM3tRTu91KvgF3bAZS9V6YFKY2wnHjJtpqX9WNE1V3WHS/CR3BF5Hd7nVKMe5NehJi8/UWHFTpu7POXNe0Q0SvOTMCMPQHY/dgNdw/e2ulqKvAq9M8tT+j8ep4UVeCSzJVmE/K9ocSW4OvAo4BDge2LeqThq2qskMeuNzeJJZb7xeVa9dqGLGpL9B/X2YPIzIpwYpahij674/QjPDMHRf4OcAT1v4ckbjpcA3gNOTfKOf9yBgB5bgnVR6flY0Z0m2Aw6j+1k6E3jy2K/j9Bq9Eel7f/2EboTtDamq2nOBShqNJH9IN9TMzScsXrLDiGiyfuDk6a4DLgROr6rZfr6al2Q34AVcf1rye8C7l9pA0lP8rGgukiyjuyXp/6a7ROZVdDc2GH2IMuiNSB/0dp3UGWOpS/Jj4NvAK5bqF9KGJLkX8BfAneg665yX5El0d4H43rDVSdLil+QUunsivwN4J3DVpPXGeF20QW9EZut1u9T1w4jsWVU/H7qWMel7Bn4W+ALwWOAeVXVGkhcDD66qJw1a4IAMwBvW30nldsCq6fOXcu9tj4lm0zfETJkUnMJIzy55jd642Ottw44H7gYY9G7o74DDqurd/dAZU44DXjxMScObEYAfDmzXL7oT8GxgSQbgPsx8lO56vKlBgqd/aY3uS2q+eUw0R4v22miD3rj8b2DWjhhLyYxbE70XOKL/pTxpGJGlepuiewKfnzD/N8DM3oRLiQF4srfRDbGyO92lEI8Gbkl3Z5UXDVjXkDwm2qiq+urQNWwug964vJWu5WHt1Iwk9wBeQtcr7lNV9S8D1TaEk7jxrYmOnLDekh1GhC7Q3Zqu99d0ewPnLng142EAnuyhwOOq6rQkBVxYVccnuZouHP/HsOUNwmOijUpyMHBsVV3dT+8B/GSqw06S1cDfVNX/GrDMiZZtfBUtoPfQteoBkGRn4Ot0d8S4G/CRJE8fqLYh3AG4Y//vbI87DlXgCHwUeEuS29AF3hV9L8IjgA8OWtmwpgLwTEs9AG9HN5wIdMfoFv3zU4Al15u/5zHRXLyHGw42fgLdNZ1TdqAbj3J0bNEbl/3oLh6f8kxgHd0F9pckeRPdsAgfnfTi1lTVWUPXsAj8LXAMcBZdy+cpdH/AfYRutPalaioA/yk3DsBHD1rZsE4D7k7XAvx94JAk5wCHAr8csK4heUw0FzOvoV8019Tb63ZE+p6le1TVmf30Z+l6CP5VP7078NWq2mW4KoeR5HXAOVX13hnzDwFuXVWvGqaycehvw7M3Xcj7XlX9bOCSBpVkJV0A/jO6X8jXcX0APmipjo+W5ABgZVUd018D+0VgZ+Bq4FlV9f8NWuAAPCaai5nDn/XX/t67qs7op28J/GqMvW4NeiOS5ELgYVX14376fODFVfWRfvqOwMlVtXrAMgeR5GzgqVX1rRnz7wd8oqpuP0xlw0py1AYWFd04T6cDH1+qYw8agGfX323m7sDZVXXRxtZfCjwmmmQxBz1P3Y7LD4CD6G6D9jBgF+DL05bfCViSX9h0181cOGH+xXQ95JaqXYAH07VY/aifd0+6VqzvAH8CvDbJg6vq+8OUOIwkTwMewfW3zHtG0p1tqaonDFjaoCYcl6n5S/a4eEw0R49Lckn/fBnwqCQX9NM3G6imjTLojcvfAV/oryvaBTimqs6btvzJdPepXIrOpgs0Z8yY/xCW9sX1x9MNyfPcqloL/69F4p/p/nB4LF2njLfSfZEtCUneAvw18BW6P448dYHHZRKPiTbBB2ZMv2vG9Cg/O566HZl+OJVHAucD/1pV101bdjBw4lJrmQHo7/TwSuBvuL6V8xHAG4A3VdWbh6ptSEnOAx5eVafOmL878F9VtVuSvYD/rKpJ9wluUv9X9qFV9YmhaxkTj8uNeUzUOlv0RiLJPsB3+i/sUyetU1VHTlv/vsAPq+qaSeu2pqre2g838w66WxSF7mLptwNvGbK2ge0A7MaNPzO79ssALmXp/awvo+tBqRvyuNyYx0Szmvb9vH6O64/q+9lx9MbjBDZtINevALedp1pGqapeTtcbbl/g/sAuVfWyWtrN0p8GPpDkqUl+v388le4Uw6f6dfYBfjpYhcM4EnjG0EWMkMflxjwm2phF/f281P7KH7MAb0iydqNrdlZtfJXFrR9e5hlVdWn/fNI6wJK+uP4Q4B+AD3P9z/O1wFHA4f30qcDzFr60hZXkHdMmlwEHJPkj4Ifc+JZ5/3MhaxuRmwFPX+rHxc/K5ktyKnCXqlpK+WFRfz8vpf+osfsaXa/auToBuHKeahmLi7n+4taLhyxkrPoOGIf01zBOfX5+XlVXTFtnqZyWuteM6an3ffcZ85dyC/DueFzAz8qWeBewZK737S3q72c7Y0iSJDXKa/QkSZIaZdAbuX5IFc3gcZnM4zKZx+XGPCaTeVwm87hMthiOi0Fv/Eb/IRqIx2Uyj8tkHpcb85hM5nGZzOMy2eiPi0FPkiSpUXbGmGBVtq3tsnroMgBYx9WsYpuhywDg2juPp8f4tZesZcVNtx+6DJafP677V69bdwWrVg3/2c3aq4Yu4QbWXXcVq5ZtO3QZo+rDua6uYlVGcEyAMX0PXVNXsXIExyUrVw5dwg2sW7+WVcuH/527bqdxHZf1a69g+fbD/8696oJzL6qqXSYtc3iVCbbLavbd9rFDlzE6v3vnrYcuYXRWv+mmQ5cwSqu+e/rQJYxSXXvt0CWMUl3jcZlp+a13HbqEUTpn/9sMXcIo/fiIw87a0DJP3UqSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNWrRBL8kxST43dB2SJEljtWLoArbAC4EMXYQkSdJYLdqgV1WXDF2DJEnSmDVx6jbJcUnek+StSX6T5MIkL0yyTZJ3JfldkrOTPHPouiVJkhbKog16ExwAXAbcH3gj8Dbg34CfAmuAY4H3J9ltsAolSZIWUEtB78dV9Zqq+hnwD8BFwDVV9faqOh14Ld01fQ+c9OIkByc5KclJ67h64aqWJEmaJy0FvR9OPamqAn4NnDxt3jXAb4FbTHpxVR1ZVWuqas0qtpnvWiVJkuZdS0HvmhnTtYF5Lb1nSZKkDTL0SJIkNcqgJ0mS1CiDniRJUqMW84DJz572/GETlt9zwrxd57cqSZKk8bBFT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWrUiqELGKOq4rp11wxdxujc7CnnD13C6Jz2jp2GLmGUdj9tu6FLGKXfPfQOQ5cwSqt/edXQJYzOsvMvGbqEUbr5j9YNXcKiY4ueJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMWRdBL8rAklWTnoWuRJElaLBZF0AP+G9gNuHjoQiRJkhaLFUMXMBdVtQ44f+g6JEmSFpMFadFLsjrJB5NcnuSCJC9P8rkkx/TLd0pybJLfJrkyyX8m2WPa629w6jbJs/ttPSLJj5JckeQrSe4wY78v7/d3eb//Vyc5cyHesyRJ0tAW6tTtW4GHAk8GHg7cG3jwtOXHAPcHngjsA6wFvphku1m2uQ3wcuA5wH7AzYD3Ti1M8mfAq4FXAnsDpwKHbZV3I0mStAjM+6nbJDvQhbEDq+o/+nnPBc7tn98FeALw0Kr6Wj/vmcDZwAHA+zew6RXAoVX1k/41RwBHJUlVFfBC4Jiqmnr9G5L8AXDXDdR5MHAwwLZsv2VvWpIkaQQWokXvTsBK4MSpGVV1BfCjfvIewHXACdOWXwKcDOw+y3avngp5vV8Bq4Cd+um7T99n71sb2lhVHVlVa6pqzUq2mfUNSZIkLQZj73Vbsyy7dgPrjv09SZIkLYiFCEU/B64B7jc1I8n2wD37yVP7OvabtvwmwL2AU7Zgv6dN32dvny3YniRJ0qIy79foVdXlSY4C3pTkIuA84G/pwl1V1c+SfAZ4X3+d3O+A1wGXAh/dgl2/HTg6ybeBr9N1BLk/8Nst2KYkSdKisVDj6B0OrAY+C1wO/CNwS+CqfvlBwNv65dsCxwOPrqorN3eHVfUvSe4IvBHYHvgUXa/cJ27uNiVJkhaTBQl6VXU58Mz+QZJtgL8GPt8v/y3wrFlefxyQadPH0A3JssF1+nmvB14/NZ3k08Dpm/9OJEmSFo8FCXpJ9qLrXXsisCPwN/2/H5/HfW4P/CXwRbqOG/vTtebtP1/7lCRJGpOFvAXaYcDd6ELX94GHVNW587i/Ah4DvALYDvgZ8Iyq+vQ87lOSJGk0FurU7feANQuxr2n7vBL4w4XcpyRJ0pg45pwkSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjVgxdwBhl2TKWbbft0GVoEdjppJVDlzBKV+5526FLGKUVz7lg6BJGaeVf2eYw0/ozzh66hFHa9tzzhi5h0fGnS5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlR8x70khyX5J/mez+SJEm6IVv0JEmSGmXQkyRJatRCBb1lSV6f5KIkv05yRJJlAEl2SnJskt8muTLJfybZY+qFSZ6d5PIkj0lyWpK1ST6b5KZJnpLkZ0kuSfKhJNtNe12SvDTJz/vtnpzkGQv0fiVJkga3UEHvAOBa4AHAC4C/Bp7WLzsGuD/wRGAfYC3wxemhDdgGeHG/nUcAa4BPAs8C9geeBDweeP601/w98FzgUGB34A3A+5I8bqu/O0mSpBFasUD7OaWq/lf//KdJngc8IslJwBOAh1bV1wCSPBM4my7UvX9anYdW1U/6dT4KvAi4ZVVd1M/7DPAHwFuTrAYOAx5ZVV/vt/GLJPvQBb//O7PAJAcDBwNsm9Vb9c1LkiQNYaGC3g9nTP8KuAVwD+A64ISpBVV1SZKT6Vrhplw9FfJ6FwDnT4W8afOmXrM7sC1dy2BNW2clcOakAqvqSOBIgJsu37kmrSNJkrSYLFTQu2bGdLHx08bTw9a1E5bNts2pf/+YrnVwtlokSZKatFBBb0NOpQtl+wFTp25vAtwLOHoLtnsKcDVw+6r68pYWKUmStBgNGvSq6mf9tXXv66+R+x3wOuBS4KNbsN3LkhwBHJEkdCFyB2Bf4Lr+NK0kSVLTxjCO3kHAicBn+3+3Bx5dVVdu4XZfBbwGOBz4MfAfdD10f7GF25UkSVoUUmW/g5luunzn2nf7xw9dhhaBCw7cc+gSRulmP/dS2EmuffHFQ5cwSjv81RjaHMZl/elnDl3CKC1btXLoEkbp36/88Heqas2kZRv96UqyzVzmSZIkaVzm8mfUCXOcJ0mSpBHZYGeMJLsCtwa2S7IXkH7RTeiuo5MkSdKIzdbr9lHAs4HbAG/l+qB3KfCK+S1LkiRJW2qDQa+qjgWOTbJ/VX1yAWuSJEnSVjCXa/SelOSmUxNJbp/kv+axJkmSJG0Fcwl63wC+leSxSZ5HNx7d2+a3LEmSJG2pjd4Zo6rel+THwFeAi4C9qur8ea9MkiRJW2Qu4+g9EzgKOBA4Bvh8knvPc12SJEnaQnO51+3+wIOq6tfAx5J8GjgWuM+8ViZJkqQtMpdTt08CSLJ9Va2tqhOT7DP/pUmSJGlLzOXU7X5JTgFO66fvjZ0xJEmSRm8uvW7fRjd48sUAVfUD4CHzWZQkSZK23FyCHlV1zoxZ6+ehFkmSJG1Fc+mMcU6SBwCVZCXwQuDU+S1LkiRJW2ouLXqHAIcCtwZ+Sdfb9vnzWZQkSZK23Fxa9O5WVQdMn5HkgcDx81OSJEmStoa5tOi9c47zJEmSNCIbbNFLsh/wAGCXJIdNW3QTYPl8FyZJkqQtM9up21XADv06O06bfynwlPksSpIkSVtug0Gvqr4KfDXJMVV11gLWJEmSpK1go9foGfIkSZIWpzkNmCxJkqTFZy73un3gXOZJkiRpXBxeRZIkqVEOryJJktQoh1eRJElqlMOrTFCrt+WaNXcbuozRWf61Hwxdwujs+qmfD13CKJ3697cfuoRRWv2V3YYuYZR2WH7h0CWMTpZ74mySZbfcZegSxunMDS+ay71uj0lSM2dW1cM3vyJJkiTNt7kEvcOnPd8W2B+4dn7KkSRJ0tay0aBXVd+ZMev4JCfOUz2SJEnaSjYa9JL83rTJZcB9gZvOW0WSJEnaKuZy6vY7QAGhO2X7C+C581mUJEmSttxcTt3eYSEKkSRJ0tY1l1O32wLPBx5E17L3deC9VXXVPNcmSZKkLTCXU7cfBC7j+tuePR34EPDU+SpKkiRJW24uQe+eVbX7tOmvJDllvgqSJEnS1rFsDut8N8m+UxNJ7g+cNH8lSZIkaWuYS4vefYH/TnJ2P3074CdJTgaqqvact+okSZK02eYS9B4971VIkiRpq5tL0Pv7qnrm9BlJPjRzniRJksZlLtfo7TF9IskKutO5kiRJGrENBr0kL09yGbBnkkuTXNZPXwB8ZsEqlCRJ0mbZYNCrqjdU1Y7AW6rqJlW1Y/+4eVW9fAFrlCRJ0maYyzV6X0jykJkzq+pr81CPJEmStpK5BL2XTHu+LbAP8B3g4fNSkSRJkraKjQa9qvrj6dNJbgu8bd4qkiRJ0lYxl163M50L3GNrFyJJkqSta6MtekneCVQ/uQy4D/Dd+SxKkiRJW24u1+hNv6/ttcDHqur4eapHkiRJW8lcgt7HgTv3z0+vqqvmsR5JkiRtJbMNmLwiyZvprsk7FvggcE6SNydZuVAFSpIkafPM1hnjLcDvAXeoqvtW1d7AnYCbAUcsRHGSJEnafLMFvccDz6uqy6ZmVNWlwF8Cj53vwiRJkrRlZgt6VVU1YeZ6ru+FK0mSpJGaLeidkuTAmTOTPAM4bf5KkiRJ0tYwW6/bQ4FPJXkO3S3PANYA2wFPnu/CJEmStGU2GPSq6pfA/ZM8HNijn/35qvqvBalshiSfAy6qqmcPsX9JkqTFZi73uv0y8OXN3UGS44AfVdULNncbkiRJ2nSbc69bSZIkLQLzGvSSHAM8FDg0SfWP30/ykCTfSnJVkguS/GOSVdNet32SY5Jc3i9/xYRtPyPJt5NcluTXSf41ya37ZUlyepLDZ7zmLn0Ne8/n+5YkSRqD+W7ReyFwAnA0sFv/uAb4AvA9YC/gucD/AN4w7XVHAH8E7A88ol/vITO2vQp4NXBvujH/dgY+Bt24MMAHgINmvOY5wPer6rtb5d1JkiSN2LwGvaq6BFgHrK2q86vqfOD5wK+A51fVqVX1OeBlwAv6lrwd6MLfS6vqS1X1I7rAdt2MbR9VVZ+vqjOq6kS6gZwfnOQ2/fOEkEQAAAsZSURBVCpHA3dNsi9AkuXAgXQB8EaSHJzkpCQnrVt3xVY+EpIkSQtviGv07gF8s6qmB7dv0LXQ3ZnuNmur6FoCAaiqy4GTp28kyd5JPpPkrCSXASf1i27Xv+Z84HN0rXgAj6a7pdtHJhVVVUdW1ZqqWrNq1eotfIuSJEnDG1tnjDndcSPJauBLwFrgmcD96IIcdCFxyvuBpyXZni7wfbqqfrv1ypUkSRqvhQh664Dl06ZPBfZNMn3fD+rX+3n/uAbYd2phH+zuOW39u9Ndk/eKqvpaVZ0G3GLCvr8IXAocAvwxcNQWvxtJkqRFYiGC3pnAPn1v252BdwO3At6d5B5JHge8Efinqlrbn6b9APCmJH+UZA+6gDY9LJ4NXE13Xd8d+2383cwd9/flPYquo8cvgUEGe5YkSRrCQgS9I+ha604BLgRWAo+h60n7fbog9jFg+hAqhwNfAT7d//sj4GtTC6vqQuBZwJP67b4aOGwD+z+K7nTu0X1vXEmSpCVho3fG2FJV9VNgvxmzzwTuP8trrqDrIXvgLOt8HPj4jNmZsOquwHrgmI1XK0mS1I55D3pDSbINsAvdKd1PV9XZA5ckSZK0oMbW63Zr+h/AWXSdNjZ0WleSJKlZzQa9qjqmqpZX1d5Vdc7Q9UiSJC20ZoOeJEnSUmfQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGrVi6ALG6OqdizOfV0OXMTp3PsGPy0x12eVDlzBKu7/hwqFLGKXL9rzF0CWM0jmP22XoEkbnNu/51dAljFL99pKhS1h0bNGTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBr1ekoOTnJTkpPWXXTF0OZIkSVvMoNerqiOrak1VrVm+4+qhy5EkSdpiBj1JkqRGGfQkSZIataSCXpIXJDlt6DokSZIWwpIKesDOwN2GLkKSJGkhLKmgV1WvqaoMXYckSdJCWFJBT5IkaSkx6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjVgxdwBjda8eLOfEPjh66jNF53K2eOHQJo7P+3POGLmGc1l45dAWjtMM3fj50CaO0+otXDF3C6NT664YuYZTW3+/OQ5cwTsdveJEtepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSowx6kiRJjTLoSZIkNcqgJ0mS1CiDniRJUqMMepIkSY0y6EmSJDXKoCdJktQog54kSVKjDHqSJEmNmregl+S4JNU/9p2v/cyxljOn1bLzkLVIkiQtlPlu0Tsa2A34DsC0sDXzcUi//GH99GlJVkzfUB/WDp82PT1IrktyXpIvJnlGksyo437A/vP7ViVJksZlvoPe2qo6v6qumTbveXThb/rj2Bmvuz3w3DlsfypI3hF4AnAC8D7g00mWT61UVRcCv9ncNyFJkrQYrdj4Klvd76rq/I2s8w7gNUk+XFVXzLLe2mnbOhf4dpJvAl8EDqQLgpIkSUvSWDtjvBO4BjhsU19YVV8CTsZTtZIkaYkbIuh9KMnlMx73mrHOVcCrgJck2WUz9nEK3encOUtycJKTkpx04cXrN2OXkiRJ4zJE0HsJcJ8Zj59MWO9DwJl0gW9TBahNeUFVHVlVa6pqzS43X77xF0iSJI3cENfonV9Vp29spaq6LsnLgH9L8vZN3MfuwBmbVZ0kSVIjxnqNHgBV9XngeOB1c31NkkcB9wQ+MV91SZIkLQZDtOjdLMmuM+ZdXlWXb2D9lwLfpOucMdP2/bZW0A2z8th+/c8AH95K9UqSJC1KQ7To/TNw3ozHyza0clV9m651bpsJiw/qX38G8H+A/YBDgCdXlT0qJEnSkragLXpVNfOOFTOXH0fXkWLm/KcBT5sx72FbszZJkqTWzHeL3sH98Cn3m+f9zCrJj4EvDFmDJEnSQpvPFr0DgO365+fM437m4rHAyv65t0KTJElLwrwFvar65Xxte1NV1VlD1yBJkrTQRj28iiRJkjafQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWqUQU+SJKlRBj1JkqRGGfQkSZIaZdCTJElqlEFPkiSpUQY9SZKkRhn0JEmSGmXQkyRJapRBT5IkqVEGPUmSpEYZ9CRJkhpl0JMkSWpUqmroGkYnyYXAWUPX0dsZuGjoIkbI4zKZx2Uyj8uNeUwm87hM5nGZbCzH5fZVtcukBQa9kUtyUlWtGbqOsfG4TOZxmczjcmMek8k8LpN5XCZbDMfFU7eSJEmNMuhJkiQ1yqA3fkcOXcBIeVwm87hMNufjkuTyrb3zJL+f5OmbumyO235Ykgdsxkv9rEzmcZnM4zLZ6I+L1+hJ0jRJLq+qHbbyNh8GHF5Vj9+UZXPc9muAy6vqiC2pUVKbbNGTpAn6lrLjknwiyWlJPpIk/bIzk7w5yclJTkxy537+MUmeMm0bU62DbwQenOT7SV40Y1c3WJZkeZK3JPl2kh8m+Yt+Wy9KclT//F5JfpRkd+AQ4EX96x88v0dF0mKzYugCJGnE9gL2AH4FHA88EPhGv+ySqrpXkgOBtwGztci9jA232t1gWZKD+23fL8k2wPFJ/h14O3BckicDrwT+oqpOSfJebNGTtAG26EnShp1YVedW1XXA94Hfn7bsY9P+3W8r7vORwIFJvg98C7g5cJe+hmcDHwK+WlXHb8V9SmqULXqStGFXT3u+nhv+zqwJz6+l/wM6yTJg1WbsM8BfVdWXJiy7C3A5cKvN2K6kJcgWPUnaPE+b9u8J/fMzgfv2z58ArOyfXwbsuIHtzFz2JeAvk6wESHLXJKuT3BR4B/AQ4ObTrgWcbduSljiDniRtnp2S/BB4ITDVweKfgYcm+QHd6dwr+vk/BNYn+cGEzhgzl70fOAX4bpIfAe+ja0n8R+BdVfVT4LnAG5PcAvg/wJPtjCFpEodXkaRNlORMYE1VjeEel5K0QbboSZIkNcoWPUmSpEbZoidJktQog54kSVKjDHqSJEmNMuhJkiQ1yqAnSZLUKIOeJElSo/5/m+8R7IXbb7YAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"markdown","source":["# Questions:\n","1. Q: Which parts of the sentence are used as a token? Each character, each word, or are some words split up?<br>\n","A: Words are used as tokens. The TextVectorization split's the words based on a seperator which is a whitespace by default. _The Combination of words like \"didn't\" would be one token according to this, am I correct? It is the oppositve of wordsplitting but maybe worth noting? However, maybe I'm wrong_\n","\n","2. Q: Do the same tokens in different language have the same ID?\n","e.g. Would the same token index map to the German word die and to the English word die?<br>\n","A: No, they are processed separately. Different dictionaries - Same combinations of characters do not correspond to the same ID's in different languages dictionries.\n","\n","3. Q: Is the decoder attending to all previous positions, including the previous decoder predictions?<br>\n","A: Yes, but not in training (?). During training the decoder uses Teacher Forcing so the decoder is taking the previous decoder predictions into account. When applied to unseen data there is no correct previous output to fallback to so the decoder uses its own previous output. _I'm actually not 100% sure, that we actually use teacher forcing in training, but it would make sense to me xD. If not the answer is simply \"yes\" I assume_\n","\n","4. Q: Does the encoder output change in different decoding steps? <br>\n","A: No, the output of the encoder is independent of the decoding process.\n","\n","5. Q: Does the context vector change in different decoding steps?<br>\n","A: Yes, different parts of the sentence are attended to at different points.\n","\n","6. Q: The decoder uses teacher forcing. Does this mean the time steps can be computed in parallel? <br>\n","A: There are hidden-to-hidden connections, so no, cannot be parallelized\n","\n","7. Q: Why is a mask applied to the loss function?<br>\n","A: To ensure all sequences are of the same length\n","\n","8. Q: When translating the same sentence multiple times, do you get the same result? Why (not)? If not, what changes need to be made to get the same result each time?<br>\n","A: Yes, we got the same result for the translations each time. \n"],"metadata":{"id":"wSgivPFnu3if"}},{"cell_type":"markdown","source":["# ----------------------------------------------------------------------------------------"],"metadata":{"id":"FRF0TPr8ss2Y"}},{"cell_type":"markdown","source":["# Extra"],"metadata":{"id":"ICxtPC66qj02"}},{"cell_type":"markdown","source":["# Hindi"],"metadata":{"id":"R4-75ePkqgju"}},{"cell_type":"markdown","source":["### Importing Hindi Dataset "],"metadata":{"id":"efYqw75eDyKU"}},{"cell_type":"code","source":["path_to_file2 = pathlib.Path(os.path.join(project_dir, \"hin.txt\"))\n","target_raw2, context_raw2 = load_data(path_to_file2)"],"metadata":{"id":"ixi4e7QrD1Pm","executionInfo":{"status":"ok","timestamp":1669569515894,"user_tz":-60,"elapsed":641,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":186,"outputs":[]},{"cell_type":"code","source":["print(target_raw2[-1])\n","print(context_raw2[-1])"],"metadata":{"id":"BN99hzl0GaAz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569515894,"user_tz":-60,"elapsed":17,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"ffde72b1-c654-4972-97c7-efc7cf9ee0e9"},"execution_count":187,"outputs":[{"output_type":"stream","name":"stdout","text":["When I was a kid, touching bugs didn't bother me a bit. Now I can hardly stand looking at pictures of them.\n","जब मैं बच्चा था, मुझे कीड़ों को छूने से कोई परेशानी नहीं होती थी, पर अब मैं उनकी तस्वीरें देखना भी बर्दाश्त नहीं कर सकता।\n"]}]},{"cell_type":"markdown","source":["### Lower and Split Punct Function"],"metadata":{"id":"bQu78hzIUQKW"}},{"cell_type":"code","source":["# import unicodedata\n","\n","# # Converts the unicode file to ascii\n","# def unicode_to_ascii(s):\n","#   return ''.join(c for c in tf_text.normalize_utf8(s, 'NFD')\n","#       if unicodedata.category(c) != 'Mn')\n","\n","def tf_lower_and_split_punct2(text):\n","  # # text = unicode_to_ascii(text)\n","  # text = re.sub(r\"([?.!,¿])\", r\" \\1 \", text)\n","  # text = re.sub(r'[\" \"]+', \" \", text)\n","\n","  # Split accented characters.\n","  text = tf_text.normalize_utf8(text, 'NFKD')\n","  text = tf.strings.lower(text)\n","  # Keep space, a to z, and select punctuation.\n","  text = tf.strings.regex_replace(text, '[^ a-zA-Z\\u0900-\\u097f?.!,¿]+', '')\n","  # text = tf.strings.regex_replace(text, pattern=r'.^ a-z.', rewrite=r'')\n","  text = tf.strings.regex_replace(text, '[,?!]', r' \\0 ')\n","  # Strip whitespace.\n","  text = tf.strings.strip(text)\n","\n","  text = tf.strings.join(['[START]', text, '[END]'], separator=' ')\n","  return text"],"metadata":{"id":"nyciqE2LrbPz","executionInfo":{"status":"ok","timestamp":1669569515896,"user_tz":-60,"elapsed":14,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":188,"outputs":[]},{"cell_type":"markdown","source":["### Hindi Data"],"metadata":{"id":"KyYKoPtFGpfq"}},{"cell_type":"code","source":["BUFFER_SIZE = len(context_raw2)\n","BATCH_SIZE = 32\n","\n","is_train = np.random.uniform(size=(len(target_raw2),)) < 0.6\n","\n","train_raw2 = (\n","    tf.data.Dataset\n","    .from_tensor_slices((context_raw2[is_train], target_raw2[is_train]))\n","    .shuffle(BUFFER_SIZE)\n","    .batch(BATCH_SIZE))\n","val_raw2 = (\n","    tf.data.Dataset\n","    .from_tensor_slices((context_raw2[~is_train], target_raw2[~is_train]))\n","    .shuffle(BUFFER_SIZE)\n","    .batch(BATCH_SIZE))"],"metadata":{"id":"aYKs679NEaIr","executionInfo":{"status":"ok","timestamp":1669569515896,"user_tz":-60,"elapsed":13,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":189,"outputs":[]},{"cell_type":"code","source":["hindi = []\n","\n","for example_context_strings2, example_target_strings2 in train_raw2.take(1):\n","  for x in example_context_strings2[:5]:\n","    print(\"\".join([chr(i) for i in tf.strings.unicode_decode(x, 'utf-8')]))\n","  # print(example_context_strings2[:5])\n","  print()\n","  print(example_target_strings2[:5])\n","  # for x in example_context_strings2:\n","    # hindi.append(\"\".join([chr(i) for i in tf.strings.unicode_decode(x, 'utf-8')]))\n","  # for x in example_context_strings2[:5]:\n","    # print(\"\".join([chr(i) for i in tf.strings.unicode_decode(x, 'utf-8')]))\n","  # h_text = tf.strings.join(['[START]', hindi, '[END]'], separator=' ')\n","  break"],"metadata":{"id":"fh58rmgLEhEF","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569515896,"user_tz":-60,"elapsed":12,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"0c860dac-fd3e-4dc3-a6b0-9e9f1fe86931"},"execution_count":190,"outputs":[{"output_type":"stream","name":"stdout","text":["बैठिए।\n","अगर आपके पास समय हो तो हमसे मिलने आईएगा।\n","टॉम अभी तक नहीं जानता कि यह कैसे करना है।\n","क्योटो में बहुत सारे पुराने मंदिर हैं।\n","मेरा भाई टोक्यो में रहता है।\n","\n","tf.Tensor(\n","[b'Please sit down.' b'If you have time, please drop in on us.'\n"," b\"Tom doesn't know how to do that yet.\"\n"," b'There are many old temples in Kyoto.' b'My brother lives in Tokyo.'], shape=(5,), dtype=string)\n"]}]},{"cell_type":"code","source":["# print(hindi[1])\n","# print(h_text)"],"metadata":{"id":"rf25rMsJKjyj","executionInfo":{"status":"ok","timestamp":1669569515896,"user_tz":-60,"elapsed":10,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":191,"outputs":[]},{"cell_type":"code","source":["example_text2 = tf.constant('हम अच्छे दोस्त हैं।')\n","\n","print(example_text2.numpy().decode())\n","print(tf_lower_and_split_punct2(example_text2).numpy().decode())"],"metadata":{"id":"NTcDHjhgHdF-","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569515896,"user_tz":-60,"elapsed":9,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"58421064-2f4e-4568-8dd2-8c3aedd91be0"},"execution_count":192,"outputs":[{"output_type":"stream","name":"stdout","text":["हम अच्छे दोस्त हैं।\n","[START] हम अच्छे दोस्त हैं। [END]\n"]}]},{"cell_type":"code","source":["max_vocab_size = 10000\n","\n","context_text_processor2 = tf.keras.layers.TextVectorization(\n","    standardize=tf_lower_and_split_punct2,\n","    max_tokens=max_vocab_size,\n","    ragged=True)\n","\n","context_text_processor2.adapt(train_raw2.map(lambda context, target: context))\n","\n","print(context_text_processor2.get_vocabulary()[:100])\n","\n","target_text_processor2 = tf.keras.layers.TextVectorization(\n","    standardize=tf_lower_and_split_punct,\n","    max_tokens=max_vocab_size,\n","    ragged=True)\n","\n","target_text_processor2.adapt(train_raw2.map(lambda context, target: target))\n","print(target_text_processor2.get_vocabulary()[:10])\n","\n","# eng2_context_text_processor =  tf.keras.layers.TextVectorization(\n","#     standardize=tf_lower_and_split_punct2,\n","#     max_tokens=max_vocab_size,\n","#     ragged=True)\n","\n","# eng2_context_text_processor.adapt(trans_data)\n","\n","# print(eng2_context_text_processor.get_vocabulary()[:10])\n","\n","# # eng2_context_text_processor =  tf.keras.layers.TextVectorization(\n","# #     standardize=tf_lower_and_split_punct,\n","# #     max_tokens=max_vocab_size,\n","# #     ragged=True)\n","\n","# # eng2_context_text_processor.adapt(train_raw_h.map(lambda context, target: context))\n","# # print(eng2_context_text_processor.get_vocabulary()[:10])\n","\n","# # h_target_text_processor =  tf.keras.layers.TextVectorization(\n","# #     standardize=tf_lower_and_split_punct2,\n","# #     max_tokens=max_vocab_size,\n","# #     ragged=True)\n","\n","# # h_target_text_processor.adapt(trans_data)\n","\n","# # print(h_target_text_processor.get_vocabulary()[:10])\n","\n","\n","# h2_target_text_processor =  tf.keras.layers.TextVectorization(\n","#     standardize=tf_lower_and_split_punct,\n","#     max_tokens=max_vocab_size,\n","#     ragged=True)\n","# h2_target_text_processor.adapt(train_raw_h.map(lambda context, target: target))\n","# print(h2_target_text_processor.get_vocabulary()[:10])"],"metadata":{"id":"SL4OGYw4S3Ft","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569516368,"user_tz":-60,"elapsed":479,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"70d83551-b3ea-4fce-8cad-0748bd0ae434"},"execution_count":193,"outputs":[{"output_type":"stream","name":"stdout","text":["['', '[UNK]', '[START]', '[END]', 'है।', '?', 'में', 'नहीं', 'वह', 'मैं', 'से', 'है', 'क्या', 'मुझे', 'के', 'को', 'की', 'हैं।', 'हो', 'बहुत', 'का', 'था।', 'उसने', 'यह', 'तुम', 'हूँ।', 'एक', 'पर', 'कि', 'कर', 'लिए', 'मेरे', ',', 'टॉम', 'हैं', 'तुम्हें', 'ने', 'उसे', 'मैंने', 'भी', 'इस', 'अपने', 'रहा', 'मेरी', 'और', 'ही', 'करने', 'हम', 'तो', 'अपनी', 'कोई', 'कल', 'पास', 'पता', 'गया।', 'था', 'काम', 'बात', 'घर', 'उसकी', 'उस', 'समय', 'उसके', 'गया', 'आप', 'करना', 'सकते', 'थी।', 'किया।', 'साथ', 'लगता', 'रही', 'अच्छा', 'मेरा', 'बजे', 'तुम्हारे', 'तक', 'भारत', 'करते', 'सकता', 'रहे', '!', '।', 'यहाँ', 'मुझसे', 'पसंद', 'दिया।', 'चाहता', 'अभी', 'होती', 'मत', 'जो', 'जाने', 'आ', 'हो।', 'हूँ', 'हुआ', 'चाहिए।', 'किताब', 'करता']\n","['', '[UNK]', '[START]', '[END]', '.', 'the', 'i', 'to', 'you', '?']\n"]}]},{"cell_type":"code","source":["example_tokens2 = context_text_processor2(example_context_strings2)\n","example_tokens2[:3, :]"],"metadata":{"id":"-lmIVjx6-j9i","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569516368,"user_tz":-60,"elapsed":19,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"530d85d1-413a-42ba-b482-ed012fb48b2d"},"execution_count":194,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<tf.RaggedTensor [[2, 800, 3], [2, 156, 223, 52, 61, 18, 48, 508, 295, 994, 3],\n"," [2, 33, 88, 76, 7, 373, 28, 23, 154, 65, 4, 3]]>"]},"metadata":{},"execution_count":194}]},{"cell_type":"code","source":["context_vocab2 = np.array(context_text_processor2.get_vocabulary())\n","tokens2 = context_vocab2[example_tokens2[0].numpy()]\n","' '.join(tokens2)"],"metadata":{"id":"CmU0k5p6_hJE","colab":{"base_uri":"https://localhost:8080/","height":298},"executionInfo":{"status":"ok","timestamp":1669569516369,"user_tz":-60,"elapsed":18,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"c31ac816-0068-4507-dfbe-7e44687911f3"},"execution_count":195,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'[START] बैठिए। [END]'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":195}]},{"cell_type":"markdown","source":["## Process dataset"],"metadata":{"id":"i-z86QvL-bt3"}},{"cell_type":"code","source":["def process_text2(context2, target2):\n","  context2 = context_text_processor2(context2).to_tensor()\n","  target2 = target_text_processor2(target2)\n","  targ_in2 = target2[:,:-1].to_tensor()\n","  targ_out2 = target2[:,1:].to_tensor()\n","  return (context2, targ_in2), targ_out2\n","\n","\n","train_ds2 = train_raw2.map(process_text2, tf.data.AUTOTUNE)\n","val_ds2 = val_raw2.map(process_text2, tf.data.AUTOTUNE)"],"metadata":{"id":"xOY8vIOp-bt7","executionInfo":{"status":"ok","timestamp":1669569516873,"user_tz":-60,"elapsed":519,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":196,"outputs":[]},{"cell_type":"code","source":["for (ex_context_tok2, ex_tar_in2), ex_tar_out2 in train_ds2.take(1):\n","  print(ex_context_tok2[0, :10].numpy()) \n","  print()\n","  print(ex_tar_in2[0, :10].numpy()) \n","  print(ex_tar_out2[0, :10].numpy()) "],"metadata":{"id":"8Qs8WLvY-bt7","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569516874,"user_tz":-60,"elapsed":8,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"5c306206-aa5f-444d-8494-00439cfce054"},"execution_count":197,"outputs":[{"output_type":"stream","name":"stdout","text":["[  2  38  37 684   6   7 212   4   3   0]\n","\n","[  2   6 411 197  40 775   4   0   0   0]\n","[  6 411 197  40 775   4   3   0   0   0]\n"]}]},{"cell_type":"code","source":["for (ex_context_tok2, ex_tar_in2), ex_tar_out2 in val_ds2.take(1):\n","  print(ex_context_tok2[0, :10].numpy()) \n","  print()\n","  print(ex_tar_in2[0, :10].numpy()) \n","  print(ex_tar_out2[0, :10].numpy()) "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0HAH0anmks05","executionInfo":{"status":"ok","timestamp":1669569516874,"user_tz":-60,"elapsed":7,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"6087323d-d7bd-463e-a19b-fdd699e3d677"},"execution_count":198,"outputs":[{"output_type":"stream","name":"stdout","text":["[  2   9  35  75 504  14  69 102  65  87]\n","\n","[  2   6  53   7 105   8  46  30 544   4]\n","[  6  53   7 105   8  46  30 544   4   3]\n"]}]},{"cell_type":"markdown","source":["## Training"],"metadata":{"id":"dYj4L2WLt6yx"}},{"cell_type":"code","source":["UNITS = 1024\n","# Encode the input sequence.\n","encoder2 = Encoder(context_text_processor2, UNITS)\n","ex_context2 = encoder2(ex_context_tok2)\n","\n","print(f'Context tokens, shape (batch, s): {ex_context_tok2.shape}')\n","print(f'Encoder output, shape (batch, s, units): {ex_context2.shape}')"],"metadata":{"id":"9_fOaqUsrM08","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569516874,"user_tz":-60,"elapsed":5,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"8906c58b-89f8-471b-92da-233b38bf0029"},"execution_count":199,"outputs":[{"output_type":"stream","name":"stdout","text":["Context tokens, shape (batch, s): (32, 14)\n","Encoder output, shape (batch, s, units): (32, 14, 1024)\n"]}]},{"cell_type":"code","source":["# attention_layer = tf.keras.layers.AdditiveAttention(UNITS)\n","\n","attention_layer = CrossAttention(UNITS)\n","\n","# Attend to the encoded tokens\n","embed2 = tf.keras.layers.Embedding(target_text_processor2.vocabulary_size(),\n","                                  output_dim=UNITS, mask_zero=True, embeddings_regularizer = regularizer, embeddings_initializer=initializer)\n","ex_tar_embed2 = embed2(ex_tar_in2)\n","\n","(x, last_attention_weights) = attention_layer([ex_tar_embed2, ex_context2], return_attention_scores=True)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"sa8M4jMzg2qR","executionInfo":{"status":"ok","timestamp":1669569516874,"user_tz":-60,"elapsed":4,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"3b1058c8-dc02-484e-db14-5b3aa19e7e6b"},"execution_count":200,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/keras/initializers/initializers_v2.py:121: UserWarning: The initializer HeNormal is unseeded and being called multiple times, which will return identical values  each time (even if the initializer is unseeded). Please update your code to provide a seed to the initializer, or avoid using the same initalizer instance more than once.\n","  f\"The initializer {self.__class__.__name__} is unseeded \"\n"]}]},{"cell_type":"code","source":["# attention_layer2 = CrossAttention(UNITS)\n","\n","# # Attend to the encoded tokens\n","# embed2 = tf.keras.layers.Embedding(target_text_processor2.vocabulary_size(),\n","#                                   output_dim=UNITS, mask_zero=True, embeddings_regularizer = regularizer, embeddings_initializer=initializer)\n","# ex_tar_embed2 = embed2(ex_tar_in2)\n","\n","# result2 = attention_layer2(ex_tar_embed2, ex_context2)\n","\n","# print(f'Context sequence, shape (batch, s, units): {ex_context2.shape}')\n","# print(f'Target sequence, shape (batch, t, units): {ex_tar_embed2.shape}')\n","# print(f'Attention result, shape (batch, t, units): {result2.shape}')\n","# print(f'Attention weights, shape (batch, t, s):    {attention_layer2.last_attention_weights.shape}')"],"metadata":{"id":"wRbWYx_crTyz","executionInfo":{"status":"ok","timestamp":1669569516875,"user_tz":-60,"elapsed":4,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":201,"outputs":[]},{"cell_type":"code","source":["# decoder2 = Decoder(target_text_processor2, UNITS)\n","decoder2 = Decoder(target_text_processor2, UNITS, attention=CrossAttention)\n","\n","logits2 = decoder2(ex_context2, ex_tar_in2)"],"metadata":{"id":"8yUiU2K-rhFR","executionInfo":{"status":"ok","timestamp":1669569517356,"user_tz":-60,"elapsed":485,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":202,"outputs":[]},{"cell_type":"code","source":["logits2 = decoder2(ex_context2, ex_tar_in2)\n","\n","print(f'encoder output shape: (batch, s, units) {ex_context2.shape}')\n","print(f'input target tokens shape: (batch, t) {ex_tar_in2.shape}')\n","print(f'logits shape shape: (batch, target_vocabulary_size) {logits2.shape}')"],"metadata":{"id":"UfoD8nNOrkyz","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569517357,"user_tz":-60,"elapsed":11,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"bb3b29f3-9d51-48e5-a75e-deb82f6fc98d"},"execution_count":203,"outputs":[{"output_type":"stream","name":"stdout","text":["encoder output shape: (batch, s, units) (32, 14, 1024)\n","input target tokens shape: (batch, t) (32, 14)\n","logits shape shape: (batch, target_vocabulary_size) (32, 14, 1841)\n"]}]},{"cell_type":"code","source":["# Setup the loop variables.\n","next_token2, done2, state2 = decoder2.get_initial_state(ex_context2)\n","tokens2 = []\n","\n","for n in range(10):\n","  # Run one step.\n","  next_token2, done2, state2 = decoder2.get_next_token(\n","      ex_context2, next_token2, done2, state2, temperature=1.0)\n","  # Add the token to the output.\n","  tokens2.append(next_token2)\n","\n","# Stack all the tokens together.\n","tokens2 = tf.concat(tokens2, axis=-1) # (batch, t)\n","\n","# Convert the tokens back to a a string\n","result2 = decoder2.tokens_to_text(tokens2)\n","result2[:3].numpy()"],"metadata":{"id":"I2UXShIVrw_P","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569517357,"user_tz":-60,"elapsed":8,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"67354865-2599-4ce7-ec0c-6b3ffc3755b1"},"execution_count":204,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([b'really mistake alarm decided getting silky knows parents near newspapers',\n","       b'really mistake alarm decided getting silky italy wherever higher sky',\n","       b'really mistake alarm decided getting silky italy wherever knows fortune'],\n","      dtype=object)"]},"metadata":{},"execution_count":204}]},{"cell_type":"code","source":["# model2 = Translator(UNITS, context_text_processor2, target_text_processor2)\n","model3 = Translator(UNITS, context_text_processor2, target_text_processor2, attention_mechanism=tf.keras.layers.Attention)\n","\n","logits2 = model3((ex_context_tok2, ex_tar_in2))\n","\n","# print(f'Context tokens, shape: (batch, s, units) {ex_context_tok2.shape}')\n","# print(f'Target tokens, shape: (batch, t) {ex_tar_in2.shape}')\n","# print(f'logits, shape: (batch, t, target_vocabulary_size) {logits2.shape}')"],"metadata":{"id":"gGZOif3lsHu8","executionInfo":{"status":"ok","timestamp":1669569517358,"user_tz":-60,"elapsed":5,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":205,"outputs":[]},{"cell_type":"code","source":["model3.compile(optimizer='adam',\n","              loss=masked_loss,\n","              metrics=[masked_acc, masked_loss])"],"metadata":{"id":"i7pLsIsFua2Q","executionInfo":{"status":"ok","timestamp":1669569517358,"user_tz":-60,"elapsed":4,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}}},"execution_count":206,"outputs":[]},{"cell_type":"code","source":["history2 = model3.fit(\n","    train_ds2.repeat(), \n","    epochs=10,\n","    steps_per_epoch = 100,\n","    validation_data=val_ds2,\n","    validation_steps = 20,\n","    callbacks=[\n","        tf.keras.callbacks.EarlyStopping(patience=3)])"],"metadata":{"id":"cpe-7brFARZL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1669569567730,"user_tz":-60,"elapsed":50376,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"ac4c15a6-471e-44ae-bbd3-e004ae0d619f"},"execution_count":207,"outputs":[{"output_type":"stream","name":"stdout","text":["Epoch 1/10\n","100/100 [==============================] - 30s 163ms/step - loss: 5.0670 - masked_acc: 0.2360 - masked_loss: 5.0648 - val_loss: 4.6315 - val_masked_acc: 0.2883 - val_masked_loss: 4.6315\n","Epoch 2/10\n","100/100 [==============================] - 6s 56ms/step - loss: 3.9271 - masked_acc: 0.3308 - masked_loss: 3.9309 - val_loss: 4.3662 - val_masked_acc: 0.3323 - val_masked_loss: 4.3662\n","Epoch 3/10\n","100/100 [==============================] - 4s 44ms/step - loss: 2.9759 - masked_acc: 0.4148 - masked_loss: 2.9818 - val_loss: 4.5669 - val_masked_acc: 0.3457 - val_masked_loss: 4.5669\n","Epoch 4/10\n","100/100 [==============================] - 5s 49ms/step - loss: 2.1112 - masked_acc: 0.4967 - masked_loss: 2.1165 - val_loss: 4.5851 - val_masked_acc: 0.3599 - val_masked_loss: 4.5851\n","Epoch 5/10\n","100/100 [==============================] - 5s 48ms/step - loss: 1.4194 - masked_acc: 0.5894 - masked_loss: 1.4198 - val_loss: 4.7793 - val_masked_acc: 0.3694 - val_masked_loss: 4.7793\n"]}]},{"cell_type":"code","source":["plt.plot(history2.history['loss'], label='loss')\n","plt.plot(history2.history['val_loss'], label='val_loss')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"id":"D3nnGoo4sUPt","colab":{"base_uri":"https://localhost:8080/","height":296},"executionInfo":{"status":"ok","timestamp":1669569567731,"user_tz":-60,"elapsed":19,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"1a828a8d-12f0-4bcd-9f0f-97da02a2823d"},"execution_count":208,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb35fa08610>"]},"metadata":{},"execution_count":208},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU9b3/8dc3eyCEHRISIOwIBJAEVJBFQNyqosLFfZeKVtyurVV/tfV626utrbVV0aoX8KKCKJUiLqgYQCiQIKsgJKwJWxIgAULW+f7+mAFDSEJCZnImk/fz8chjtjPnfOaQefPN93zP9xhrLSIiEniCnC5ARER8QwEvIhKgFPAiIgFKAS8iEqAU8CIiASrE6QLKa9OmjU1ISHC6DBGRBiMtLS3HWtu2stf8KuATEhJITU11ugwRkQbDGLOrqtfURSMiEqAU8CIiAUoBLyISoBTwIiIBKiACvrTM5XQJIiJ+p8EHfEmZi+teW87/fLaFE8VlTpcjIuI3fBrwxpidxpgNxpi1xhifjH8sKnVxXmwzpqVkcOlfUli85aAvNiMi0uDURwv+EmvtQGttsi9WHhUewosTBjB78oVEhAZz1/TVPDArjQP5hb7YnIhIg9Hgu2hOuqBraxZOHc4Tl/Xi680HGfNSCtO/20GZS/Pdi0jj5OuAt8CXxpg0Y8zkyhYwxkw2xqQaY1Kzs7PrtLGwkCAevKQ7Xz46gkGdW/Lbf/3A+Fe/Y0NmXp3WKyLSEBlfXtHJGBNnrc0yxrQDFgEPWWuXVLV8cnKy9dZUBdZaFqzfx3MLfiD3WBG3X5TA4+N60iwi1CvrFxHxB8aYtKq6wH3agrfWZnluDwLzgCG+3F55xhiuHtCBrx8fya0XdmbGip2M/XMKn23Yhy5TKCKNgc8C3hjT1BjT7OR9YByw0Vfbq0p0RCjPXduPeQ8Mo3XTcKbMWsPd01ez51BBfZciIlKvfNmCbw8sM8asA1YBn1prP/fh9qo1sGML5v9iGM9cdR4rdxzi0r+k8Pq3GZToJCkRCVA+7YOvLW/2wVdn75ET/Hb+Jr784QC92jfj99f3I6lzK59vV0TE2xzrg/dXHVpE8ubtyfzj9mSOFpZww+sr+PXH6zlSUOx0aSIiXtMoA/6kS/u0Z9FjI5k8oitzUjMZ81IK877P1EFYEQkIjbKLpjI/7M3nqXkbWLvnCEO7teb58f3o2jbKkVpEJICUlcDR/XB0H+TvrXC7D47uhaBQ+MWqc1p9dV00fnXJPif16RDNx1OG8t6q3bzw+RYuf3kpU0Z1Y8qobkSEBjtdnoj4G2uhKP+nkD7tdh/kZ7nvH8/Gfc5nOcFh0CwWojtA7EBo2dknJaoFX4mDRwt5fsFm5q/bS9c2TXl+fD+Gdm/jdFkiUl/KSuH4QXdA52dV3fouOX7meyNbQrMOEO0J8JP3y982aQXGeKXU6lrwCvhqLNmazf/7ZCO7cgu47vw4nr7qPNpEhTtdlojURdHRalrdnvA+dgBshSHUQaGeVnfsT63v0249z4dG1uvHUcDXQWFJGa8tTuf1lAwiQ4N58orzuHFwR4KCvPO/r4h4iavM3R1SVT93vifIi4+e+d6I5me2tCu2vpu0hiD/G5eigPeC9IPHeHreBlbuOERS55b893X96B0T7XRZIo1DcUGF0M46s/V9dD/YChf9CQqBqJhqWt2e27AmznwuLwj8gH/ncgiPhtbdoFVXz203aB4PQd47QGqt5aM1Wfz3pz9wtLCUe4Z34eExPWgSpmPVIufE5YKCnGpa3Z7bwkpmhA2PLtc10uH0ED/Z+m7axqsZ4I8CexSNqwyi2kFuBuxcCiXl5pgJDoOWXc4M/tbd3P/4tfxzyxjDhKR4xvRuxx8+28wbKdtZsG4f/zW+L6N7t/fyBxNp4EoKK+/fPu12P7hKTn+fCfqp1d26G3QZXnmrO1zDmM8mMFrwJ1nr/sXJzYBD2+FQRrn726G03FWeQiLcoX/y57Twj63REe5VOw7x1LwNpB88xhX9Ynj26r7ENI849/rF/1gLJw67D8zZMvdjV5n7AJz13J567Cr3uLrXKr7Xuh9X91ql762shhrUWKP6a7Kdk69V8npRvnu/VRQWVUmru8Jt03YQ3PDbnvUl8LtoasLlcrcmcjNOD/7cDDi8A8rKTVMQ2qTy4G/Vzf3XQrnwLy518Y+l23nl622EBBkeH9eLO4YmEKyDsP7vVPdAlucA3N7K75f6+eUfTZDnJ9jdHXHyvjEVHgeVexxUyWtBlSxb3WtVrScYwppW3uqO0HErb1PAn42rDPIyzwz+QxlweCe4Sn9aNqwZtOpyRvBnBsXy9Bf7SNmWQ7+4aH5/XSL941vU/2cRN1eZe6jbaaFdMbz3ndk9EBTiaUmW/4lzB9NpAVpdSFYMwboEaHD1rxnjtfHU0jAp4OuirBTydkNu+S4fz+2R3acdtbfh0eRFduLfeS3YWtKO+G79uGzkMJrG9HSf2CDeUVoMx/ZX3eI+2bdbcURFcPhPgV0+vMvfb9rWL4fCiVRFAe8rZSVweNcZwe/KzYC8PQSVOz3ZRrbElO/qKX/gN6K5gx/Cz5w6MFdNl8mxg5xx6ndoU2geV3WAe/nsQRF/EdijaJwUHApturt/ygkCKC1iy+b1zPkihaDD2xkWdYSLTB4RO7+D9bNPX0+TNuWCv+vp/wGEN6u3j+NzRccqD+1T45r3QkHume+LaP5TUMckVh7g4dEKb5EK1IL3sdIyFzNW7OLPX/5ImbVMHdOD+y6MJTR/d+UHfI/uPX0FUe0rCX5P+PvLyRnWuscpn9HqLh/ke6GokrHMTVpX020Sp+FwImehLho/sC/PfRWpLzYdoGf7KH5/XSLJCZX0yxcfh0M7KnT7ePr/jx04fdlmHSof49+yC4R6abimtVBwqJoDlZ77Z0y6ZNz/OVUW2uVPA/dWnSKNlALej3z1wwGenb+JrCMnuHFwR568ojctmoTV7M1FR08f4VP+wG9BTrkFjfss3lZdTm/1t+4GLRMgxDNhmsvlmbvjLMMEy4pOr8MEn37GYKV93jHuLiwR8SkFvJ8pKC7l5a+28fayHTSPDOXpK8/j+kFxmLr0IRfmnTnE8+Rt+RNOTJA7/C3u7qDyQ0Ch3DzVVXSZRHdwnwsQ4Kd/izQUCng/tXmf+ypS3+8+wkVdW/P8df3o5ourSBUcOj34D213t8IrC3A/nTFPRCqngPdjLpfl/dW7eeGzLRSWuLh/VDce0FWkRKSGqgt4NdUcFhRkuOWCznz9+CiuSIzhla+3ccVfl7JsW87Z3ywiUg0FvJ9o2yycv954Pu/eMwRrLbe+vZJHPvie7KNFZ3+ziEglFPB+ZniPtnz+yAimjunBwg37GfPSt7y3cjcul/90pYlIw6CA90MRocE8dmlPFj48nD4donlq3gYmTFvOlv35TpcmIg2IAt6PdW8Xxfv3XchLEwewM7eAq15Zxh8WbqaguPTsbxaRRk8B7+eMMdyQFM/Xj41kwqB43liynUv/vISvNx84+5tFpFFTwDcQLZuG8cKE/sz5+UU0CQvmnhmp3P9uGvvyTjhdmoj4KQV8AzOkSys+nTqcJy7rxeIfDzL2pRTeXraD0jKX06WJiJ9RwDdAYSFBPHhJdxY9OpLBXVrxXwt+YPxr37FuzxGnSxMRP6KAb8A6tW7C/945mFdvHsTB/CLGv/Ydz36ykfzCkrO/WUQCns8D3hgTbIz53hizwNfbaoyMMVzVP5avHh/J7Rd2Zua/dzH2pRQ+Xb8Pf5qGQkTqX3204B8GNtfDdhq16IhQfndtPz55cBjtosN58L013DV9NbtzC5wuTUQc4tOAN8bEA1cBb/lyO/KT/vEt+OcDw/jNz/qweschLv1LCq8uTqe4VAdhRRobX7fgXwZ+CVSZLsaYycaYVGNManZ2to/LaRxCgoO4++IufPX4SC7p1Y4/fvEjP/vbUlbvPOR0aSJSj3wW8MaYnwEHrbVp1S1nrX3TWptsrU1u27atr8pplGKbRzLttiTeviOZ40VlTJy2gl/NXc/h48VOlyYi9cCXLfhhwDXGmJ3AB8BoY8z/+XB7UoUx57Vn0WMj+PnIrsxdk8mYP6fw3srdlGjsvEhAq5cLfhhjRgH/aa39WXXLNcYLftS3LfvzeWbeRlJ3HaZz6yY8MrYH1wyIIzioDpcLFBHH6IIfckrvmGg+vP8i3r4jmSZhITw6ex2Xv7yEzzZoWKVIoNEl+xoxl8vy2cb9/HnRj2RkH6dfXDSPX9qLUb3a1u0C4CJSb9SCl0oFBblPkvry0ZG8NHEAeSdKuGv6aiZMW8HyDF0yUKShUwteTikudfFh2h7+9nU6+/MLGda9NY+P68WgTi2dLk1EqlBdC14BL2coLClj1srdvLY4ndzjxYzp3Y7HxvWkb4fmTpcmIhUo4OWcHC8qZfrynbyRkkF+YSlXJcby6KU96d4uyunSRMRDAS91kneihLeWbuedZTs4UVLG+PPjeGRMTzq1buJ0aSKNngJevCL3WBHTUjKYuWIXZS7LfwzuyEOjuxPbPNLp0kQaLQW8eNWB/EL+/k06H6zejTGG2y7szJRR3WgTFe50aSKNjgJefGLPoQJe+XobH63JJCI0mLuGJTB5eDeaNwl1ujSRRkMBLz6VkX2Ml7/axr/W7aVZRAiTh3flrou7EBUe4nRpIgFPAS/1YvO+fF76citfbT5Aq6ZhTBnZjdsu6kxEaLDTpYkELAW81Ku1e47w0pc/snRbDu2ahfPQ6O5MGtyJsBCdOC3ibQp4ccTK7bn86csfWb3zMHEtInl4bA+uPz+OkGAFvYi3aC4accQFXVsz5+cXMePuIbSOCuOXc9cz7i9LmL9uLy6X/zQsRAKVAl58yhjDyJ5t+eTBYbxxWxKhwUFMff97rnxlKV9u2q8pikV8SAEv9cIYw2V9Y1j48HD+euNAikpdTH43jfGvfseSrdkKehEfUMBLvQoOMlw7MI5Fj47gxRv6k3OsmNvfWcWkN/+ti4KLeJkOsoqjikrL+GDVHv6+OJ3so0WM6NmW/xzXk/7xLZwuTaRB0Cga8XsnisuYuWIn01IyOFxQwrg+7Xl8XC96xTRzujQRv6aAlwbjaGEJ7yzbyVtLt3OsuJSr+3fg0Ut70qVNU6dLE/FLCnhpcI4UFPPGku1M/24nxWUubhgUx9QxPYhvqSmKRcpTwEuDdfBoIa9/m8Gsf+/GYrlpSCd+cUl32kVHOF2aiF9QwEuDt/fICf72TTofpu4hOMhwx9AE7h/ZjVZNw5wuTcRRCngJGLtyj/PXr7Yxb20WTcNCuPviLtw7vAvREZqiWBonBbwEnG0HjvKXr7aycMN+mkeG8vORXblzaAJNwjRFsTQuCngJWBuz8njpyx9Z/GM2baLCeGBUd26+oJOmKJZGQwEvAS9t1yH+9MVWVmzPJbZ5BA+N7sHE5HhCNXOlBDgFvDQay9Nz+OOXP/L97iN0atWER8b24NqBcQQHGadLE/EJTRcsjcbQ7m34eMpQ3rkzmajwEB6bs47LXl7Cwg37NEWxNDoKeAk4xhhG927Pgocu5rVbBgHwwKw1XP33ZXyz5YBmrpRGQwEvASsoyHBlYixfPDKClyYOIL+whLunp3LD68tZnp7jdHkiPqc+eGk0SspczEndw9++Tmd/fiFDu7Xm8XG9SOrc0unSRM6ZDrKKlFNYUsaslbt5bXE6uceLGd27HY9d2pN+cc2dLk2k1rwS8MaYoUACcOpMEmvtzGqWjwCWAOGe98y11j5b3TYU8FKfjheVMn35Tt5IySC/sJQrE2N47NKedG+nKYql4ahzwBtj3gW6AWuBMs/T1lo7tZr3GKCptfaYMSYUWAY8bK39d1XvUcCLE/JOlPD20u28vWwHJ0rKGD8wjkfG9qRTa81cKf6vuoCv6XndyUAfW4v+HM+yxzwPQz0//tMfJOLRPDKUx8b14o6hCbyxZDszlu9k/rq9TEzuyNQx3YltHul0iSLnpKajaDYCMbVduTEm2BizFjgILLLWrqztOkTqS+uocJ668jyW/PISbr6gE3PT9jDyj9/yu39tIvtokdPlidRaTbtoFgMDgVXAqd90a+01NdqIMS2AecBD1tqNFV6bDEwG6NSpU9KuXbtqXLyIL+05VMDfvtnG3LRMQoOD+I/kjkwe0ZWOrdR1I/7DG33wIyt73lqbUosifgMUWGv/VNUy6oMXf7Qj5zjTvs3g4+8zcVm4un8s94/qRu+YaKdLE/HaKJrOQA9r7VfGmCZAsLX2aDXLtwVKrLVHjDGRwJfAC9baBVW9RwEv/mx/XiFvLd3Oe6t2U1Bcxpje7Xjgkm4kdW7ldGnSiHmjBX8f7m6UVtbabsaYHsA0a+2Yat7TH5gBBOPu659jrX2uuu0o4KUhOFJQzIzlu5i+fAeHC0oYktCKKZd0Y1TPtrgHj4nUH28E/FpgCLDSWnu+57kN1tpEbxaqgJeGpKC4lA9W7eEfS7ezL6+Q82KjmTKqG1f2iyFE0xRLPfHGbJJF1tricisMQUMepZFr4rlkYMoTl/DHCf0pLi1j6vvfM/qlFGat3EVhSdnZVyLiQzUN+BRjzFNApDHmUuBD4F++K0uk4QgLCWJickcWPTqSabcm0bJJKE/P28jwFxczLSWDo4UlTpcojVRNu2iCgHuAcYABvrDW/sPbxaiLRgKBtZYVGbm89m0Gy9JzaBYRwu0XdeauYV1oExXudHkSYLzRB/+ctfY35R4HAzOttbd4r0wFvASe9ZlHeP3bDD7ftJ+w4CAmDe7IfcM1ll68xxt98B2NMb/2rCwM+AjY5qX6RAJW//gWvH5rEl89NpLxA+N4f9VuRv3pWx6dvZYf91c5yljEK2ragjfALGADcAnwmbX2L94uRi14CXT78k7w9tIdp8bSjz2vHVNGaSy9nLtz7qIxxgwq9zAUeAP4DngbwFq7xot1KuCl0Th8vJiZK8qNpe/SiimjNJZeaq8uAb+4mvVaa+3ouhZXngJeGhuNpZe60hWdRPxccamLT9ZmMS0lg4zs43Ru3YTJI7pyw6B4IkKDnS5P/Jg3RtE0B54FRnieSgGes9bmea1KFPAiLpflyx8O8Pq36azLzKNts3DuvbgLN1/QiWYRoU6XJ37IGwH/Ee454Wd4nroNGGCtvd5rVaKAFznJWsvyjFxe94ylj44I4faLErhzWILG0stpvDIXjbV24NmeqysFvMiZ1u05wrQUjaWXynnjkn0njDEXW2uXeVY4DDjhrQJFpGoDOrrH0mdkH+ONlAzeX7WbWSt3c82ADtw/shu9YnSRcKlcTVvwA4CZQHPPU4eBO6y1671ZjFrwIme3L+8Eby3dwfunjaXvTlLnlk6XJg7wRhdNF2vtDmNMNIC1Nv/kc94sVAEvUnOHjxczY8VOpi/fyZGCEi7wjKUfqbH0jYo3An6NtXZQhefSrLVJXqoRUMCLnIuC4lLeX7WHtzxj6fucHEufGEtwkII+0NXlRKfeQF/gReCJci9FA09Ya/t6s1AFvMi5Ky518U/PWPrtnrH0Px/RjRuS4ggP0Vj6QFWXgL8WGA9cA8wv99JR4ANr7XJvFqqAF6k791j6/bz2bQbrM/No1yycezSWPmDVJeBvwn2x7J7W2hU+qu8UBbyI92gsfeNQl2GSnXBfvSnUGPM18BmwyvrT/AYiUiljDMO6t2FY9zanxtK/+m06by3bzqTkjtw3oivxLTWWPpDV9CBrM2AscDnui29vBj7HfWWnA94qRi14Ed9KP3iMN5dkMO/7LFwWrh3QgftHdaNne42lb6i8PtmYMaYPcAUwzlp7WR3rO0UBL1I/9h45wdvLdvDeyt2cKClj7HnteeCSbgzqpLH0Dc05X9HJGHNrufvDTt631v4AFHkz3EWk/nRoEcn/+1kflj85mkfG9iB11yGuf205k95YQcrWbNQLGxjOdpD11Pj3imPhKxsbX1dqwYs443hRKR+s/mksfd8O7rH0V/TTWHp/V5drspoq7lf2WEQaqKbhIdxzcRdSnriEFyf050RJGb9473vGvPQt76/aTVFpmdMlyjk4W8DbKu5X9lhEGriwkCD+I7kjix4dybRbBxEdGcqvP97A8BcW8+aSDI4VlTpdotTC2bpoCoB03K31bp77eB53tdY29WYx6qIR8S8nx9K/9m0636XnEh0Rwh1DE7hzaAKtNZbeL9TlRKceQHtgT4WXOgL7rbXpZ77r3CngRfxX+Xnpw0OCuHFwJ+4d3kVj6R1Wl4BfAPzaWruhwvOJwO+ttVd7s1AFvIj/Kz+W3lq4ZqB7XnqNpXdGXQJ+tbV2cBWvbbDWJnqpRkABL9KQ7D3y07z0J0rKuLRPe6aM0lj6+laXgN9mre1RxWvp1truXqoRUMCLNESHjxczfflOZqxwz0t/YddW3De8KyN7tiUk+GzjOKSu6hLw7wPfWGv/UeH5e4FLrbWTvFmoAl6k4TpeVMr7q3bz1tId7M8vpF2zcK4bFMfEpHi6t1P3ja/UJeDbA/OAYiDN83QyEAZcZ63d781CFfAiDV9xqYvFPx5kblomi7ccpNRlGdixBROS4rl6QAeaR2rKYm/yxhWdLgH6eR5ustZ+U4P3dMR9Hdf2uMfMv2mt/Wt171HAiwSWnGNF/PP7LOamZbJl/1HCQoK4rG8ME5Liubh7G50l6wVen2yshhuNBWKttWs8s1GmAeM989hUSgEvEpistWzam8+HqXv4ZN1ejhSUEBMdwfWD4piQFE/XtlFOl9hgORLwlRTxCfB3a+2iqpZRwIsEvqLSMr7ZfJAP0zJJ2ZpNmcuS1LklE5Liuap/LNG66lStOB7wxpgEYAnQz1qbX+G1ycBkgE6dOiXt2rXL5/WIiH84mF/IvO+z+DAtk/SDx4gIDeLyvjFMTO7IRV1bE6QunLNyNOCNMVFACvDf1tqPq1tWLXiRxslay7rMPOam7WH+2r3kF5YS1yLyVBdO59ZenRUloDgW8MaYUGAB7is//flsyyvgRaSwpIxFPxxgblomS7dl47IwJKEVE5LjuTIxlqjws11ptHFx6iCrAWYAh6y1j9TkPQp4ESlvf14hH3+fydy0TLZnHycyNJgrEmOYmNSRC7q0UhcOzgX8xcBSYAPg8jz9lLV2YVXvUcCLSGWstazZfYS5aZksWLeXo0WlxLeM5IZB8UxIiqdjq8Y74ZnjB1lrSgEvImdzoriML3/Yz9y0TJal52AtXNi1FROTOnJFYgxNwhpXF44CXkQCUtaRE8xb4+7C2ZlbQNOwYK5MjGVickcGJ7TE3VMc2BTwIhLQrLWk7jrMh6l7+HT9Po4Xl9G5dRMmDIrn+qR44lpEOl2izyjgRaTRKCgu5fON+/kwNZMV23MxBoZ1a8OEpHgu6xtDZFiw0yV6lQJeRBqlPYcK+GhNJh+tyWTPoRM0Cw/hZwNimZAUz6BOgdGFo4AXkUbN5bKs2nmID1MzWbhhHydKyujapik3JMVz/aA4Yps33C4cBbyIiMexolIWbtjH3LRMVu04RJCBi3u0ZUJSPOP6tCcitGF14SjgRUQqsSv3OB+lZfLRmiyyjpwgOiKEqwd0YEJSPAM7tmgQXTgKeBGRarhclhXbc5mblslnG/dRWOKie7soJiTFc/35cbSLjnC6xCop4EVEaii/sISF6/fxYVomabsOE2RgZM+2TEjqyNg+7QgP8a8uHAW8iMg52J59zD0KJy2L/fmFNI8M5dqB7i6cxLjmftGFo4AXEamDMpflu/Qc5qZl8vmm/RSXuujVvhkTkuIZf34cbZuFO1abAl5ExEvyTpSwYP1ePkzNZO2eIwQHGS7p5e7CGd27HWEhQfVajwJeRMQH0g8e5cO0TOatyeLg0SJaNQ071YXTt0PzeqlBAS8i4kOlZS6WpucwNzWTRT8coLjMxXmx0UxMiufagR1oHeW7LhwFvIhIPTlSUMz8dXuZm5bJ+sw8QoIMo3u3Y2JyR0b1aktosHe7cBTwIiIO2LI/n4/SMpn3fRY5x4ppExXG+IFxTEiOp3dMtFe2oYAXEXFQSZmLlB+zmZuWyddbDlBSZukXF83EpI5cM6ADLZuGnfO6FfAiIn7i0PFiPlmbxYepmfywL5+w4CDG9mnHy5POP6cRONUFfOO6tpWIiMNaNQ3jrmFduGtYFzbtzeOjtCz2HC7wyfBKBbyIiEP6dmju0+GU9TsiX0RE6o0CXkQkQCngRUQClAJeRCRAKeBFRAKUAl5EJEAp4EVEApQCXkQkQCngRUQClAJeRCRAKeBFRAKUAl5EJED5LOCNMe8YYw4aYzb6ahsiIlI1X7bgpwOX+3D9IiJSDZ8FvLV2CXDIV+sXEZHqOd4Hb4yZbIxJNcakZmdnO12OiEjAcDzgrbVvWmuTrbXJbdu2dbocEZGA4XjAi4iIb/j9JftKSkrIzMyksLDQ6VL8WkREBPHx8YSGhjpdioj4CZ8FvDHmfWAU0MYYkwk8a619u7bryczMpFmzZiQkJGCM8XaZAcFaS25uLpmZmXTp0sXpckTET/gs4K21N3ljPYWFhQr3szDG0Lp1a3SQWkTKaxB98Ar3s9M+EpGKGkTAi4hI7SngayAqKsrpEkREak0BLyISoPx+mGR5v/vXJn7Ym+/VdfbpEM2zV/et0bLWWn75y1/y2WefYYzhmWeeYdKkSezbt49JkyaRn59PaWkpr7/+OkOHDuWee+4hNTUVYwx33303jz76qFdrFxGpToMKeKd9/PHHrF27lnXr1pGTk8PgwYMZMWIE7733HpdddhlPP/00ZWVlFBQUsHbtWrKysti40T2Z5pEjRxyuXkQamwYV8DVtafvKsmXLuOmmmwgODqZ9+/aMHDmS1atXM3jwYO6++25KSkoYP348AwcOpGvXrmzfvp2HHnqIq666inHjxjlau4g0PuqD94IRI0awZMkS4uLiuPPOO5k5cyYtW7Zk3bp1jBo1imnTpnHvvfc6XaaINDIK+FoYPnw4s2fPpqysjOzsbJYsWcKQIUPYtWsX7du357777uPee+9lzZo15OTk4HK5uOGGG3j++edZs2aN0+WLSCPToLponHbdddexYsUKBgwYgA04aSMAAApUSURBVDGGF198kZiYGGbMmMEf//hHQkNDiYqKYubMmWRlZXHXXXfhcrkA+MMf/uBw9SLS2BhrrdM1nJKcnGxTU1NPe27z5s2cd955DlXUsGhfiTQ+xpg0a21yZa+pi0ZEJEAp4EVEApQCXkQkQCngRUQClAJeRCRAKeBFRAKUAl5EJEAp4L2surnjd+7cSb9+/eqxGhFpzBrWmayfPQn7N3h3nTGJcMX/eHedIiJ+QC34s3jyySd59dVXTz3+7W9/y/PPP8+YMWMYNGgQiYmJfPLJJ7Veb2FhIXfddReJiYmcf/75LF68GIBNmzYxZMgQBg4cSP/+/dm2bRvHjx/nqquuYsCAAfTr14/Zs2d77fOJSOBqWC14B1rakyZN4pFHHuHBBx8EYM6cOXzxxRdMnTqV6OhocnJyuPDCC7nmmmtqdeHrV199FWMMGzZsYMuWLYwbN46tW7cybdo0Hn74YW655RaKi4spKytj4cKFdOjQgU8//RSAvLw8n3xWEQksasGfxfnnn8/BgwfZu3cv69ato2XLlsTExPDUU0/Rv39/xo4dS1ZWFgcOHKjVepctW8att94KQO/evencuTNbt27loosu4ve//z0vvPACu3btIjIyksTERBYtWsSvfvUrli5dSvPmzX3xUUUkwCjga2DixInMnTuX2bNnM2nSJGbNmkV2djZpaWmsXbuW9u3bU1hY6JVt3XzzzcyfP5/IyEiuvPJKvvnmG3r27MmaNWtITEzkmWee4bnnnvPKtkQksDWsLhqHTJo0ifvuu4+cnBxSUlKYM2cO7dq1IzQ0lMWLF7Nr165ar3P48OHMmjWL0aNHs3XrVnbv3k2vXr3Yvn07Xbt2ZerUqezevZv169fTu3dvWrVqxa233kqLFi146623fPApRSTQKOBroG/fvhw9epS4uDhiY2O55ZZbuPrqq0lMTCQ5OZnevXvXep0PPPAAU6ZMITExkZCQEKZPn054eDhz5szh3XffJTQ09FRX0OrVq3niiScICgoiNDSU119/3QefUkQCjeaDDyDaVyKNj+aDFxFphNRF4wMbNmzgtttuO+258PBwVq5c6VBFItIYNYiAt9bWaoy50xITE1m7dm29btOfutpExD/4fRdNREQEubm5CrBqWGvJzc0lIiLC6VJExI/4fQs+Pj6ezMxMsrOznS7Fr0VERBAfH+90GSLiR/w+4ENDQ+nSpYvTZYiINDg+7aIxxlxujPnRGJNujHnSl9sSEZHT+SzgjTHBwKvAFUAf4CZjTB9fbU9ERE7nyxb8ECDdWrvdWlsMfABc68PtiYhIOb7sg48D9pR7nAlcUHEhY8xkYLLn4TFjzI/nuL02QM45vteXVFftqK7aUV21E4h1da7qBccPslpr3wTerOt6jDGpVZ2u6yTVVTuqq3ZUV+00trp82UWTBXQs9zje85yIiNQDXwb8aqCHMaaLMSYMuBGY78PtiYhIOT7rorHWlhpjfgF8AQQD71hrN/lqe3ihm8dHVFftqK7aUV2106jq8qvpgkVExHv8fi4aERE5Nwp4EZEA1eAC/mzTHxhjwo0xsz2vrzTGJPhJXXcaY7KNMWs9P/fWQ03vGGMOGmM2VvG6Mca84ql5vTFmkK9rqmFdo4wxeeX21W/qqa6OxpjFxpgfjDGbjDEPV7JMve+zGtZV7/vMGBNhjFlljFnnqet3lSxT79/HGtZV79/HctsONsZ8b4xZUMlr3t1f1toG84P7YG0G0BUIA9YBfSos8wAwzXP/RmC2n9R1J/D3et5fI4BBwMYqXr8S+AwwwIXASj+paxSwwIHfr1hgkOd+M2BrJf+O9b7PalhXve8zzz6I8twPBVYCF1ZYxonvY03qqvfvY7ltPwa8V9m/l7f3V0Nrwddk+oNrgRme+3OBMcb3Vwvxy2kZrLVLgEPVLHItMNO6/RtoYYyJ9YO6HGGt3WetXeO5fxTYjPuM7PLqfZ/VsK5659kHxzwPQz0/FUdt1Pv3sYZ1OcIYEw9cBbxVxSJe3V8NLeArm/6g4i/6qWWstaVAHtDaD+oCuMHzZ/1cY0zHSl6vbzWt2wkXef7E/swY07e+N+750/h83K2/8hzdZ9XUBQ7sM093w1rgILDIWlvl/qrH72NN6gJnvo8vA78EXFW87tX91dACviH7F5Bgre0PLOKn/6XlTGuAztbaAcDfgH/W58aNMVHAR8Aj1tr8+tx2dc5SlyP7zFpbZq0diPtM9SHGmH71sd2zqUFd9f59NMb8DDhorU3z9bZOamgBX5PpD04tY4wJAZoDuU7XZa3NtdYWeR6+BST5uKaa8MvpJKy1+Sf/xLbWLgRCjTFt6mPbxphQ3CE6y1r7cSWLOLLPzlaXk/vMs80jwGLg8govOfF9PGtdDn0fhwHXGGN24u7GHW2M+b8Ky3h1fzW0gK/J9AfzgTs89ycA31jPEQsn66rQT3sN7n5Up80HbveMDLkQyLPW7nO6KGNMzMl+R2PMENy/pz4PBc823wY2W2v/XMVi9b7PalKXE/vMGNPWGNPCcz8SuBTYUmGxev8+1qQuJ76P1tpfW2vjrbUJuDPiG2vtrRUW8+r+cnw2ydqwVUx/YIx5Dki11s7H/UV41xiTjvtA3o1+UtdUY8w1QKmnrjt9XZcx5n3coyvaGGMygWdxH3DCWjsNWIh7VEg6UADc5euaaljXBGCKMaYUOAHcWA//SYO7hXUbsMHTfwvwFNCpXG1O7LOa1OXEPosFZhj3xX2CgDnW2gVOfx9rWFe9fx+r4sv9pakKREQCVEProhERkRpSwIuIBCgFvIhIgFLAi4gEKAW8iEiAUsBLQDLGlJWbKXCtqWSGzzqsO8FUMRNmFcs3NcZ85bm/zHMCi4jP6RdNAtUJz6nq/uAiYIUxpiVw3DPHiIjPqQUvjYoxZqcx5kVjzAbjnjO8u+f5BGPMN57Jp742xnTyPN/eGDPPM4nXOmPMUM+qgo0x/zDu+ca/9JwxWXFb3TwnJv0fcDOQBgzw/EXRrp4+sjRiCngJVJEVumgmlXstz1qbCPwd9+x+4J6ga4Zn8qlZwCue518BUjyTeA0CTl44vgfwqrW2L3AEuKFiAdbaDM9fEWm4p5SeAdxjrR1orT3o1U8rUgmdySoByRhzzFobVcnzO4HR1trtngm89ltrWxtjcoBYa22J5/l91to2xphsIL7cxFQnp+xdZK3t4Xn8KyDUWvt8FbWsttYONsZ8BDxsrc308scVqZRa8NIY2Sru10ZRuftlVHI8yxgzzXMwtoenq+ZyYIEx5tFz3KZIrSjgpTGaVO52hef+cn6a2OkWYKnn/tfAFDh1EYnmNd2ItfZ+4HfAfwHjgU893TN/qVv5IjWjUTQSqCLLzbwI8Lm19uRQyZbGmPW4W+E3eZ57CPhfY8wTQDY/zRL5MPCmMeYe3C31KUBtpgceCcwEhgMp5/RJRM6R+uClUfH0wSdba3OcrkXE19RFIyISoNSCFxEJUGrBi4gEKAW8iEiAUsCLiAQoBbyISIBSwIuIBKj/DyDlQZjBxpsvAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["plt.plot(history2.history['masked_acc'], label='accuracy')\n","plt.plot(history2.history['val_masked_acc'], label='val_accuracy')\n","plt.ylim([0, max(plt.ylim())])\n","plt.xlabel('Epoch #')\n","plt.ylabel('CE/token')\n","plt.legend()"],"metadata":{"id":"gmrJA2hYsdNP","colab":{"base_uri":"https://localhost:8080/","height":298},"executionInfo":{"status":"ok","timestamp":1669569568373,"user_tz":-60,"elapsed":651,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"32a36aae-97b1-4107-8d21-d3c7e512eebf"},"execution_count":209,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<matplotlib.legend.Legend at 0x7fb36c200fd0>"]},"metadata":{},"execution_count":209},{"output_type":"display_data","data":{"text/plain":["<Figure size 432x288 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAYIAAAEICAYAAABS0fM3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwV9b3/8deHJBD2NWxZCCjKHpYAIlYpSosb2Coieq3idr2tteqv7VXrVWttb9drNx+tSLFSF0QsXrSo1wWrIAgBkV2lBEhYQ4BAgJDt8/vjHCDEhCSQyUly3s/H4zxyZuZ7Zj5n4Hw/M9+Z+X7N3RERkejVJNIBiIhIZCkRiIhEOSUCEZEop0QgIhLllAhERKKcEoGISJQLNBGY2Xgz+8zMNprZ/ZWUudbM1pnZWjN7Ich4RETkyyyo5wjMLAb4HBgHZAPLgCnuvq5Mmd7AbGCsu+8zs87uvvtU6+3UqZOnpqYGErOISGO1fPnyPe6eUNGy2AC3OwLY6O6bAMxsFjARWFemzO3Ak+6+D6CqJACQmppKRkZGAOGKiDReZralsmVBNg0lAlllprPD88o6BzjHzBaZ2RIzGx9gPCIiUoEgzwiqu/3ewBggCfjAzAa6+/6yhczsDuAOgJSUlLqOUUSkUQvyjGAbkFxmOik8r6xsYJ67F7l7JqFrCr3Lr8jdp7l7urunJyRU2MQlIiKnKcgzgmVAbzPrSSgBXAdcX67Mq8AU4Bkz60SoqWhTTTdUVFREdnY2BQUFZxiy1Ib4+HiSkpKIi4uLdCgijcbugwW0iY8jPi6m1tcdWCJw92Izuwt4C4gBZrj7WjN7DMhw93nhZV8zs3VACfADd8+t6bays7Np3bo1qampmFltfg2pIXcnNzeX7OxsevbsGelwRBq8NdvymLEok9c/3cFjE/tz3Yjabx4P9BqBu88H5peb93CZ9w7cF36dtoKCAiWBesLM6NixIzk5OZEORaTBKil13l63ixmLMlmauZcWTWOYMiKZ83p1DGR7kb5YXGuUBOoP/VuInJ4DBUXMXpbFXz/aTPa+IyS2a86PLuvLtcOTads8uKbWRpMIREQaqsw9h/jrokzmLM/mUGEJI1I78NDlfbmkbxdiY4LvCUiJQEQkAtydRRtzeWZRJu99tpvYJsaVad25ZXRPBiS2rdNYlAgamOLiYmJj9c8m0lAVFJUw95NtPLMok8935dOpVVPuHtubG85LoXPr+IjEpN5Ha9FVV13FsGHD6N+/P9OmTQPgzTffZOjQoaSlpXHxxRcDkJ+fz9SpUxk4cCCDBg3ilVdeAaBVq1bH1zVnzhxuvvlmAG6++WbuvPNORo4cyQ9/+EOWLl3KqFGjGDJkCOeffz6fffYZACUlJXz/+99nwIABDBo0iD/84Q+89957XHXVVcfX+/bbb/ONb3yjLnaHiJSxM6+AX721gVH//S4P/H01sU2a8KtrBrHwP8dy77hzIpYEoBGeEfz4tbWs236gVtfZr3sbHrmyf5XlZsyYQYcOHThy5AjDhw9n4sSJ3H777XzwwQf07NmTvXv3AvCTn/yEtm3bsnr1agD27dtX5bqzs7P56KOPiImJ4cCBA3z44YfExsbyzjvv8OCDD/LKK68wbdo0Nm/ezMqVK4mNjWXv3r20b9+eb3/72+Tk5JCQkMAzzzzDLbfccmY7RESqbWXWfmYszGT+6h2UuDOubxduuaAnI3t2qDc3VjS6RBBJv//975k7dy4AWVlZTJs2jQsvvPD4/fQdOnQA4J133mHWrFnHP9e+ffsq1z1p0iRiYkIPkuTl5XHTTTfxxRdfYGYUFRUdX++dd955vOno2PZuvPFGnnvuOaZOncrixYuZOXNmLX1jEalIUUkpb67ZyTOLMlmxdT+tmsVy0/mp3DQqlZSOLSId3pc0ukRQnSP3ILz//vu88847LF68mBYtWjBmzBgGDx7Mhg0bqr2OskcH5Z+Sbtmy5fH3//Vf/8VXv/pV5s6dy+bNmxkzZswp1zt16lSuvPJK4uPjmTRpkq4xiARk/+FCXlyaxczFm9mRV0CPji145Mp+XDMsidbx9fdJe10jqCV5eXm0b9+eFi1asGHDBpYsWUJBQQEffPABmZmZAMebhsaNG8eTTz55/LPHmoa6dOnC+vXrKS0tPX5mUdm2EhNDHbn+9a9/PT5/3LhxPPXUUxQXF5+0ve7du9O9e3cef/xxpk6dWntfWkQA+GLXQR6cu5rz/vtdfvHmBnp2asn0b6Xz3v8bw9TRPet1EgAlglozfvx4iouL6du3L/fffz/nnXceCQkJTJs2jW9+85ukpaUxefJkAB566CH27dvHgAEDSEtLY8GCBQD8/Oc/54orruD888+nW7dulW7rhz/8IQ888ABDhgw5XukD3HbbbaSkpDBo0CDS0tJ44YUTA77dcMMNJCcn07dv34D2gEh0KS11Fny2m2/NWMq4Jz5gzvJsJqYl8uY9X+GF28/jkn5diGlSP64BVCWwEcqCkp6e7uUHplm/fr0quCrcddddDBkyhFtvvbVOtqd/E2msDhcW88qK0O2fm3IO0bl1M741qgdTRqTQsVWzSIdXKTNb7u7pFS1TY3EUGDZsGC1btuQ3v/lNpEMRabC27T/CzI828+LSrRwoKGZQUlt+d91gLh3QjaaxDbtxRYkgCixfvjzSIYg0SO7O8i37mLEok7fW7gJgfP+u3HJBKkNT2teb2z/PlBKBiEg5hcWl/GP1dp5ZtJlV2Xm0iY/ltq/05FujUkls1zzS4dU6JQIRkbDc/KO88PFWZi7ZQs7Bo5yV0JLHrxrAN4cm0qJp460uG+83ExGppvU7DvDMokxeXbmdwuJSLjonganXpHJh7wSaNJA7f86EEoGIRKWSUue9DbuZsTCTxZtyaR4Xw6RhSUwdncrZnVtHOrw6pUQgIlHlYEERL2dk8+zizWzJPUz3tvHcf2kfrhueTLsWTSMdXkQoEURAq1atyM/Pj3QYIlFla+5h/vrRZmZnZJF/tJhhPdrzw6/34ev962bwl/pMiSCKaWwDaezcnSWb9jJjUSbvrN9FjBmXD+rG1NE9GZzcLtLh1RuNrxZ4437Yubp219l1IFz680oX33///SQnJ/Od73wHgEcffZTY2FgWLFjAvn37KCoq4vHHH2fixIlVbio/P5+JEydW+LmZM2fy61//GjNj0KBB/O1vf2PXrl3ceeedbNq0CYA//elPdO/enSuuuII1a9YA8Otf/5r8/HweffTR453hLVy4kClTpnDOOefw+OOPU1hYSMeOHXn++efp0qUL+fn5fPe73yUjIwMz45FHHiEvL49Vq1bx29/+FoCnn36adevW8cQTT5zR7hWpbQVFJcz7NHT75/odB+jQsinfGXM2N47qQZc2kev3v75qfIkgAiZPnsw999xzPBHMnj2bt956i7vvvps2bdqwZ88ezjvvPCZMmFDlAyjx8fHMnTv3S59bt24djz/+OB999BGdOnU63qHc3XffzUUXXcTcuXMpKSkhPz+/yvENCgsLOdZNx759+1iyZAlmxvTp0/nlL3/Jb37zmwrHTIiLi+OnP/0pv/rVr4iLi+OZZ57hqaeeOtPdJ1Jrdh8s4LklW3l+yRZyDxVybpfW/OLqgUwcnEh8XEykw6u3Gl8iOMWRe1CGDBnC7t272b59Ozk5ObRv356uXbty77338sEHH9CkSRO2bdvGrl276Nq16ynX5e48+OCDX/rce++9x6RJk+jUqRNwYqyB99577/j4AjExMbRt27bKRHCs8zsIDXgzefJkduzYQWFh4fGxEyobM2Hs2LG8/vrr9O3bl6KiIgYOHFjDvSVS+1Zn5/HMokxeW7Wd4lLn4j6duWV0T0ad1bHRPP0bpMaXCCJk0qRJzJkzh507dzJ58mSef/55cnJyWL58OXFxcaSmpn5pjIGKnO7nyoqNjaW0tPT49KnGNvjud7/Lfffdx4QJE3j//fd59NFHT7nu2267jZ/97Gf06dNHXVpLRBWXlPL2ul3MWJTJss37aNk0hhtG9uDm81NJ7dSy6hXIcdF9qbwWTZ48mVmzZjFnzhwmTZpEXl4enTt3Ji4ujgULFrBly5Zqraeyz40dO5aXX36Z3Nxc4MRYAxdffDF/+tOfgNCYxXl5eXTp0oXdu3eTm5vL0aNHef3110+5vWNjGzz77LPH51c2ZsLIkSPJysrihRdeYMqUKdXdPSK1Ju9IEU9/sImLfvU+//H8CnbkFfDQ5X1Z/ODFPDqhv5LAaVAiqCX9+/fn4MGDJCYm0q1bN2644QYyMjIYOHAgM2fOpE+fPtVaT2Wf69+/Pz/60Y+46KKLSEtL47777gPgd7/7HQsWLGDgwIEMGzaMdevWERcXx8MPP8yIESMYN27cKbf96KOPMmnSJIYNG3a82QkqHzMB4Nprr2X06NHVGmJTpLZsysnn4f9dw6j/fpefzl9PUvvmPHXjMP75g69y21d60aaeD/5Sn2k8AqmxK664gnvvvZeLL7640jL6N5Ha4O4s3LiHGQszWfBZDk1jmjBhcHemjk6lf/e2kQ6vQYnYeARmNh74HRADTHf3n5dbfjPwK2BbeNYf3X16kDHJ6du/fz8jRowgLS3tlElA5EwdKSxh7iehwV++2J1Pp1bNuOeS3twwsgcJrevv4C8NVWCJwMxigCeBcUA2sMzM5rn7unJFX3L3u4KKo75avXo1N95440nzmjVrxscffxyhiKrWrl07Pv/880iHIY3YjrwjzFy8hReXbmX/4SL6d2/DbyalcUVaN5rF6vbPoAR5RjAC2OjumwDMbBYwESifCGqFuzeo28QGDhzIypUrIx1GIBpac6NE3idb9zFj0Wbmr96Bu/O1fl255YKeDE9tPIO/1GdBJoJEIKvMdDYwsoJyV5vZhcDnwL3unlVBmVOKj48nNzeXjh11z3CkuTu5ubnEx+vpTTm1opJS3lizkxkLM1mZtZ/WzWK5ZXQq3xqVSnKHFpEOL6pE+jmC14AX3f2omf078CwwtnwhM7sDuAMgJSXlSytJSkoiOzubnJycgMOV6oiPjycpKSnSYUg9tTX3MC8vz+LljGx2HiigZ6eW/HhCf64elkSrZpGukqJTkHt9G5BcZjqJExeFAXD33DKT04FfVrQid58GTIPQXUPll8fFxR1/IlZE6p+CohLeXLOT2RlZfPSvXMzgwt4J/OybAxhzTueoGPylPgsyESwDeptZT0IJ4Drg+rIFzKybu+8IT04A1gcYj4jUsTXb8nhpWRb/u3IbBwqKSe7QnP837hyuHpZE90Y49m9DFVgicPdiM7sLeIvQ7aMz3H2tmT0GZLj7POBuM5sAFAN7gZuDikdE6sb+w4W8+sk2Zmdks27HAZrFNuHSAV25dngy5/XsqKP/eqhRPFAmIpFVWuos+tceZmdk89banRQWlzIgsQ2T05OZMDiRts311G+kReyBMhFp3LbtP8LLGaELv9v2H6Ft8ziuH5HCpPQkPfnbgCgRiEiNHC0u4f/W7mJ2RhYLN+4B4IKzO/Gfl/bha/26qN//BkiJQESqZd32A8zOyOLVldvYf7iIxHbNuXtsb64ZlqT7/hs4JQIRqVTekSLmfbqd2cuyWL0tj6YxTfha/y5MHp7M+Wd1IkYXfhsFJQIROUlpqbMkM5fZy7J4Y81OjhaX0qdrax65sh9XDU6kfcumkQ5RapkSgYgAsDOvgDnLs5idkc3WvYdpHR/LpPQkJqenMCCxjbpvacSUCESiWGFxKe+u38VLGVl88HkOpQ6jenXkvnHnMH5AV134jRJKBCJR6PNdB3lpWRZzP9nG3kOFdG0Tz7fHnM2k9CR6dNRQj9FGiUAkShwsKOL1VTt4aVkWK7P2ExdjXNK3C9cOT+bC3gm68BvFlAhEGjF3Z9nmfby0LIv5q3dwpKiEc7q04qHL+/KNIYl0bKXRvkSJQKRR2n2ggDkrsnk5I5vMPYdo1SyWq4Z059r0ZAYnt9OFXzmJEoFII1FUUsqCDbuZnZHFgs9yKCl1RqR24DtfPZvLBnalRVP93KVi+p8h0sBt3J3PyxlZvLJiG3vyj5LQuhl3XNiLScOS6JXQKtLhSQOgRCDSAB06Wsw/Vu9g9rIsMrbsI6aJMbZPZyanJzPm3ARiY5pEOkRpQJQIRBoId2fF1v3MXpbF66u2c6iwhF4JLXng0j58Y2ginVtrnGg5PUoEIvXcnvyj/H1FNrMzstm4O58WTWO4fGA3Jg9PZliP9rrwK2dMiUCkHiouKeWDL3J4aVkW767fTXGpMzSlHb+4eiCXD+quQd6lVul/k0g9snnPIWZnZPHKimx2HThKp1ZNueWCnlybnsTZnVtHOjxppJQIRCLsSGEJ81fvYHZGFh9n7qWJwZhzO/PjCclc3LczcbrwKwFTIhCJAHdnVXYeL2Vk8drK7Rw8Wkxqxxb84Ovncs2wJLq00YVfqTtKBCJ1aO+hQuZ+so2XM7LYsPMg8XFNuGxgN65NT2Zkzw668CsRoUQgErCSUufDL3J4OSOb/1u3k6ISJy25HT/9xgCuTOtOm/i4SIcoUU6JQCQgWXsP83JGFnOWZ7M9r4D2LeK48bxUrh2eRJ+ubSIdnshxSgQitaigqIS31u7kpWVZfPSvXMzgwt4J/OjyflzSrzPNYjXQi9Q/SgQitWDt9jxeWpbFq59s40BBMUntm3PfuHO4ZlgS3ds1j3R4IqekRCBymvKPFvPap9t5celWVmXn0TS2CZcO6Mq16cmM6tWRJhroRRoIJQKRGnB3Ps3OY9bSrcz7dDuHC0s4t0trHrmyH98Ykki7Fk0jHaJIjQWaCMxsPPA7IAaY7u4/r6Tc1cAcYLi7ZwQZk8jpyDtSxKufbOPFpVvZsPMgzeNiuDKtG9eNSGGIBnqRBi6wRGBmMcCTwDggG1hmZvPcfV25cq2B7wEfBxWLyOlwdzK27OPFpVuZv3oHBUWlDEhsw+NXDWDi4O601m2f0kgEeUYwAtjo7psAzGwWMBFYV67cT4BfAD8IMBaRatt7qJC/r8hm1rIsNu7Op1WzWK4emsSUESkMSGwb6fBEal2QiSARyCoznQ2MLFvAzIYCye7+DzNTIpCIKS11lmzK5cVlWby1ZieFJaUMSWnHL68exOWDutFSvX1KIxax/91m1gT4H+DmapS9A7gDICUlJdjAJKrkHDzKnOXZzFq2lS25h2kTH8v1I1O4bkSyHvqSqBFkItgGJJeZTgrPO6Y1MAB4P3yhrSswz8wmlL9g7O7TgGkA6enpHmDMEgWOdfkwa2kW76zfRXGpM6JnB+65pDeXDuhGfJwe+pLoEmQiWAb0NrOehBLAdcD1xxa6ex7Q6di0mb0PfF93DUlQduQd4eWMbF5alsW2/Ufo0LIpU0enMnl4Cmd31iDvEr0CSwTuXmxmdwFvEbp9dIa7rzWzx4AMd58X1LZFjikuKWXBZznMWrqVBZ/tptThgrM78cBlfRjXr4u6fBAh4GsE7j4fmF9u3sOVlB0TZCwSXbL2HmZ2RhazM7LYdeAoCa2bcedFZzF5eDI9OraMdHgi9YpuhZBGo7C4lHfW7+LFpVtZuHEPAGPOSeCxiSmM7aORvkQqo0QgDV7mnkPMWraVV5Znsye/kG5t47l7bG+uHZ5Mojp8E6mSEoE0SMe6e35x6VaWbNpLTBPj4j6dmTIihQvPSSBGHb6JVJsSgTQoX+w6yItLs/j7J9nsP1xEcofm/ODr5zJpWBKdNc6vyGlRIpB670hhCf9YvYMXl25l+ZZ9xMUYX+vflSnDUzj/LHX3LI2MOxQfhcJDUHQo9LfwMBTmQ8ezoW1irW9SiUDqrbXb85i1NItXV27jYEExvTq15MHL+nD10CQ6tmoW6fAk2pWWQvGRcEUdfhWFK+zCwxVX5EWHqygfnvbSird5+f/A8Ftr/asoEUi9UtFgL5cP7MZ1w5MZ0bODunuWmistqUZFXXZ5NSv2okM1iyOmGTRteeIV1yL0t01imelW0DQ8P+5Y2fD8uBbQqXcgu0iJQCJOg70IAMWFp3EEXVn5MtPFBTWL41gFXb5ibpnw5Yr5SxV7JRV5XEuIqb/Vbf2NTBo9DfbSiLiHKt0j+6FgfzX+7oOCvJMr+dKi6m/PmpQ7Yg5XvPHtoE33mlfUx8rHtYAm0fe8iRKB1CkN9lKPVbsy31fxslNW5AbxbUIVdfN2ob+tu0GzNqeumCuryGPjQQcKtabaicDMzgdSy37G3WcGEJM0QhrspY6cqjI/sq/qo/WaVuZtEk+8P9XfZm2gifp1qq+qlQjM7G/AWcBKoCQ82wElAqmUBns5TV+qzCs5Aj/tyrztyZV0tSvztlHZbBINqvtLTAf6ubvGApAqabAXylTmNazEA63M24ePzFWZy8mqmwjWEBo4ZkeAsUgD1igHeykuhKMHwhV0XqiSLsir4FVmfq1V5u0rr9BVmUstq24i6ASsM7OlwNFjM919QiBRSYNRrwd7KSkOV+T7KqnAK6jIy76KDp96/U1iQxVzfNsTr7ZJVTexqDKXeqa6ieDRIIOQhqXOBnspLSl3RF7Dirww/9TrtybhCrxMZd6pS5mK/VjF3bbiV1wL3bkijUK1EoG7/9PMegC93f0dM2tBaNQxiSI1HuyltBQKD5ZpMqlhZX70QBURHbuLpUxl3qHXl4/SK6vMm7ZSRS5C9e8auh24A+hA6O6hRODPwMXBhSYR507h4QMsXPMv3lnxGf/K2kYbO8xtXWO4oF8c57QtJeboe/Bh2Yq8bIV+gNDNZafQrM3JR+DtelR+BF6+Qm/aWs0rIrWguk1D3wFGAB8DuPsXZtY5sKik7pUUwe51sG0FbF9B4dblxORuoKmXMBYYC3Csp4fc8AtCR9VlK+c2idC5f/Uqc91bLlIvVDcRHHX3wmOP/JtZLFUe6km9VVoKuRth+4rjFT87Vx/vk+VQk9YsL05lvV9Gh4TuDDwrhd49kohpXrYyD1/wrMf9p4hI9VT3V/xPM3sQaG5m44BvA68FF5bUGnfIyw5X+stDFf+OT0+0v8e1wLulsbnndczd3YVXd3dlX7PuXD+qBzeNSqW7hnoUafSqmwjuB24FVgP/Dsx396cDi0pO36E9J47yj/09lBNa1iQOuvSHgddA96EcTkhj9ubm/GVxFll7j5DSoQW3XJHKpPRkPfUrEkWqffuouz8MPA1gZjFm9ry73xBcaFKlggOwY2WZiv8TyNsaXmiQcC6cPQ4Sh0L3odB1AMQ2Y0feEf66aDMvzNvKwYJi0nu050eX9WVcv64a61ckClU3ESSb2QPu/t9m1hSYTajfIakrRQWwa82J5p3tK2DPFxy/VNOuByQNgxG3hyr+bmnQrPVJq1idncf0hev4x6odlLpz6cBu3HZBT4aktK/77yMi9UZ1E8EtwPNm9gDwVeANd38iuLCiXEkx5Gw4uXln11ooLQ4tb9k5VNkPuObE0X7LjhWuqrTUeXfDbp7+cBNLM/fSqlksN5+fyk3np5LcoUUdfikRqa9OmQjMbGiZyd8BTwGLCF08HuruK4IMLiq4w95NJ7fr71x1onuDZm2h+2A4/7uhCj9xaOgWzSoehDpcWMwry7OZsWgzmXsOkdiuOQ9d3pdrhyfTRn3+i0gZVZ0R/Kbc9D6gX3i+E769XGrgwPaTm3e2fxJ6+ApCg210S4OhN5040u/Qq0YPTe06UMDMxZt5/uOt7D9cRFpyO/54/RDG9+9KbIwevhKRLztlInD3r57Jys1sPKEziRhgurv/vNzyOwk9rFYC5AN3uPu6M9lmvXJ474mLuMeO9vN3hpZZDHTpB/2uClX6icMgoe9p35e/bvsBpi/cxGufbqe41Pl6v67c9pWeDOvRXkM+isgpVbeLibbAI8CF4Vn/BB5z97xTfCYGeBIYB2QDy8xsXrmK/gV3/3O4/ATgf4DxNf4W9cHR/ND9+WXb9fdtPrG8Y2/oddGJ5p2uAyHuzO7RLy11/vl5DtMXbmLRxlxaNI3hhpE9mDo6teK+f0REKlDdw88ZhMYkuDY8fSPwDPDNU3xmBLDR3TcBmNksYCJwPBG4e9lexVrSUJ5WLi4M3cFzrNLftgL2fAZeGlreNhm6D4FhN4cq/u6DQ0/k1pKCohL+vmIbf1m4iX/lHKJrm3juv7QPU4an0LaF2v9FpGaqmwjOcvery0z/2Myqun00EcgqM50NjCxfyMy+A9xHqCebCq85mNkdhDq9IyUlpZoh15LSEtjz+ckXc3etgZLC0PIWHUOVfb+J4Xb9IdAqmG6Ycg4e5W9LtvDcki3sPVRI/+5t+O3kwVw+qBtxav8XkdNU3URwxMwucPeFAGY2GjhSGwG4+5PAk2Z2PfAQcFMFZaYB0wDS09ODO2twDzXnHG/e+STU3HOsX/umrUNH9yPvPHExt11K4F0Zf77rINM/3MSrn2ynsKSUS/p25rav9GJkzw5q/xeRM1bdRHAnMDN8rQBCdw99qcIuZxuQXGY6KTyvMrOAP1UzntpxcNfJffBs/wSO7A0ti2kWascffP2Jdv2Oveus22N358Mv9jB9YSYffJ5DfFwTrh2exC2je9IrIcIjf4lIo1LdRHDA3dPMrA2E2vbNrGcVn1kG9A6X2wZcB1xftoCZ9Xb3L8KTlwNfEJSCvHJ98HwCB8J5yZqE7tjpc/mJI/3O/SC26anXGYCjxSX878rt/OXDTD7bdZCE1s34/tfO4fqRPejQsu7jEZHGr7qJ4BVgaLmLu3OAYZV9wN2Lzewu4C1Ct4/OcPe1ZvYYkOHu84C7zOwSoIjqnWWcvo+fggU/Db3v0AtSRp2o9LsNgqaRvctm76FCnluyhZmLt7An/yh9urbm15PSuDKtW+0N/SgiUoGqnizuA/QH2ppZ2TuE2gDxVa3c3ecD88vNe7jM++/VKNozMfAaSEoPXcxtXn/61tm4O58ZizJ5ZXk2R4tLGXNuArdd0IvRZ3dU+7+I1ImqzgjOBa4A2gFXlpl/ELg9qKAC0aFX6FUPuDuLN+Xylw8zeXfDbprGNuGbQxK59YKe9O7SuuoViIjUoqoSQQvg+8A0d19cB/E0aoXFpby+ajvTP8xk3Y4DdGzZlJKCPxEAAA2JSURBVHsu6c2/ndeDTq2aRTo8EYlSVSWCFOBlIM7M3gXeAJa6e8N48Kue2H+4kBeWbuXZjzaz68BRzu7cip9/cyBXDUkkPk7t/yISWVX1NfQL4Bdm1hq4hFB31H82s/XAm8Bb7r4r+DAbps17DjFjUSYvZ2RzpKiEr/TuxC+uHsSFvRNoogFgRKSeqNZdQ+5+EJgbfmFm/YBLgZnA1wOLrgFyd5Zt3sf0Dzfx9vpdxDYxJg4Otf/37dYm0uGJiHxJVXcN/Zu7Pxd+P9rdFwG4+zozG+vuSgJhRSWlvLFmJ9M/3MSq7DzatYjjO2PO5lujetC5TZU3WImIRExVZwT3Ac+F3/8BKDtQzS3AH4MIqiE5UFDErKVb+euizWzPK6BXp5Y8ftUArh6aRPOmav8XkfqvqkRglbyvaDqqZO09zDOLNvPSsq0cKizhvF4deGziAMb26az2fxFpUKpKBF7J+4qmo8LyLfv4y8JNvLlmJ03MuGJQN277Si8GJNZeN9MiInWpqkTQx8xWETr6Pyv8nvB0/Xg6qw4Ul5Tyf+t2Mf3DTazYup828bHcceFZ3HR+D7q1PbPBZUREIq2qRJAGdOHkcQUg1KvozkAiqkfyjxYze1kWMxZlkr3vCCkdWvDolf2YlJ5My2anN6SkiEh9U1Vt9gTwgLtvKTsz3AvpE5zc7USjsW3/EZ79aDMvfryVg0eLGZ7anocu78e4fl2IUfu/iDQyVSWCLu6+uvxMd19tZqmBRBRBq7L3M/3DTP6xegcAlw7oym1f6cXg5HYRjkxEJDhVJYJT1YCNonG8pNR5Z/0u/vJhJks376VVs1imnp/KzaNTSWrfItLhiYgErqpEkGFmt7v702VnmtltwPLgwgre4cJi5izPZsbCTDbnHiaxXXMeurwvk4cn0zpeA8CLSPSoKhHcA8w1sxs4UfGnExpo/htBBhaUXQcKePajzTz/8VbyjhSRltyOP379XMb370qsBoAXkShUVadzu4DzzeyrwIDw7H+4+3uBR1bLNuw8wLR/buK1VdspLnW+3q8rt1/Yk6Ep7TUAjIhEtep2OrcAWBBwLIFatDGXN9fu5IaRPZg6OpUeHSM7NKWISH0RNTfDTxmRzDVDk2jbQu3/IiJlRU0iaNE0NnRlQ0RETqKroyIiUU6JQEQkyikRiIhEOSUCEZEop0QgIhLllAhERKJcoInAzMab2WdmttHM7q9g+X1mts7MVpnZu2bWI8h4RETkywJLBGYWAzwJXAr0A6aYWb9yxT4B0t19EDAH+GVQ8YiISMWCPCMYAWx0903uXgjMAiaWLeDuC9z9cHhyCZAUYDwiIlKBIBNBIicPcZkdnleZW4E3AoxHREQqUC+6mDCzfyPUvfVFlSy/A7gDICUlpQ4jExFp/II8I9hGaJD7Y5LC805iZpcAPwImuPvRilbk7tPcPd3d0xMSEgIJVkQkWgWZCJYBvc2sp5k1Ba4D5pUtYGZDgKcIJYHdAcYiIiKVCCwRuHsxcBfwFrAemO3ua83sMTObEC72K6AV8LKZrTSzeZWsTkREAhLoNQJ3nw/MLzfv4TLvLwly+yIiUjU9WSwiEuWUCEREopwSgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlAs0EZjZeDP7zMw2mtn9FSy/0MxWmFmxmV0TZCwiIlKxwBKBmcUATwKXAv2AKWbWr1yxrcDNwAtBxSEiIqcWG+C6RwAb3X0TgJnNAiYC644VcPfN4WWlAcYhIiKnEGTTUCKQVWY6OzxPRETqkQZxsdjM7jCzDDPLyMnJiXQ4IiKNSpCJYBuQXGY6KTyvxtx9mrunu3t6QkJCrQQnIiIhQSaCZUBvM+tpZk2B64B5AW5PREROQ2CJwN2LgbuAt4D1wGx3X2tmj5nZBAAzG25m2cAk4CkzWxtUPCIiUrEg7xrC3ecD88vNe7jM+2WEmoxERCRCGsTFYhERCY4SgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlFMiEBGJckoEIiJRTolARCTKKRGIiEQ5JQIRkSinRCAiEuWUCEREopwSgYhIlFMiEBGJcoEmAjMbb2afmdlGM7u/guXNzOyl8PKPzSw1yHhEROTLAksEZhYDPAlcCvQDpphZv3LFbgX2ufvZwBPAL4KKR0REKhbkGcEIYKO7b3L3QmAWMLFcmYnAs+H3c4CLzcwCjElERMoJMhEkAlllprPD8yos4+7FQB7QMcCYRESknNhIB1AdZnYHcEd4Mt/MPjvNVXUC9tROVLVKcdWM4qq5+hqb4qqZM4mrR2ULgkwE24DkMtNJ4XkVlck2s1igLZBbfkXuPg2YdqYBmVmGu6ef6Xpqm+KqGcVVc/U1NsVVM0HFFWTT0DKgt5n1NLOmwHXAvHJl5gE3hd9fA7zn7h5gTCIiUk5gZwTuXmxmdwFvATHADHdfa2aPARnuPg/4C/A3M9sI7CWULEREpA4Feo3A3ecD88vNe7jM+wJgUpAxlHPGzUsBUVw1o7hqrr7GprhqJpC4TC0xIiLRTV1MiIhEuUaZCOpr1xbViOtmM8sxs5Xh1211FNcMM9ttZmsqWW5m9vtw3KvMbGg9iWuMmeWV2V8PV1SulmNKNrMFZrbOzNaa2fcqKFPn+6uacUVif8Wb2VIz+zQc148rKFPnv8dqxhWR32N42zFm9omZvV7BstrfX+7eqF6ELkz/C+gFNAU+BfqVK/Nt4M/h99cBL9WTuG4G/hiBfXYhMBRYU8nyy4A3AAPOAz6uJ3GNAV6v433VDRgaft8a+LyCf8c631/VjCsS+8uAVuH3ccDHwHnlykTi91iduCLyewxv+z7ghYr+vYLYX43xjKC+dm1Rnbgiwt0/IHTXVmUmAjM9ZAnQzsy61YO46py773D3FeH3B4H1fPmJ+TrfX9WMq86F90F+eDIu/Cp/YbLOf4/VjCsizCwJuByYXkmRWt9fjTER1NeuLaoTF8DV4eaEOWaWXMHySKhu7JEwKnx6/4aZ9a/LDYdPyYcQOposK6L76xRxQQT2V7iZYyWwG3jb3SvdX3X4e6xOXBCZ3+NvgR8CpZUsr/X91RgTQUP2GpDq7oOAtzmR9aViK4Ae7p4G/AF4ta42bGatgFeAe9z9QF1ttypVxBWR/eXuJe4+mFDvAiPMbEBdbLcq1Yirzn+PZnYFsNvdlwe9rbIaYyKoSdcW2Cm6tqjruNw9192PhienA8MCjqm6qrNP65y7Hzh2eu+hZ1bizKxT0Ns1szhCle3z7v73CopEZH9VFVek9leZ7e8HFgDjyy2KxO+xyrgi9HscDUwws82Emo/Hmtlz5crU+v5qjImgvnZtUWVc5dqRJxBq560P5gHfCt8Ncx6Q5+47Ih2UmXU91jZqZiMI/X8OtAIJb+8vwHp3/59KitX5/qpOXBHaXwlm1i78vjkwDthQrlid/x6rE1ckfo/u/oC7J7l7KqE64j13/7dyxWp9fzWI3kdrwutp1xbVjOtuM5sAFIfjujnouADM7EVCd5R0MrNs4BFCF89w9z8Tejr8MmAjcBiYWk/iugb4DzMrBo4A19VBQh8N3AisDrcvAzwIpJSJKxL7qzpxRWJ/dQOetdBAVU2A2e7+eqR/j9WMKyK/x4oEvb/0ZLGISJRrjE1DIiJSA0oEIiJRTolARCTKKRGIiEQ5JQIRkSinRCBRzcxKyvQuudIq6BX2DNadapX0nFpJ+ZZm9k74/cLww0IigdN/NIl2R8LdDNQHo4DFZtYeOBTuR0YkcDojEKmAmW02s1+a2WoL9Vt/dnh+qpm9F+6I7F0zSwnP72Jmc8Mdun1qZueHVxVjZk9bqM/7/ws/xVp+W2eFHwJ7DrgeWA6khc9QOtfRV5YopkQg0a55uaahyWWW5bn7QOCPhHqEhFBnbc+GOyJ7Hvh9eP7vgX+GO3QbCqwNz+8NPOnu/YH9wNXlA3D3f4XPSpYT6q78WeBWdx/s7rtr9duKVEBPFktUM7N8d29VwfzNwFh33xTuzG2nu3c0sz1AN3cvCs/f4e6dzCwHSCrTSdmx7qDfdvfe4en/BOLc/fFKYlnm7sPN7BXge+6eXctfV6RCOiMQqZxX8r4mjpZ5X0IF1+XM7M/hi8q9w01E44HXzeze09ymSI0oEYhUbnKZv4vD7z/iRCdfNwAfht+/C/wHHB/wpG11N+LudwI/Bn4CXAX8I9ws9MSZhS9SPbprSKJd8zK9dQK86e7HbiFtb2arCB3VTwnP+y7wjJn9AMjhRM+i3wOmmdmthI78/wOoSdfTFwEzga8A/zytbyJymnSNQKQC4WsE6e6+J9KxiARNTUMiIlFOZwQiIlFOZwQiIlFOiUBEJMopEYiIRDklAhGRKKdEICIS5ZQIRESi3P8HhE7SIauc1noAAAAASUVORK5CYII=\n"},"metadata":{"needs_background":"light"}}]},{"cell_type":"code","source":["result2 = model3.translate(['मैं आज घर जाऊंगा।'])\n","result2[0].numpy().decode()"],"metadata":{"id":"80VG6TLhAgcO","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1669569568373,"user_tz":-60,"elapsed":19,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"f79d3b80-c6e7-4299-95ed-75c396747a2a"},"execution_count":210,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'i answer to come . '"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":210}]},{"cell_type":"code","source":["model3.plot_attention('मैं आज घर जाऊंगा।')"],"metadata":{"id":"1qeY3urAskx4","colab":{"base_uri":"https://localhost:8080/","height":650},"executionInfo":{"status":"ok","timestamp":1669569568375,"user_tz":-60,"elapsed":17,"user":{"displayName":"Kai Ponel","userId":"09877000230544749752"}},"outputId":"22de01a9-2df1-4744-ac63-b11e7c673aaf"},"execution_count":211,"outputs":[{"output_type":"display_data","data":{"text/plain":["<Figure size 720x720 with 1 Axes>"],"image/png":"iVBORw0KGgoAAAANSUhEUgAAAnUAAAJ5CAYAAADINcYvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZSsdX3n8c8XLougmBE1LknUiBo3xHBFIS4YTXBNYkxi4jYuI64zjkYTzYzn6JxkIGpGozFR9BhQo1lcxmg0ZFFQcUFR3BCNIgZUcA2LKLJ854+qO7Z9+t5bfbnV1fdXr9c5fbrreaqqv/0cwLe/p56q6u4AALBn22vRAwAAcM2JOgCAAYg6AIABiDoAgAGIOgCAAYg6AIABiDoAgAGIOgCAAYg6AIABiDoAgAFsWfQAzEdV/fouPOzd3f393T4MADB35bNfx1RVV6/zIZ3kVt19zjzmAQDmy0rd2G7U3d+Y5Y5Vdcm8hwEA5sdr6sZ1UpL1nEp9Q5KL5zQLADBnTr8CAAzASt3AquqqqrrhoucAAOZP1I2tFj0AALAxRB0AwABc/Tq+36qqHV4A0d2v26hhAID5cKHEwKbvVXdZJu9Btz3d3Qdt0EgAwJyIuoFNo27m96oDAPZcXlM3NsUOAEtC1I3N1a8AsCRcKDG2HX6qRFVtTfKH3X2/jRtpLFV1vfU+pru/M49ZAFhuXlM3uKr6pSS/nOSKJK/p7nOq6tZJXpTkQUn+WdTtuunrFtfzL1EnuXV3nzOnkQBYUlbqBlZV/znJXyb5TpLrJXl8VT09yauSvDXJYd396QWOOIrfyOQY70wledecZwFgSVmpG1hVnZnkr7v7+Kr6rSR/neQTSX6ru7+02OnGUFVfTrK1u7894/0/k+T+3X3efCcDYNmIuoFV1SVJDu3uL1fVXkkuT3Lf7j51waMBALuZq1/HdmCS7yVJd1+d5AdJrBABwIC8pm58D6yqi6Y/75XkmKq6cOUduvutGz/WWKqqkjw6yUOT/GwmF0Sck+TvkvxVWxIHYM6cfh3Y9MrMnenu3nvuwwyuqt6a5NeSfDrJWZlcFHG7JHdI8rbufugCxwNgCVipG1h3O72+AarqEZm8bcz9uvufVu07Jslbqurh3f3GhQwIwFLwP/pLrqruu+gZBvDIJH+8OuiSpLtPzuQ9AR+54VMBsFRE3RKqqptW1f+sqnOSnLzoeQZwp+z4/ef+IclhGzQLAEtK1C2Jqtq7qn69qv4hyblJHpLklUkOWehgYzg4ydd3sP/rmbz5MwDMjdfUDa6qbpPkv2RyZeb3krwxk9d/Paq7z1rkbAPZJ5OPYdueK6f3AYC5EXUDq6r3Z3L15Vsy+RSJU6fbf3+hg43puKq6bDv7DtjQSQBYSqJubEcmeUWSE7r7s4seZmDvS3LLGe4DAHMj6sZ2l0xOvX6gqs5N8rokb1roRAPq7qMXPQMAePPhJVBV+yf5zSSPS3L3TC6QeU6S13T3dxc5GwCwe4i6gVXVzyQ5b+VHVFXVIfnRhRMHJ3lPd99/QSMOoaqeOcv9uvv/zHsWAJaXqBtYVV2V5Mbd/Y019u2d5EFJHtfdv7rhww2kqr68g92d5EZJ9vNxbADMk6gb2PSzX2+0VtQxf1X1s0n+KJNT32/u7t9e8EgADMybD8NuVlUHV9VLk5yV5IZJ7iboAJg3V7+O71lVdemO7tDd/2ujhhlZVV0ryTOT/F6mn9rR3e9e6FAALA2nXwc2Pf36+Uw+0WB7ursP3aCRhlRVeyV5fJIXZPLJEs9L8vr2LxcAG0jUDcxr6jZGVZ2V5GZJXpbk5Ul+sNb9uvs7GzkXAMtF1A1sR1e/svtM43mbtf6FqkxWRF39CsDceE3d2GrRAyyJey96AAAQdWN7QZIdXiTBNdfdpy56BgDwliZj+5Mk11q5oapuW1Wvraq/rSpvs7EbVNWxVbXfitu3r6otK24fWFWuMAZgrrymbmBV9fokF3X306a3r5/k7CRXJ/l6kjskeVR3v3FxU+75Vr92saouTnJYd58zvf2TSb7mNXUAzJOVurEdmeRtK24/KskPk9yqu++U5MVJnraIwQaz+rWLXssIwIYTdWO7cZIvrbh97yRv6e6LprdPSnKrDZ8KANjtRN3YLkty4IrbRyT58IrbP0hywIZOBADMhatfx/bJJI/N5KPCjk5ygyTvWbH/lkm+toC5RvTAqtq2ArpXkmOq6sLp7Z9Y0EwALBEXSgysqu6V5N1JvpVJ0L2xux+/Yv+fJ7lWdz92QSMOYdWbD2+PNx8GYK6s1A2su0+tqsOT/HKSC5L83aq7nJnk9A0fbDDd7WUMACyclbpBVdURSc7o7qtmvP/hST7V3VfMd7KxOM4AbBaiblDT9067UXd/c8b7/9h7qzEbxxmAzcLp13FVkuOq6rIZ77/vPIcZmOMMwKYg6sb1vkyubp3Vh5J8f06zjMxxBmBTcPoVAGAArtoDABiAqFsyVXXsomdYFo71xnCcN45jvXEc640x2nEWdctnqH+ANznHemM4zhvHsd44jvXGGOo4izoAgAG4UOIa2Lf26/1z4KLHWJcrcnn2yX6LHmMp7InH+pBDv7foEdbtW9++Ktc/eM/7BLYvfnrP+m9HklzRl2ef2rP+mU6S7IH/M7cn/vdjT7SnHudL8t1vdfcNVm/3libXwP45MHet+yx6DNht/v4fP7roEZbGr97syEWPsDT6yisXPQLsVv/Sb/7KWtudfgUAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqFtDVZ1YVe9c9BwAALPasugBNqmnJ6lFDwEAMCtRt4buvmjRMwAArIfTr2tw+hUA2NOIOgCAATj9uk5VdWySY5Nk/xyw4GkAACas1K1Td5/Q3Vu7e+s+2W/R4wAAJBF1AABDEHUAAAMQdQAAAxB1AAADcPXrGrr7MYueAQBgPazUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMYMuiBwA2j1+56V0WPcISuXLRAyyPqkVPsBTedf4Zix5haex7k7W3W6kDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKjbgarad9EzAADMYq5RV1X3q6r3V9V3q+o7VXVyVd12uu/mVdVV9dCq+uequqyqzqqqX1rx+H2q6mVV9bWquryqzquq46f7nlRVZ6+4732nz/ecFdveUFWvWXH7qKo6dfq7vlpVf1FVB63Yf8p024ur6ptJTpvn8QEA2F3mvVJ3YJKXJjkiydFJLkryjlUrYH+U5GVJ7pTko0n+uqquPd3335I8JMlvJ7lVkocl+fx03ylJblNVN5rePjrJt6bft7nX9H6pqjsm+ackfz/9Xb+e5LAkr1018yOTVJJ7JHn0+v9kAICNt2WeT97db1l5u6oem+TiTCLv/Onml3T3O6b7/yCTkDosyQeS3CzJF5K8v7s7yb8n+eD0uc+uqguS3DvJmzKJuRcneV5VbUly8yQ/lWnUJXl2kr/p7j9ZMc+Tk3yiqm7Y3d+Ybv5yd//u9v6mqjo2ybFJsn8OWN8BAQCYk3mffr1lVb2xqr5UVRcnuXD6O39mxd0+teLnr02/33D6/cRMAu8LVfWKqnpgVa2c+dQkR1fVAUnuMr3/t6Y/H53kS929LR4PT/LIqrp021d+dHr1liue84wd/U3dfUJ3b+3urftkv50cAQCAjTHXlbok78xkRe6JSb6a5MokZyVZefr1im0/dHdXVTKNze7+eFXdPMkxSe6T5KQkn6yqX+ruqzNZhXtmkqOSfLG7L6yqUzJZvbtdfrRKt+05X5PkJWvM+dUVP39vV/5QAIBFmlvUVdXBSX4uyVO6+73TbT+/3t/Z3ZckeXOSN1fViUk+nOSQTE7LnpLkL5I8Ij8KuFOmt38uyXNXPNXHk9y+u7+4K38PAMBmNs+Vuu9mcir0CVV1XpKbJnlRJqt1M6mqZyb5epIzM1nRe3gmr8k7P/mx19U9MsnvTB92SpJXZ/K3nbLi6f44yYer6pVJXpXkkkzC78Hd/cRd+gsBADaJub2mbnp69GFJDk3ymSSvSPK8JJev42kuyeQCh9MzWWk7LMn9u/uyFfc5Ncne0+/p7nMzOZ268vV06e5PJblnJhdQnJrkk0mOy+R1fgAAe7SaXFTKrjiortd3rfssegwAdmTyWm3m7F3n7/A6Q3ajfW9yzhndvXX1dp8oAQAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADCALYseAGAZfeUFRy16hKVRVy96guXw4PsesugRlshxa261UgcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwAFEHADAAUQcAMABRBwAwgKWKuqo6par+bNFzAADsbksVdQAAo1qaqKuqE5PcK8lTq6qnXzevqntW1Ueq6gdVdWFVvaSq9l3wuAAA67I0UZfk6Uk+lOQvk9x4+nVFkncn+USSOyd5fJLfSXLcgmYEANglSxN13X1Rkh8muay7L+juC5I8JcnXkjyluz/X3e9M8pwkT6uqA9Z6nqo6tqo+VlUfuyKXb9j8AAA7sjRRtx23TfLh7r56xbYPJNk3ySFrPaC7T+jurd29dZ/stxEzAgDs1LJH3Y70ogcAAJjVskXdD5PsveL255LcrapWHoe7T+/3pY0cDADgmli2qDs3yRHTq16vn+TPk9wkyZ9X1W2r6oFJjk/yZ9192QLnBABYl2WLuhdnsgp3VpJvJtknyf0zufL1zCSvTfKmJH+wqAEBAHbFlkUPsJG6+wtJjly1+dwkd934aQAAdp9lW6kDABiSqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABjATqOuqvabZRsAAIszy0rdh2bcBgDAgmzZ3o6qulGSmya5VlXdOUlNdx2U5IANmA0AgBltN+qSHJPkMUl+Ksmf5EdRd3GSP5jvWAAArMd2o667T0pyUlU9tLvfsoEzAQCwTrO8pu7Xquq6225U1c2q6l/nOBMAAOs0S9R9IMlHquoBVfWEJP+c5KXzHQsAgPXY0WvqkiTd/aqq+myS9yb5VpI7d/cFc58MAICZzfI+dY9K8tokj05yYpJ3VdWd5jwXAADrsNOVuiQPTXL37v5GkjdV1duSnJTksLlOBgDAzGY5/fprSVJVB3T3Zd19elUdMf/RAACY1SynX4+sqrOSnD29fae4UAIAYFOZ5erXl2byRsTfTpLu/mSSe85zKAAA1meWqEt3n7dq01VzmAUAgF00y4US51XVUUm6qvZJ8vQkn5vvWAAArMcsK3VPSvLUJDdN8tVMrnp9yjyHAgBgfWZZqbtNdz9i5Yaq+oUkp81nJAAA1muWlbqXz7gNAIAF2e5KXVUdmeSoJDeoqmeu2HVQkr3nPRgAALPb0enXfZNce3qf66zYfnGS35jnUAAArM92o667T01yalWd2N1f2cCZAABYp52+pk7QAQBsfjO9+TAAAJvbLJ/9+guzbAMAYHG8pQkAwAC8pQkAwAC8pQkAwAC8pQkAwABm+ezXE6uqV2/s7l+cwzwAS+Gn3/P9RY+wNLZ85HOLHmEp3P6Dly96hKVx8p3X3j5L1D1rxc/7J3lokiuv+UgAAOwuO4267j5j1abTqur0Oc0DAMAu2GnUVdX1VtzcK8nhSa47t4kAAFi3WU6/npGkk1Qmp12/nOTx8xwKAID1meX06y02YhAAAHbdLKdf90/ylCR3z2TF7v1JXtndP5jzbAAAzGiW06+vS3JJfvTRYA9P8vokvzmvoQAAWJ9Zou4O3X27FbffW1VnzWsgAADWb68Z7vPxqrrbthtVddckH5vfSAAArNcsK3WHJ/lgVf379PbPJPl8VX06SXf3oXObDgCAmcwSdfeb+xQAAFwjs0TdH3b3o1ZuqKrXr94GAMDizPKautuvvFFVWzI5JQsAwCax3airqudW1SVJDq2qi6vqkuntC5O8fcMmBABgp7Ybdd19XHdfJ8mLuvug7r7O9Ovg7n7uBs4IAMBOzPKaundX1T1Xb+zu981hHgAAdsEsUffsFT/vn+SIJGck+cW5TAQAwLrtNOq6+8Erb1fVTyd56dwmAgBg3Wa5+nW185PcdncPAgDArtvpSl1VvTxJT2/uleSwJB+f51AAAKzPLK+pW/k5r1cmeVN3nzaneQAA2AWzRN3fJDlk+vMXu/sHc5wHAIBdsKM3H95SVS/M5DV0JyV5XZLzquqFVbXPRg0IAMDO7ehCiRcluV6SW3T34d3980lumeQnkrx4I4YDAGA2O4q6ByV5Qndfsm1Dd1+c5MlJHjDvwQAAmN2Ooq67u9fYeFV+dDUsAACbwI6i7qyqevTqjVX1yCRnz28kAADWa0dXvz41yVur6nGZfCxYkmxNcq0kD5n3YAAAzG67UdfdX01y16r6xSS3n25+V3f/64ZMBgDAzGb57Nf3JHnPBswCAMAu2pXPfgUAYJMRdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAAD2FRRVxO/W1X/VlWXV9X5VXXcdN8dq+pfqur7VfWdqjqxqq674rEnVtU7q+r3q+qCqrqoqo6vqr2q6vlV9Y3p9t9f9TuvW1UnTPdfUlWnVtXWjf7bAQCuiU0VdUn+d5LnJTkuye2T/GaS86rqwCQnJ7k0yRFJHpLkqCSvXfX4eya5RZKjkzwpye8leVeS/ZLcPcnzkxxfVYcnk4hM8g9JbprkQUnunOR9Sd5TVTee098IALDbbVn0ANtU1bWTPCPJf+/ubbH2xSQfqqonJDkwyaO6+5Lp/Y9N8t6qOqS7vzi9/0VJntrdVyU5u6p+N8mNu/t+0/1fqKrnJLl3kjOm3w9LcoPu/v70Ps+rqgcneVSSF64x57FJjk2S/XPAbjwCAAC7btNEXZLbZbKi9q9r7Lttkk9tC7qpDya5evq4bVF31jTotrkwyX+seq4Lk9xw+vPhSQ5I8s3Jot3/t3+SW641ZHefkOSEJDmortc7/pMAADbGZoq6XbUyrK5YY99a27addt4rk8i7xxrPe/FumQ4AYANspqj7XJLLk9wnyb+tse9xVXWdFat1R2USZZ+7Br/z40l+MsnV3X3ONXgeAICF2jQXSkxj7U+THFdVj62qW1bVEVX15CR/leSyJK+bXgV7zySvSvLWFa+n2xX/kuS0JG+vqvtX1S2q6siqekFVrbV6BwCwKW2aqJt6bpI/zuQK2M8leUuSn+ruy5Ick+SgJKcneXuSDyV53DX5Zd3dSR6Q5D1JXp3k80n+Nsltknztmjw3AMBG2kynX9PdVyc5fvq1et+nMzk1u73HPmaNbQ9aY9vdVt2+JMnTp18AAHukzbZSBwDALhB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAADEHUAAAMQdQAAAxB1AAAD2LLoAQCW0d4X/3DRIyyNe5z+3UWPsBQ+cPefXPQIS89KHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwABEHQDAAEQdAMAARB0AwAC2LHqAPU1VHZvk2CTZPwcseBoAgAkrdevU3Sd099bu3rpP9lv0OAAASUQdAMAQRB0AwABE3Rqq6mlVdfai5wAAmJWoW9v1k9xm0UMAAMxK1K2hu5/f3bXoOQAAZiXqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAYg6gAABiDqAAAGIOoAAAawZdED7MlufehlOfnkMxc9xlI45iaHLXoE2K36E59d9AhL49RDr7XoEZbExYseYOlZqQMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGICoAwAYgKgDABiAqAMAGMDCo66qTqmqnn7dbcGznLtilusvchYAgPVYeNRN/WWSGyc5I0lWhNXqrydN9x89vX12VW1Z+UTTMHvWitsro/GHVfX1qvrHqnpkVdWqOe6S5KHz/VMBAHa/zRJ1l3X3Bd19xYptT8gk9FZ+nbTqcTdL8vgZnn9bNP5skl9J8qEkr0rytqrae9uduvubSb6zq38EAMCibNn5XRbmP7r7gp3c52VJnl9Vb+ju7+3gfpeteK7zk3y0qj6c5B+TPDqT6AMA2GNtlpW6XfXyJFckeeZ6H9jdJyf5dJxuBQAGsJmj7vVVdemqrzuuus8PkjwvybOr6ga78DvOyuSU7Myq6tiq+lhVfeyb375qF34lAMDut5mj7tlJDlv19fk17vf6JOdmEnfrVUl6PQ/o7hO6e2t3b73BwXvv/AEAABtgM7+m7oLu/uLO7tTdV1fVc5L836r603X+jtslOWeXpgMA2EQ280rdzLr7XUlOS/JHsz6mqo5Jcockb57XXAAAG2Uzr9T9RFXdaNW2S7v70u3c//eSfDiTCydWO2D6XFsyeWuTB0zv//Ykb9hN8wIALMxmXql7dZKvr/p6zvbu3N0fzWTVbb81dj92+vhzkrwjyZFJnpTkId3tagcAYI+3KVfqunv1Jz2s3n9KJhc5rN7+sCQPW7Xt6N05GwDAZrRZVuqOnb5lyV0WOURVfTbJuxc5AwDArtgMK3WPSHKt6c/nLXKQTF5rt8/0Zx8XBgDsMRYedd391UXPsE13f2XRMwAA7IrNcvoVAIBrQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAygunvRM+yxquqbSb6y6DnW6fpJvrXoIZaEY70xHOeN41hvHMd6Y+ypx/lm3X2D1RtF3ZKpqo9199ZFz7EMHOuN4ThvHMd64zjWG2O04+z0KwDAAEQdAMAARN3yOWHRAywRx3pjzOU4V9Wlc3jOm1fVw9e7b8bnPrqqjtr16Wbin+mN41hvjKGOs9fUAayhqi7t7mvv5uc8OsmzuvtB69k343M/P8ml3f3iazIjsOeyUgewA9MVsFOq6s1VdXZV/VVV1XTfuVX1wqr6dFWdXlWHTLefWFW/seI5tq36HZ/kHlV1ZlU9Y9Wv+ursoxgAAAHASURBVLF9VbV3Vb2oqj5aVZ+qqidOn+sZVfXa6c93rKrPVNXtkjwpyTOmj7/HfI8KsBltWfQAAHuAOye5fZKvJTktyS8k+cB030XdfceqenSSlybZ0Urbc7L91bgf21dVx06f+y5VtV+S06rqn5L8aZJTquohSf5Hkid291lV9cpYqYOlZqUOYOdO7+7zu/vqJGcmufmKfW9a8f3I3fg7fznJo6vqzCQfSXJwkltNZ3hMktcnObW7T9uNvxPYg1mpA9i5y1f8fFV+/L+dvcbPV2b6f5qraq8k++7C76wk/7W7T15j362SXJrkJrvwvMCgrNQBXDMPW/H9Q9Ofz01y+PTnX0myz/TnS5JcZzvPs3rfyUmeXFX7JElV3bqqDqyq6yZ5WZJ7Jjl4xWv3dvTcwBIQdQDXzH+qqk8leXqSbRc/vDrJvarqk5mckv3edPunklxVVZ9c40KJ1ftek+SsJB+vqs8keVUmK4QvSfKK7v5CkscnOb6qbpjkHUke4kIJWF7e0gRgF1XVuUm2dvee+NmRwGCs1AEADMBKHQDAAKzUAQAMQNQBAAxA1AEADEDUAQAMQNQBAAxA1AEADOD/AdlTudS6UsuWAAAAAElFTkSuQmCC\n"},"metadata":{"needs_background":"light"}}]}]}