igbo_transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Jvl0f3umdSIh"
   },
   "source": [
    "# Set up environment and load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "ZNhMX1pmOi3c"
   },
   "outputs": [],
   "source": [
    "# install datasets library for hugging face and cohere\n",
    "# !pip install --upgrade pip -qU\n",
    "!pip install gspread oauth2client -qU                                    \n",
    "# !pip install gspread pandas matplotlib datasets oauth2client -qU         #uncomment if running on runpod.io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "icyCPlp6O6tc"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import gspread\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from torch.nn import functional as F\n",
    "from datasets import load_dataset\n",
    "# from google.colab import auth\n",
    "# from google.auth import default\n",
    "from oauth2client.service_account import ServiceAccountCredentials\n",
    "\n",
    "# auth.authenticate_user()                                                             #helps colab identify and authenticate the current user\n",
    "# creds, _ = default()                                                                 #get credentials to be used to access and use Google's services and APIs\n",
    "# scope = ['https://www.googleapis.com/auth/drive', 'https://spreadsheets.google.com/feeds']\n",
    "scope = ['https://www.googleapis.com/auth/drive']\n",
    "creds = ServiceAccountCredentials.from_json_keyfile_name('service_account_credentials.json', scope)\n",
    "gc = gspread.authorize(creds)                                                         #log into Google API using credentials gotten above. function returns a gspread object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CsvTLJd9O80y",
    "outputId": "e7b831a2-b2d3-4efe-c7aa-5224ec627951"
   },
   "outputs": [],
   "source": [
    "dataset = load_dataset(\"ccibeekeoc42/english_to_igbo\")                                 #load the eng-igbo dataset from huggingface\n",
    "training_data = dataset[\"train\"].select(range(522322))                                 #select first 522322 rows from training set to serve as context database\n",
    "validation_data = dataset[\"test\"].select(range(3296))                                  #select first 1000 rows from validation set to serve as validation database\n",
    "training_data_df = pd.DataFrame(training_data, columns = [\"English\", \"Igbo\"])          #create context dataframe from context data\n",
    "validation_data_df = pd.DataFrame(validation_data, columns = [\"English\", \"Igbo\"])      #create context dataframe from validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "Q9oiZ7ZMa4OV"
   },
   "outputs": [],
   "source": [
    "#merge huggingface training and validation data into one lists\n",
    "hf_igbo_text_data = training_data_df['Igbo'].tolist() + validation_data_df['Igbo'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "GTP5Yb16O9tC"
   },
   "outputs": [],
   "source": [
    "def get_list_from_sheet(workbook_name, sheet_name, column_number):\n",
    "    sheet = gc.open(workbook_name).worksheet(sheet_name)\n",
    "    sentence_list = sheet.col_values(column_number)\n",
    "    sentence_list = sentence_list[1:]\n",
    "    return sentence_list\n",
    "\n",
    "tiny_stories_test_igbo = get_list_from_sheet('Tiny_Stories_Test', 'Igbo', 3)\n",
    "tiny_stories_test2_igbo = get_list_from_sheet('Tiny_Stories_Test2', 'Igbo', 3)\n",
    "# bbc_news_igbo = get_list_from_sheet('BBC_News', 'Igbo', 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Cr1gTTMHc7no",
    "outputId": "f96038e9-f61e-4d0c-dd4f-086b66b6c86f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2496"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# igbo_raw_text_data = hf_igbo_text_data + tiny_stories_test_igbo + tiny_stories_test2_igbo + bbc_news_igbo\n",
    "igbo_raw_text_data = tiny_stories_test_igbo + tiny_stories_test2_igbo\n",
    "random.shuffle(igbo_raw_text_data)\n",
    "len(igbo_raw_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"ụbọchị na-ekpo ọkụ. Enwere m ike ịhụ anyanwụ ka ọ dị elu na mbara igwe wee nwee ahụ ọkụ ya. Obi tọrọ m ụtọ n’ihi na ezinụlọ m na-eme oriri. Ahụrụ m ọtụtụ mmadụ n'èzí. Ha niile nọ na-ekwu okwu, na-eri ihe, na-aṅụkwa ihe ọṅụṅụ.\\n\\nM gbara ọsọ gburugburu, na-anụ ụtọ ụbọchị ahụ. Ebe ọ bụla m lere anya, ahụrụ m ndị mmadụ ka ha na-achị ọchị ma na-amụmụ ọnụ ọchị. ụfọdụ n’ime ha makụrụ m ma nye m swiit. Ọ mara mma nke ukwuu.\\n\\nMma ntọn̄ọ ndikpa mba, ntre mma nsụhọde ntie n̄kụt usọrọ oro. Ahụrụ m ọtụtụ ndị okenye ka ha na-akparịta ụka ma na-eri nri. Ahụkwara m ụfọdụ ụmụaka dị ka m ka ha na-egwu egwu na-agbagharịkwa gburugburu. Onye ọ bụla nọ na-ekpori ndụ.\\n\\nna mberede, anụrụ m otu onye na-akpọ aha m. Ọ bụ mama m na-arịọ m ka m bata n'ime maka nri abalị. M wee sị ndị enyi m niile nke ọma wee soro ya. Ka m lere anya azụ, m mụmụrụ ọnụ ọchị. Ememe ahụ dị ụtọ nke ukwuu.\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(igbo_raw_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "B9rz4bNrdDRx"
   },
   "outputs": [],
   "source": [
    "no_of_rows_for_vocab = 2496                                                                 #number of rows for building vocab \n",
    "no_of_rows_for_training = int(0.9*len(igbo_raw_text_data))                                 #first 90% will be train, rest val\n",
    "no_of_rows_for_validation = len(igbo_raw_text_data) - no_of_rows_for_training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "4fUuBkKyQRwe"
   },
   "outputs": [],
   "source": [
    "igbo_vocab_data = igbo_raw_text_data[:no_of_rows_for_vocab]\n",
    "igbo_vocab_data = \"\\n\".join(igbo_vocab_data)\n",
    "igbo_vocab_data_file_path = \"igbo_vocab_data.txt\"\n",
    "with open(igbo_vocab_data_file_path, \"w\", encoding='utf-8') as file:\n",
    "    file.write(igbo_vocab_data)\n",
    "\n",
    "igbo_training_data = igbo_raw_text_data[:no_of_rows_for_training]\n",
    "igbo_training_data = \"\\n\".join(igbo_training_data)\n",
    "igbo_training_data_file_path = \"igbo_training_data.txt\"\n",
    "with open(igbo_training_data_file_path, \"w\", encoding='utf-8') as file:\n",
    "    file.write(igbo_training_data)\n",
    "\n",
    "igbo_validation_data = igbo_raw_text_data[no_of_rows_for_training:no_of_rows_for_training + no_of_rows_for_validation]\n",
    "igbo_validation_data = \"\\n\".join(igbo_validation_data)\n",
    "igbo_validation_data_file_path = \"igbo_validation_data.txt\"\n",
    "with open(igbo_validation_data_file_path, \"w\", encoding='utf-8') as file:\n",
    "    file.write(igbo_validation_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "P46tkTcutoUK"
   },
   "source": [
    "# Set up sentencepiece tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "ZthX3K1atr9G"
   },
   "outputs": [],
   "source": [
    "!pip install sentencepiece -q\n",
    "import sentencepiece as spm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "E1Rv61WU0vAH"
   },
   "outputs": [],
   "source": [
    "# train a sentencepiece model on it: settings here are (best effort) those used for training Llama 2\n",
    "import os\n",
    "vocab_size = 400\n",
    "\n",
    "options = dict(\n",
    "\n",
    "      # input spec\n",
    "      input = \"igbo_vocab_data.txt\",\n",
    "      input_format = \"text\",\n",
    "\n",
    "      # output spec\n",
    "      model_prefix = \"igbo_vocab\",              # output filename prefix\n",
    "\n",
    "      # algorithm spec: BPE algorithm\n",
    "      model_type = \"bpe\",\n",
    "\n",
    "      vocab_size = vocab_size,\n",
    "\n",
    "      # normalization\n",
    "      normalization_rule_name = \"identity\",  # ew, turn off normalization\n",
    "\n",
    "      remove_extra_whitespaces = False,\n",
    "      input_sentence_size = 200000000,       # max number of training sentences\n",
    "      max_sentence_length = 4192,            # max number of bytes per sentence\n",
    "      seed_sentencepiece_size = 1000000,\n",
    "      shuffle_input_sentence = True,\n",
    "\n",
    "      # rare word treatment\n",
    "      character_coverage = 0.99995,\n",
    "      byte_fallback = True,\n",
    "\n",
    "      # merge rules\n",
    "      split_digits = True,\n",
    "      split_by_unicode_script = True,\n",
    "      split_by_whitespace = True,\n",
    "      split_by_number = True,\n",
    "      max_sentencepiece_length = 16,\n",
    "      add_dummy_prefix = True,\n",
    "      allow_whitespace_only_pieces = True,\n",
    "\n",
    "      # special tokens\n",
    "      unk_id = 0,                           # the UNK token MUST exist\n",
    "      bos_id = 1,                           # the others are optional, set to -1 to turn off\n",
    "      eos_id = 2,\n",
    "      pad_id = -1,\n",
    "\n",
    "      # systems\n",
    "      num_threads = os.cpu_count(),          # use ~all system resources\n",
    ")\n",
    "\n",
    "spm.SentencePieceTrainer.train(**options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "nwm3xpvP1eY2"
   },
   "outputs": [],
   "source": [
    "sp = spm.SentencePieceProcessor()\n",
    "sp.load('igbo_vocab.model')\n",
    "vocab = {idx:sp.id_to_piece(idx) for idx in range(vocab_size)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "vShIsBkV26A0"
   },
   "outputs": [],
   "source": [
    "with open(\"igbo_vocab_sentencepiece.json\", \"w\") as f:\n",
    "    json.dump(vocab, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "OXD1yfWJ8H2H"
   },
   "outputs": [],
   "source": [
    "def encode(text):\n",
    "    return sp.EncodeAsIds(text)\n",
    "\n",
    "def decode(ids):\n",
    "    return sp.DecodeIds(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "fPCUc5DAwTZp"
   },
   "outputs": [],
   "source": [
    "igbo_raw_text_data_enc = encode(igbo_raw_text_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VhEVUUjS-iW-",
    "outputId": "b962010f-d027-4cad-d947-4aea4606f450"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median token length: 540.0\n",
      "Average token length: 585.4855769230769\n",
      "Max token length: 2633\n"
     ]
    }
   ],
   "source": [
    "#inspect the tokens in the corpus\n",
    "igbo_raw_text_data_token_len = [len(x) for x in igbo_raw_text_data_enc]\n",
    "longest_token_pos = np.argmax(igbo_raw_text_data_token_len)\n",
    "sorted_igbo_raw_text_data_token_len = sorted(igbo_raw_text_data_token_len, reverse=True)\n",
    "\n",
    "print(f\"Median token length: {np.median(igbo_raw_text_data_token_len)}\")\n",
    "print(f\"Average token length: {np.average(igbo_raw_text_data_token_len)}\")\n",
    "print(f\"Max token length: {np.max(igbo_raw_text_data_token_len)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 472
    },
    "id": "thy4ZQvyCjOb",
    "outputId": "8a722a5c-b49d-4abf-c010-dc2d451931b5"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHFCAYAAAAUpjivAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABTHUlEQVR4nO3deVxUVeM/8M/AwLCIJCAMCCLmRoJpYi5ZoigumVsu5b70zXJJU9PMMuwx91zK0urxwS3TFjEfc8NUUnFXcklNC8UFJAjZhGGZ8/vDH/dxmAGGYWCGy+f9es2r7rnn3jlnnOXDuffcqxBCCBARERHJlI2lG0BERERUmRh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHaIiIhI1hh2iIiISNYYdoiIiEjWGHYIAKBQKIx6HD582Kh9TZo0qfIbbYTQ0FAEBQVZuhkl2r17NyIiIgyuq8jrePPmTSgUCixbtszg+mXLlkGhUODmzZtS2ejRo9GgQYNyPc+9e/cQERGBuLg4k9opJ7/88gtCQkLg7OwMhUKBHTt2VMrzrFq1CgqFAnv37i2xztdffw2FQoHt27dXShvKMnr0aNSqVcvk7YUQ2Lp1K55//nl4enrCwcEBvr6+6N69O/7973/r1FUoFCV+hizp999/h0qlgkKhwJkzZ/TWJycnY/To0fDw8ICTkxPat2+PX375xeC+Dhw4gPbt28PJyQkeHh4YPXo0kpOTK7sLssKwQwCA48eP6zx69eoFR0dHvfJnnnnG0k2Vld27d2PevHmWbgYA4IMPPkBUVFS5trl37x7mzZtX48OOEAKDBw+GnZ0ddu7ciePHj6NTp06V8lzDhw+HSqXCf/7znxLrREZGom7dunjppZcqpQ2Vbfbs2Xj11VcRGBiIf//739izZw/mz58PLy8v/PTTTzp1jx8/jtdee81CLTWssLAQY8eOhYeHh8H1Go0GYWFh+OWXX7Bq1Sr89NNP8PLyQo8ePRATE6NTNyYmBj179pT6vmrVKhw4cABhYWHQaDRV0R1ZUFq6AWQd2rVrp7Nct25d2NjY6JWTfD355JOWbkK5CSGQm5sLR0dHi7bj3r17+Oeff9C/f3+EhYWZZZ85OTlwcHCAQqHQKXd3d0ffvn2xY8cOpKamwt3dXWf91atXcfz4cUyfPh12dnZmaUtVysnJwcqVKzFy5Eh89dVXOutGjx4NrVarU2aN31ErVqzAnTt3MGvWLEyZMkVv/bp163Dp0iXExsaiffv2AIDOnTvj6aefxsyZM3Hy5Emp7jvvvIMmTZrghx9+gFL56Cc7ICAAzz33HP7zn//gzTffrJpOVXMc2SGj/fPPP5gwYQLq1asHe3t7NGzYEHPmzCnzrwshBN577z3Y2dnh66+/lsq3bduG9u3bw9nZGbVq1UL37t1x/vx5nW2LhsNv3LiBXr16oVatWvDz88P06dPN+leNudty584dDBw4EC4uLnjiiScwbNgwnD59GgqFAuvXr5f29/nnnwPQPYz4+KElANi0aRMCAwPh5OSEp59+Grt27TJbv4v3r/hhrO+//x5t27aFq6srnJyc0LBhQ4wdOxYAcPjwYbRp0wYAMGbMGKn9jx9S2LlzpzT87uLigm7duuH48eN6z/3TTz+hRYsWUKlUaNiwIVatWoWIiAi9H/qiQ3tr165FYGAgVCoVNmzYAACYN28e2rZtCzc3N9SuXRvPPPMM1q1bh+L3Om7QoAF69+6NXbt2oVWrVnB0dERgYKD0uq5fvx6BgYFwdnbGs88+a/AQxOMiIiLg6+sLAJg1axYUCoXO63j06FGEhYXBxcUFTk5O6NChA37++Wedfaxfvx4KhQL79+/H2LFjUbduXTg5OZX4Hh83bhzy8vKwZcsWvXWRkZEAIP07Gfu51Wq1+Oyzz9CyZUs4OjriiSeeQLt27bBz506pzrZt2xAeHg5vb2/pdXv33XeRnZ1tsJ2XL19GWFgYnJ2dUbduXUyaNAkPHz4s9fXMzs6GRqOBt7e3wfU2Nro/W8Xfcw0aNDDqMPz169cxdOhQeHp6QqVSITAwUPo8VsT169cxd+5cfPHFF6hdu7bBOlFRUWjatKkUdABAqVRi+PDhOHXqFO7evQsAuHv3Lk6fPo0RI0ZIQQcAOnTogCZNmpR7JLZGE0QGjBo1Sjg7O0vLOTk5okWLFsLZ2VksW7ZM7N+/X3zwwQdCqVSKXr166WwLQEycOFEIIURubq545ZVXhIuLi9izZ49U5+OPPxYKhUKMHTtW7Nq1S2zfvl20b99eODs7i8uXL+u0w97eXgQGBoply5aJAwcOiLlz5wqFQiHmzZtXZj86deokmjdvXmodc7clKytLNGrUSLi5uYnPP/9c7Nu3T7z99tsiICBAABCRkZFCCCFu3LghBg4cKACI48ePS4/c3FzpdWzQoIF49tlnxXfffSd2794tQkNDhVKpFH/++WepfYqPjxcAxOLFi0V+fr7eY/HixQKAiI+P1+mfv7+/tBwbGysUCoV45ZVXxO7du8XBgwdFZGSkGDFihBBCiPT0dBEZGSkAiPfff19q/+3bt4UQQnzzzTcCgAgPDxc7duwQ27ZtE61btxb29vbiyJEj0vPs2bNH2NjYiNDQUBEVFSW+//570bZtW9GgQQNR/CsKgKhXr55o0aKF2LJlizh48KC4dOmSEEKI0aNHi3Xr1ono6GgRHR0t/vWvfwlHR0e994m/v7/w9fUVQUFB4ttvvxW7d+8Wbdu2FXZ2dmLu3LniueeeE9u3bxdRUVGiSZMmwsvLSzx8+LDE1/r27dti+/btAoCYPHmyOH78uDh37pwQQojDhw8LOzs70bp1a7Ft2zaxY8cOER4eLhQKhdi6dau0j6LXsV69euL1118Xe/bsET/88IMoKCgw+JyFhYXC399ftGzZUqe8oKBAeHt7i3bt2gkhyve5HTFihFAoFOK1114TP/30k9izZ4/4+OOPxapVq6Q6//rXv8SKFSvEzz//LA4fPizWrl0rAgICROfOnXX2VfRZqV+/vvj444/F/v37RUREhFAqlaJ3794lvpZFGjVqJFxcXMQnn3wirly5IrRabYl1AYgPP/xQWj537pzO5+nYsWMiODhYODs7S5+by5cvC1dXVxEcHCw2btwo9u/fL6ZPny5sbGxERESEzv47deqk9z4siVarFS+88IIYNGiQEOJ//66nT5/WqadWq6U6j9u1a5cAIPbt2yeEEGLv3r0CgPj555/16g4cOFB4e3sb1S4SgmGHDCoedtauXSsAiO+++06nXtGP5v79+6WyorCTmpoqOnbsKOrVqyfi4uKk9QkJCUKpVIrJkyfr7CszM1Oo1WoxePBgnXYYet5evXqJpk2bltmPssJOZbTl888/FwB0wp0QQowfP14n7AghxMSJE0v8IgUgvLy8REZGhlSWlJQkbGxsxMKFC0vutPhf2CnrUVrYWbZsmQAgHjx4UOLznD59Wq9PQjz6Mfbx8RHBwcGisLBQKs/MzBSenp6iQ4cOUlmbNm2En5+f0Gg0OvXc3d0Nhh1XV1fxzz//lNr/wsJCkZ+fLz766CPh7u6u82Pp7+8vHB0dxZ07d6SyuLg4AUB4e3uL7OxsqXzHjh0CgNi5c2epz1f0ei9dulSnvF27dsLT01NkZmZKZQUFBSIoKEj4+vpK7Sr6URw5cmSpz/O4Dz/8UACQgpUQQvz3v/8VAMTXX38thDD+c/vrr78KAGLOnDlGP79WqxX5+fkiJiZGABC//fabtK7os/J4UBLi0R8WAMTRo0dL3fepU6dE/fr1pfepi4uL6N27t9i4caNe8CkedoqbNGmSUCqVYvfu3VJZ9+7dha+vr0hPT9er6+DgoPP+6tKli7C1tS21vUU+++wzUadOHZGUlCSEKDns2NnZifHjx+ttHxsbKwCILVu2CCH+9wfD8ePH9eq+/vrrwt7e3qh2kRA8jEVGOXjwIJydnTFw4ECd8tGjRwOA3iyC+Ph4tG/fHhkZGThx4gSefvppad2+fftQUFCAkSNHoqCgQHo4ODigU6dOejO+FAqF3omWLVq0wK1btyrcr8poS0xMDFxcXNCjRw+deq+++mq529e5c2e4uLhIy15eXvD09DS671OmTMHp06f1HobOIyiu6BDV4MGD8d1330lD68a4du0a7t27hxEjRugcdqhVqxZefvllnDhxAg8fPkR2djbOnDmDfv36wd7eXqdeSSfXdunSBXXq1NErP3jwILp27QpXV1fY2trCzs4Oc+fORWpqqt7MlZYtW6JevXrScmBgIIBHs/ecnJz0yk15r2VnZ+PkyZMYOHCgzswkW1tbjBgxAnfu3MG1a9d0tnn55ZeN3v+YMWNgY2Ojc6JyZGQknJ2dMWTIEADGf2737NkDAJg4cWKpz/nXX39h6NChUKvV0mtcdCL2lStX9OoPGzZMZ3no0KEAgEOHDpX6PG3atMGNGzewd+9evPfee9JMpZEjR6JPnz56hyZLsmjRIqxevRpr165Fz549AQC5ubn45Zdf0L9/fzg5Oel87nv16oXc3FycOHFC2scvv/yCgoKCMp/r1q1bmD17NpYuXQovL68y6xc/RFvaupLqlrYP0sUTlMkoqampUKvVeh8uT09PKJVKpKam6pSfOnUKKSkp+Pjjj6XzGYrcv38fwP9+TIsrfkzeyckJDg4OOmUqlQq5ubkm9aWy25Kammrwy86YL8Diip98WvR8OTk5Rm3v6+uLkJAQvXJjLiHwwgsvYMeOHfj0008xcuRIaDQaNG/eHHPmzCkzuBW9Hwydd+Hj4wOtVou0tDSIR6PL5Xq9DO3z1KlTCA8PR2hoKL7++mv4+vrC3t4eO3bswMcff6z3erm5ueksFwWtkspNea8V9a+k1wCA3uempPNUDPH390dYWBi2bNmCZcuWITMzE7t27cLQoUOlgGzs5/bvv/+Gra0t1Gp1ic+XlZWF559/Hg4ODpg/fz6aNGkCJycn3L59GwMGDNB7jZVKpd77t2j/xfttiJ2dHbp3747u3btL2wwcOBC7du3Cnj170KtXr1K337x5M9577z3MnTsX48aNk8pTU1NRUFCAzz77DJ999pnBbVNSUspsX3ETJ05EUFAQXn75ZTx48AAApPOTsrKykJ6eDldXVwCPPteGXoN//vkHwP/eh0WvX0l1i79fqWQMO2QUd3d3nDx5EkIInS/O5ORkFBQU6E2xHDJkCNRqNebMmQOtVov3339fWldU94cffoC/v3/VdKAEldEWd3d3nDp1Sq88KSnJLPuvSn379kXfvn2h0Whw4sQJLFy4EEOHDkWDBg10Tq4sruhLOjExUW/dvXv3YGNjgzp16kjvp6LQ+biSXi9Df81u3boVdnZ22LVrl04Yraxr3RijTp06sLGxKfE1AKD3uSnvX+rjxo1DdHQ0fvrpJ9y7dw95eXk6P+zGfm7r1q2LwsJCJCUllRi4Dh48iHv37uHw4cM60+qLftiLKygo0JstVvRvaijEl8Xd3R1Tp07F4cOHcenSpVLDTnR0NMaOHYvRo0frXdqhTp060uhaSSNZAQEB5W7fpUuXcOvWLYOjjp07d4arq6v0WgUHB+PixYt69YrKiq4NVvTfixcv6vX34sWLVn0NMWvDw1hklLCwMGRlZen9eGzcuFFaX9z777+PlStXYu7cuZg9e7ZU3r17dyiVSvz5558ICQkx+KgqldGWTp06ITMzUzo0UGTr1q16dVUqFQAYPVJjKSqVCp06dcLixYsBQJqpVlL7mzZtinr16mHLli06hxyys7Px448/SjO0nJ2dERISgh07diAvL0+ql5WVVa5ZZwqFAkqlEra2tlJZTk4ONm3aVP7OmomzszPatm2L7du367w+Wq0Wmzdvhq+vL5o0aVKh5+jXrx/c3d3xn//8B5GRkWjSpAk6duworTf2c1t0iGfNmjUlPldRWCr6Ny/y5ZdflrjNN998o7NcNHssNDS0xG3y8/NLHPkpOlRWNDJmSFxcHF5++WV06dJFb+o68Gh0tnPnzjh//jxatGhh8DNvShjbunUrDh06pPOYNWsWAGDt2rU67+f+/fvj6tWrOlPMCwoKsHnzZrRt21bqX7169fDss89i8+bNKCwslOqeOHEC165dw4ABA8rdzpqKIztklJEjR+Lzzz/HqFGjcPPmTQQHB+Po0aNYsGABevXqha5duxrcbsqUKahVqxZef/11ZGVl4dNPP0WDBg3w0UcfYc6cOfjrr7/Qo0cP1KlTB/fv38epU6fg7Oxs1gvtZWRk4IcfftArr1u3Ljp16mT2towaNQorVqzA8OHDMX/+fDRq1Ah79uzBvn37AOgeGgsODgYALF68GD179oStrS1atGihc/6KpcydOxd37txBWFgYfH198eDBA6xatUrnPI0nn3wSjo6O+OabbxAYGIhatWrBx8cHPj4+WLJkCYYNG4bevXtj/Pjx0Gg0WLp0KR48eIBFixZJz/PRRx/hxRdfRPfu3TFlyhQUFhZi6dKlqFWrljSsX5YXX3wRy5cvx9ChQ/H6668jNTUVy5Yt0/thrmoLFy5Et27d0LlzZ8yYMQP29vb44osvcOnSJXz77bcVPudCpVJh2LBh+OyzzyCE0HldAeM/t88//zxGjBiB+fPn4/79++jduzdUKhXOnz8PJycnTJ48GR06dECdOnXwxhtv4MMPP4SdnR2++eYb/PbbbwbbZm9vj08++QRZWVlo06YNYmNjMX/+fPTs2VMnkBWXnp6OBg0aYNCgQejatSv8/PyQlZWFw4cPY9WqVQgMDCzxRz4jI0O6IOqMGTP0Lhvw1FNPoXbt2li1ahU6duyI559/Hm+++SYaNGiAzMxM3LhxA//9739x8OBBaZuwsDDExMSUed6Ooev9FF1GonXr1jp/OI0dOxaff/45Bg0ahEWLFsHT0xNffPEFrl27hgMHDujsY/HixejWrRsGDRqECRMmIDk5Ge+++y6CgoIwZsyYUttEj7HYqdFk1YrPxhJCiNTUVPHGG28Ib29voVQqhb+/v5g9e7Y0VboIHpt6XuTbb78VSqVSjBkzRpqds2PHDtG5c2dRu3ZtoVKphL+/vxg4cKA4cOBAqe0Q4n8zUcpSNG3U0KNTp05SPXO3JSEhQQwYMEDUqlVLuLi4iJdfflns3r1bABA//fSTVE+j0YjXXntN1K1bVygUCp0ZUoZeRyEezSYaNWpUqf0uaXZQkaVLl5Y5G2vXrl2iZ8+eol69esLe3l54enqKXr166UwbF+LRv22zZs2EnZ2d3syYHTt2iLZt2woHBwfh7OwswsLCxLFjx/TaExUVJYKDg6XpyosWLRJvvfWWqFOnjk69kl4TIYT4z3/+I5o2bSpUKpVo2LChWLhwoVi3bp1eP/39/cWLL76ot72hfZf1OhpT78iRI6JLly7C2dlZODo6inbt2on//ve/OnVKmrVjjN9++00AELa2tuLevXt664393BYWFooVK1aIoKAgYW9vL1xdXUX79u112hobGyvat28vnJycRN26dcVrr70mzp07pzcjr+izcuHCBREaGiocHR2Fm5ubePPNN0VWVlap/dFoNGLZsmWiZ8+eon79+kKlUgkHBwcRGBgoZs6cKVJTU3XqP/6eK2sW4qFDh6Tt4uPjxdixY0W9evWEnZ2dqFu3rujQoYOYP3++zv7LM/W8uNL+XZOSksTIkSOFm5ubcHBwEO3atRPR0dEG97N//37Rrl074eDgINzc3MTIkSPF/fv3TWpTTaUQwsjT2omoQhYsWID3338fCQkJeidtk678/HxpxtT+/fst3RwiquZ4GIuoEqxevRoA0KxZM+Tn5+PgwYP49NNPMXz4cAYdA8aNG4du3brB29sbSUlJWLt2La5cuYJVq1ZZumlEJAMMO0SVwMnJCStWrMDNmzeh0WhQv359zJo1S2dWGv1PZmYmZsyYgb///ht2dnZ45plnsHv37hLPBSMiKg8exiIiIiJZs+jU8zVr1qBFixaoXbs2ateujfbt2+tM1xVCICIiAj4+PnB0dERoaCguX76ssw+NRoPJkyfDw8MDzs7O6NOnD+7cuVPVXSEiIiIrZdGw4+vri0WLFuHMmTM4c+YMunTpgr59+0qBZsmSJVi+fDlWr16N06dPQ61Wo1u3bsjMzJT2MXXqVERFRWHr1q04evQosrKy0Lt3b51rEhAREVHNZXWHsdzc3LB06VKMHTsWPj4+mDp1qnRhJo1GAy8vLyxevBjjx49Heno66tati02bNkn3grl37x78/Pywe/du6TLjREREVHNZzQnKhYWF+P7775GdnY327dsjPj4eSUlJCA8Pl+oUXcU1NjYW48ePx9mzZ5Gfn69Tx8fHB0FBQYiNjS0x7Gg0Gmg0GmlZq9Xin3/+gbu7O2+sRkREVE0IIZCZmQkfHx+9exk+zuJh5+LFi2jfvj1yc3NRq1YtREVF4amnnkJsbCwA/ZsBenl5SXcgTkpKgr29vd69SLy8vEq9D9HChQvNeoVeIiIispzbt2+XelkPi4edpk2bIi4uDg8ePMCPP/6IUaNGISYmRlpffKRFFLuhnSFl1Zk9ezamTZsmLaenp6N+/fqIj49H7dq1TewJERERVaWMjAwEBATAxcWl1HoWDzv29vZo1KgRACAkJASnT5/GqlWrpPN0it+FNzk5WRrtUavVyMvLQ1pams7oTnJyMjp06FDic6pUKoP3zHFzc2PYISIiqiaUykcxpqxBEKu767kQAhqNBgEBAVCr1YiOjpbW5eXlISYmRgoyrVu3hp2dnU6dxMREXLp0qdSwQ0RERDWHRUd23nvvPfTs2RN+fn7IzMzE1q1bcfjwYezduxcKhQJTp07FggUL0LhxYzRu3BgLFiyAk5MThg4dCgBwdXXFuHHjMH36dLi7u8PNzQ0zZsxAcHAwr7xKREREACwcdu7fv48RI0YgMTERrq6uaNGiBfbu3Ytu3boBAGbOnImcnBxMmDABaWlpaNu2Lfbv369zbG7FihVQKpUYPHgwcnJyEBYWhvXr18PW1tZS3SIiIiIrYnXX2bGEjIwMuLq6Ij09nefsEBERVRPG/n5b3Tk7RERERObEsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyZtGws3DhQrRp0wYuLi7w9PREv379cO3aNZ06o0ePhkKh0Hm0a9dOp45Go8HkyZPh4eEBZ2dn9OnTB3fu3KnKrhAREZGVsmjYiYmJwcSJE3HixAlER0ejoKAA4eHhyM7O1qnXo0cPJCYmSo/du3frrJ86dSqioqKwdetWHD16FFlZWejduzcKCwursjtERERkhZSWfPK9e/fqLEdGRsLT0xNnz57FCy+8IJWrVCqo1WqD+0hPT8e6deuwadMmdO3aFQCwefNm+Pn54cCBA+jevXvldYCIiIisnlWds5Oeng4AcHNz0yk/fPgwPD090aRJE/zf//0fkpOTpXVnz55Ffn4+wsPDpTIfHx8EBQUhNja2ahpOREREVsuiIzuPE0Jg2rRp6NixI4KCgqTynj17YtCgQfD390d8fDw++OADdOnSBWfPnoVKpUJSUhLs7e1Rp04dnf15eXkhKSnJ4HNpNBpoNBppOSMjAwBQUFCAgoKCSugdERERmZuxv9lWE3YmTZqECxcu4OjRozrlQ4YMkf4/KCgIISEh8Pf3x88//4wBAwaUuD8hBBQKhcF1CxcuxLx58/TKz5w5A2dnZxN7QERERFWp+Dm+JbGKsDN58mTs3LkTv/76K3x9fUut6+3tDX9/f1y/fh0AoFarkZeXh7S0NJ3RneTkZHTo0MHgPmbPno1p06ZJyxkZGfDz80NISAhq165thh4RERFRZSs6MlMWi4YdIQQmT56MqKgoHD58GAEBAWVuk5qaitu3b8Pb2xsA0Lp1a9jZ2SE6OhqDBw8GACQmJuLSpUtYsmSJwX2oVCqoVCq9cqVSCaXSKvIfERERlcHY32yL/rJPnDgRW7ZswU8//QQXFxfpHBtXV1c4OjoiKysLERERePnll+Ht7Y2bN2/ivffeg4eHB/r37y/VHTduHKZPnw53d3e4ublhxowZCA4OlmZnERERUc1l0bCzZs0aAEBoaKhOeWRkJEaPHg1bW1tcvHgRGzduxIMHD+Dt7Y3OnTtj27ZtcHFxkeqvWLECSqUSgwcPRk5ODsLCwrB+/XrY2tpWZXeIiIjICimEEMLSjbC0jIwMuLq6Ij09nefsEBERVRPG/n5b1XV2iIiIiMyNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZM2iYWfhwoVo06YNXFxc4OnpiX79+uHatWs6dYQQiIiIgI+PDxwdHREaGorLly/r1NFoNJg8eTI8PDzg7OyMPn364M6dO1XZFSIiIrJSFg07MTExmDhxIk6cOIHo6GgUFBQgPDwc2dnZUp0lS5Zg+fLlWL16NU6fPg21Wo1u3bohMzNTqjN16lRERUVh69atOHr0KLKystC7d28UFhZaoltERERkRRRCCGHpRhT5+++/4enpiZiYGLzwwgsQQsDHxwdTp07FrFmzADwaxfHy8sLixYsxfvx4pKeno27duti0aROGDBkCALh37x78/Pywe/dudO/evcznzcjIgKurK9LT01G7du1K7SMRERGZh7G/38oqbFOZ0tPTAQBubm4AgPj4eCQlJSE8PFyqo1Kp0KlTJ8TGxmL8+PE4e/Ys8vPzder4+PggKCgIsbGxBsOORqOBRqORljMyMgAABQUFKCgoqJS+ERERkXkZ+5ttNWFHCIFp06ahY8eOCAoKAgAkJSUBALy8vHTqenl54datW1Ide3t71KlTR69O0fbFLVy4EPPmzdMrP3PmDJydnSvcFyIiIqp8j5/2UhqrCTuTJk3ChQsXcPToUb11CoVCZ1kIoVdWXGl1Zs+ejWnTpknLGRkZ8PPzQ0hICA9jERERVRNFR2bKYhVhZ/Lkydi5cyd+/fVX+Pr6SuVqtRrAo9Ebb29vqTw5OVka7VGr1cjLy0NaWprO6E5ycjI6dOhg8PlUKhVUKpVeuVKphFJpFS8JERERlcHY32yLzsYSQmDSpEnYvn07Dh48iICAAJ31AQEBUKvViI6Olsry8vIQExMjBZnWrVvDzs5Op05iYiIuXbpUYtghIiKimsOiwxgTJ07Eli1b8NNPP8HFxUU6x8bV1RWOjo5QKBSYOnUqFixYgMaNG6Nx48ZYsGABnJycMHToUKnuuHHjMH36dLi7u8PNzQ0zZsxAcHAwunbtasnuERERkRWwaNhZs2YNACA0NFSnPDIyEqNHjwYAzJw5Ezk5OZgwYQLS0tLQtm1b7N+/Hy4uLlL9FStWQKlUYvDgwcjJyUFYWBjWr18PW1vbquoKERERWSmrus6OpfA6O0RERNWPsb/fvDcWERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyRrDDhEREckaww4RERHJGsMOERERyZpJYSc+Pt7c7SAiIiKqFCaFnUaNGqFz587YvHkzcnNzzd0mIiIiIrMxKez89ttvaNWqFaZPnw61Wo3x48fj1KlT5m4bERERUYWZFHaCgoKwfPly3L17F5GRkUhKSkLHjh3RvHlzLF++HH///be520lERERkkgqdoKxUKtG/f3989913WLx4Mf7880/MmDEDvr6+GDlyJBITE83VTiIiIiKTVCjsnDlzBhMmTIC3tzeWL1+OGTNm4M8//8TBgwdx9+5d9O3b11ztJCIiIjKJ0pSNli9fjsjISFy7dg29evXCxo0b0atXL9jYPMpOAQEB+PLLL9GsWTOzNpaIiIiovEwKO2vWrMHYsWMxZswYqNVqg3Xq16+PdevWVahxRERERBWlEEIISzfC0jIyMuDq6or09HTUrl3b0s0hIiIiIxj7+23SOTuRkZH4/vvv9cq///57bNiwwZRdEhEREVUKk8LOokWL4OHhoVfu6emJBQsWVLhRREREROZiUti5desWAgIC9Mr9/f2RkJBQ4UYRERERmYtJYcfT0xMXLlzQK//tt9/g7u5e4UYRERERmYtJYeeVV17BW2+9hUOHDqGwsBCFhYU4ePAgpkyZgldeecXcbSQiIiIymUlTz+fPn49bt24hLCwMSuWjXWi1WowcOZLn7BAREZFVqdDU8z/++AO//fYbHB0dERwcDH9/f3O2rcpw6jkREVH1Y+zvt0kjO0WaNGmCJk2aVGQXRERERJXKpLBTWFiI9evX45dffkFycjK0Wq3O+oMHD5qlcUREREQVZVLYmTJlCtavX48XX3wRQUFBUCgU5m4XERERkVmYFHa2bt2K7777Dr169TJ3e4iIiIjMyqSp5/b29mjUqJG520JERERkdiaFnenTp2PVqlXgPUSJiIjI2pl0GOvo0aM4dOgQ9uzZg+bNm8POzk5n/fbt283SOCIiIqKKMinsPPHEE+jfv7+520JERERkdiaFncjISHO3g4iIiKhSmHTODgAUFBTgwIED+PLLL5GZmQkAuHfvHrKysszWOCIiIqKKMmlk59atW+jRowcSEhKg0WjQrVs3uLi4YMmSJcjNzcXatWvN3U4iIiIik5g0sjNlyhSEhIQgLS0Njo6OUnn//v3xyy+/mK1xRERERBVl8mysY8eOwd7eXqfc398fd+/eNUvDiIiIiMzBpJEdrVaLwsJCvfI7d+7AxcXF6P38+uuveOmll+Dj4wOFQoEdO3borB89ejQUCoXOo127djp1NBoNJk+eDA8PDzg7O6NPnz64c+eOKd0iIiIiGTIp7HTr1g0rV66UlhUKBbKysvDhhx+W6xYS2dnZePrpp7F69eoS6/To0QOJiYnSY/fu3Trrp06diqioKGzduhVHjx5FVlYWevfubTCMERERUc1j0mGsFStWoHPnznjqqaeQm5uLoUOH4vr16/Dw8MC3335r9H569uyJnj17llpHpVJBrVYbXJeeno5169Zh06ZN6Nq1KwBg8+bN8PPzw4EDB9C9e3fjO0VERESyZFLY8fHxQVxcHL799lucO3cOWq0W48aNw7Bhw3ROWDaHw4cPw9PTE0888QQ6deqEjz/+GJ6engCAs2fPIj8/H+Hh4TptCwoKQmxsbIlhR6PRQKPRSMsZGRkAHk2nLygoMGv7iYiIqHIY+5ttUtgBAEdHR4wdOxZjx441dRdl6tmzJwYNGgR/f3/Ex8fjgw8+QJcuXXD27FmoVCokJSXB3t4ederU0dnOy8sLSUlJJe534cKFmDdvnl75mTNn4OzsbPZ+EBERkfllZ2cbVc+ksLNx48ZS148cOdKU3eoZMmSI9P9BQUEICQmBv78/fv75ZwwYMKDE7YQQUCgUJa6fPXs2pk2bJi1nZGTAz88PISEhqF27tlnaTkRERJWr6MhMWUwKO1OmTNFZzs/Px8OHD2Fvbw8nJyezhZ3ivL294e/vj+vXrwMA1Go18vLykJaWpjO6k5ycjA4dOpS4H5VKBZVKpVeuVCqhVJo82EVERERVyNjfbJNmY6Wlpek8srKycO3aNXTs2LFcJyiXV2pqKm7fvg1vb28AQOvWrWFnZ4fo6GipTmJiIi5dulRq2CEiIqKaw2zDGI0bN8aiRYswfPhwXL161ahtsrKycOPGDWk5Pj4ecXFxcHNzg5ubGyIiIvDyyy/D29sbN2/exHvvvQcPDw/pjuuurq4YN24cpk+fDnd3d7i5uWHGjBkIDg6WZmcRERFRzWbWYza2tra4d++e0fXPnDmDzp07S8tF59GMGjUKa9aswcWLF7Fx40Y8ePAA3t7e6Ny5M7Zt26Zz4cIVK1ZAqVRi8ODByMnJQVhYGNavXw9bW1vzdYyIiIiqLYUQQpR3o507d+osCyGQmJiI1atXw8/PD3v27DFbA6tCRkYGXF1dkZ6ezhOUiYiIqgljf79NGtnp16+fzrJCoUDdunXRpUsXfPLJJ6bskoiIiKhSmBR2tFqtudtBREREVClMmo1FREREVF2YNLLz+AX5yrJ8+XJTnoKIiIjILEwKO+fPn8e5c+dQUFCApk2bAgD++OMP2Nra4plnnpHqlXYVYyIiIqKqYFLYeemll+Di4oINGzZIVy5OS0vDmDFj8Pzzz2P69OlmbSQRERGRqUyael6vXj3s378fzZs31ym/dOkSwsPDy3WtHWvAqedERETVT6VOPc/IyMD9+/f1wk5ycjIyMzNN2SWRVUtISEBKSoq07OHhgfr161uwRUREZCyTwk7//v0xZswYfPLJJ2jXrh0A4MSJE3jnnXdKvRs5UXWUkJCAps0CkZvzUCpzcHTCtatXGHiIiKoBk8LO2rVrMWPGDAwfPhz5+fmPdqRUYty4cVi6dKlZG0hkaSkpKcjNeQj33tNh5+6H/NTbSN31CVJSUhh2iIiqAZPCjpOTE7744gssXboUf/75J4QQaNSoEZydnc3dPiKrYefuB5W6kaWbQURE5VShiwomJiYiMTERTZo0gbOzM0w415mIiIioUpkUdlJTUxEWFoYmTZqgV69eSExMBAC89tprnHZOREREVsWksPP222/Dzs4OCQkJcHJyksqHDBmCvXv3mq1xRERERBVl0jk7+/fvx759++Dr66tT3rhxY9y6dcssDSMiIiIyB5NGdrKzs3VGdIqkpKRApVJVuFFERERE5mJS2HnhhRewceNGaVmhUECr1WLp0qXo3Lmz2RpHREREVFEmHcZaunQpQkNDcebMGeTl5WHmzJm4fPky/vnnHxw7dszcbSQiIiIymUkjO0899RQuXLiAZ599Ft26dUN2djYGDBiA8+fP48knnzR3G4mIiIhMVu6Rnfz8fISHh+PLL7/EvHnzKqNNRERERGZT7pEdOzs7XLp0CQqFojLaQ0RERGRWJh3GGjlyJNatW2futhARERGZnUknKOfl5eHf//43oqOjERISondPrOXLl5ulcUREREQVVa6w89dff6FBgwa4dOkSnnnmGQDAH3/8oVOHh7eoprhy5Yr0/x4eHmXeAT0hIQEpKSnl2oaIiCquXGGncePGSExMxKFDhwA8uj3Ep59+Ci8vr0ppHJE1KsxKAxQKDB8+XCpzcHTCtatXSgwvCQkJaNosELk5D43ehoiIzKNcYaf4Xc337NmD7OxsszaIyNppNVmAEHDvPR127n7IT72N1F2fICUlpcTgkpKSgtych3rbHDlyBIGBgVI9jvYQEZmfSefsFCkefohqEjt3P6jUjUzaxtDoEMDRHiKiylCusKNQKPTOyeE5OkTlV3x0CIBRI0RERFR+5T6MNXr0aOlmn7m5uXjjjTf0ZmNt377dfC0kkjFTRoeIiKh8yhV2Ro0apbNcfAieiCquvLO8iIiodOUKO5GRkZXVDqIaz5RZXkREVLYKnaBMROZjyiwvIiIqG8MOkZXheTxERObFsENkJo+fawPwfBsiImvBsENUTPHbOhQPMcXxmjlERNaNYYfoMYZu61AWXjOHiMi6MewQPab4bR0AIOevM0g/srnMbXmuDRGRdWLYITLg8eCSn3rbwq0hIqKKsLF0A4iIiIgqE8MOERERyRoPYxFVoqKZXGXN6CIiosrDsENUCUqajk5ERFWPYYeoEhSfjm7sjC4iIjI/nrNDVImKZnUpXb0s3RQiohqLYYeIiIhkjYexiKwc77lFRFQxDDtEVor33CIiMg+GHSIrxXtuERGZB8MOkZXjPbeIiCqGJygTERGRrFk07Pz666946aWX4OPjA4VCgR07duisF0IgIiICPj4+cHR0RGhoKC5fvqxTR6PRYPLkyfDw8ICzszP69OmDO3fuVGEviIiIyJpZNOxkZ2fj6aefxurVqw2uX7JkCZYvX47Vq1fj9OnTUKvV6NatGzIzM6U6U6dORVRUFLZu3YqjR48iKysLvXv3RmFhYVV1g4iIiKyYRc/Z6dmzJ3r27GlwnRACK1euxJw5czBgwAAAwIYNG+Dl5YUtW7Zg/PjxSE9Px7p167Bp0yZ07doVALB582b4+fnhwIED6N69e5X1hYiIiKyT1Z6zEx8fj6SkJISHh0tlKpUKnTp1QmxsLADg7NmzyM/P16nj4+ODoKAgqQ4RERHVbFY7GyspKQkA4OWle5l9Ly8v3Lp1S6pjb2+POnXq6NUp2t4QjUYDjUYjLWdkZAAACgoKUFBQYJb2U/Wk1Wphb28PO1sF7GwEAMDe1kanrKzlytoGALS2Ctjb20Or1fK9SkQ1nrHfg1YbdoooFAqdZSGEXllxZdVZuHAh5s2bp1d+5swZODs7m9ZQkoXMzEx88MEHsFd7wsZeCwAo9GuO/JD/lZW1XFnbAIC2oSfyAj5ASkoKTp48aZkXiYjISmRnZxtVz2rDjlqtBvBo9Mbb21sqT05OlkZ71Go18vLykJaWpjO6k5ycjA4dOpS479mzZ2PatGnSckZGBvz8/BASEoLatWubuytUjcTFxeFf//oXvIYvhcqrIQAg+/fLSN2zSiora7mytgEAzf1k3N/8Lxw7dgwtW7a0yGtERGQtio7MlMVqw05AQADUajWio6PRqlUrAEBeXh5iYmKwePFiAEDr1q1hZ2eH6OhoDB48GACQmJiIS5cuYcmSJSXuW6VSQaVS6ZUrlUoolVb7klAVsLGxQV5eHvILBWy0j0YH8wq1OmVlLVfWNgCQXyiQl5cHGxsbvleJqMYz9nvQot+WWVlZuHHjhrQcHx+PuLg4uLm5oX79+pg6dSoWLFiAxo0bo3HjxliwYAGcnJwwdOhQAICrqyvGjRuH6dOnw93dHW5ubpgxYwaCg4Ol2VlERERUs1k07Jw5cwadO3eWlosOLY0aNQrr16/HzJkzkZOTgwkTJiAtLQ1t27bF/v374eLiIm2zYsUKKJVKDB48GDk5OQgLC8P69etha2tb5f0hIiIi62PRsBMaGgohRInrFQoFIiIiEBERUWIdBwcHfPbZZ/jss88qoYVERERU3fGgP9V4CQkJSElJAQBcuXLFwq0hIiJzY9ihGi0hIQFNmwUiN+ehpZtCRESVhGGHarSUlBTk5jyEe+/psHP3Q85fZ5B+ZLOlm1Wmx0egPDw8UL9+fQu2hojIujHsEAGwc/eDSt0I+am3Ld2UUhVmpQEKBYYPHy6VOTg64drVKww8REQlYNghqka0mixACGkkKj/1NlJ3fYKUlBSGHSKiEjDsEFVDRSNRRERUNqu96zkRERGROTDsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGs8QrKRDLAG4MSEZWMYYeoGuONQYmIysawQ1SN8cagRERlY9ihGiUhIQEpKSnS8uOHf6oz3hiUiKhkDDtUYyQkJKBps0Dk5jy0dFOIiKgKMexQjZGSkoLcnIfSIR8AyPnrDNKPbLZwy4iIqDIx7FCN8/ghn/zU2xZuTdUofviOM7aIqCZh2CGSOUOH7zhji4hqEoYdIpkrfviOM7aIqKZh2CGqIThji4hqKoYdkrXHz1WRyzRzYzze15rUbyIiQxh2SLZq4lRzQ1dUJiKq6Rh2SLaKn6tSE6aZF7+iMsDp9UREDDske0XnqtSUaeZAzZxeT0RUEhtLN4CIiIioMjHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGs8QrKRDXU4zcI9fDwQP369S3YGiKiysOwQ1TDGLpZqErlgB9//AHe3t5SGQMQEckFww5RDVP8ZqG5dy7jwcF/o3fv3jr1HBydcO3qFQYeIqr2GHaIaiidG6QWu1N6fuptpO76BCkpKQw7RFTtMewQEQDdO6UTEckJZ2MRERGRrDHsEBERkawx7BAREZGsMewQERGRrDHsEBERkawx7BAREZGsceo5yUpCQgJSUlIA6N4OgYiIai6GHZKNhIQENG0WiNych5ZuChERWRGrPowVEREBhUKh81Cr1dJ6IQQiIiLg4+MDR0dHhIaG4vLlyxZsMVlSSkoKcnMewr33dKhHrYTr88PL3oiIiGTPqsMOADRv3hyJiYnS4+LFi9K6JUuWYPny5Vi9ejVOnz4NtVqNbt26ITMz04ItJksruhKw0tXL0k0hIiIrYPVhR6lUQq1WS4+6desCeDSqs3LlSsyZMwcDBgxAUFAQNmzYgIcPH2LLli0WbjURERFZC6s/Z+f69evw8fGBSqVC27ZtsWDBAjRs2BDx8fFISkpCeHi4VFelUqFTp06IjY3F+PHjS9ynRqOBRqORljMyMgAABQUFKCgoqLzOkFnduXNHOhkZAK5duwZ7e3vY2SpgZyNgb2tT6jIAs9Spqm2qsi1aWwXs7e2h1Wr5mSAiq2Xs95NCCCEquS0m27NnDx4+fIgmTZrg/v37mD9/Pq5evYrLly/j2rVreO6553D37l34+PhI27z++uu4desW9u3bV+J+IyIiMG/ePL3yffv2wdnZuVL6QuaVm5uLkydPQqvV6q2zVzeCjb0TCrPTkJ96u8RlAGapU1XbVGVbtHkPkZd0AyEhIXBxcancf0wiIhNlZ2eje/fuSE9PR+3atUusZ9UjOz179pT+Pzg4GO3bt8eTTz6JDRs2oF27dgAAhUKhs40QQq+suNmzZ2PatGnSckZGBvz8/BASElLqi0XWIy4uDvPmzYN7zylQuvsCAHLjzyH92LfwGr4UKq+GyP79MlL3rCpxGYBZ6lTVNlXZFs39ZNzf/C8cO3YMLVu2rLJ/VyKi8ig6MlMWqw47xTk7OyM4OBjXr19Hv379AABJSUnw9vaW6iQnJ8PLq/QTU1UqFVQqlV65UqmEUlmtXpIay8bGBnl5eRBP1INN3ScBANrkBOTl5SG/UMBGq0BeobbUZQBmqVNV21RlW/ILBfLy8mBjY8PPBBFZLWO/n6z+BOXHaTQaXLlyBd7e3ggICIBarUZ0dLS0Pi8vDzExMejQoYMFW0lERETWxKr/ZJsxYwZeeukl1K9fH8nJyZg/fz4yMjIwatQoKBQKTJ06FQsWLEDjxo3RuHFjLFiwAE5OThg6dKilm06VgFdHrnqPv84eHh6oX7++BVtDRGQaqw47d+7cwauvvoqUlBTUrVsX7dq1w4kTJ+Dv7w8AmDlzJnJycjBhwgSkpaWhbdu22L9/P0+olCFeHblqFWalAQoFhg//34UZHRydcO3qFQYeIqp2rDrsbN26tdT1CoUCERERiIiIqJoGkcU8fnVkO3c/5Px1BulHNlu6WbKl1WQBQkivd37qbaTu+gQpKSkMO0RU7Vh12CEqrujqyPmpty3dlBqh6PUmIqrOqtUJykRERETlxbBDREREssawQ0RERLLGsENERESyxrBDREREssawQ0RERLLGsENERESyxrBDREREssaLChKR0Yrfk4z3yyKi6oBhh4jKZOheWQDvl0VE1QPDDhGVqfi9sgDwfllEVG0w7BCR0XivLCKqjhh2iKhCHj+Ph+fwEJE1YtghIpMYOo+H5/AQkTVi2CEikxQ/j4fn8BCRtWLYIaIK4Xk8RGTteFFBIiIikjWGHSIiIpI1HsYiIrPi7CwisjYMO0RkFoZmZ6lUDvjxxx/g7e0tlTEAEVFVY9ghIrMoPjsr985lPDj4b/Tu3VunHqenE1FVY9ghq5WQkICUlBQA+jegJOtVNDsrP/U2bzFBRFaBYYesUkJCApo2C0RuzkNLN4UqiFPTicjSGHbIKqWkpCA356E0KpDz1xmkH9ls6WaRmfAkZiKqSgw7ZNV0DolQtcdbTBCRJTDsEFGV4S0miMgSGHbIKjx+MjLAE5LlrrTzeIq/FwAe6iKiimHYIYvjychUpKT3Ag91EVFFMOyQxRU/GRkAT0iuoQy9F3ioi4gqimGHrMbjhzZ4QnLNxunqRGROvBEoERERyRrDDhEREckaww4RERHJGs/ZoUrHqcRERGRJDDtUqTiVmIxRdF0lXl+JiCoDww5VqtKmEh85cgSBgYH8gavBDN0+wlTFRxA5ekhERRh2qEo8PpXYnD9wVL0Vv32EqddXMjSCyNFDIirCsENVzlw/cCQfxtzwtbQ7pRcfQSw+emhoGyKqORh2yGJ4R3MyRnnulF70nuLd1YnocQw7RGTVTLlTOu+uTkSPY9ihCuOJoVQVTLmFBG87QUQAww5VEE8MJSIia8ewQxVS0omhPFxAcsWRTKLqh2GHzKL44QJeJI7kiCOZRNUTww6ZFa+hQ1Xl8SBtbKguXq+8ozIcySSqnhh2yKx4DR2qbKYE6pK2KWtUpvghq6KwVNaJz2Ud6pLj/eJ4eI+sGcMOVQpeQ4cqS/FADaDMUG1om7JGZUq6r1tZyjrUJcf7xfHwHlk7hh0qVfG/1jQaDVQqlbTMc3LIUh4fXTE2VBsakXn8Pfz4+/vKlSt693UzZqSyrKs5G9qvoeBlydGf8o7S8PAeWTuGnRqsrC9Tg3+BKmwAoa3KZhJVCoOHtgy8v40JVYbOHyrtas7F91ucsaM/Zf0xYqis+LKhQ2ymjtLI/bpGPFRXfckm7HzxxRdYunQpEhMT0bx5c6xcuRLPP/+8pZtltYz5Mi3+11rRX7Xl/UuXyBqVdH5Zec43M+b8IVPOYyv+2QOgN0KUmJiIlwcOgiY3538bGvpjpHhZseXiQcaY+4wB5vmhNyWsWSpgVOWhusp6XaoqrFnjOWmyCDvbtm3D1KlT8cUXX+C5557Dl19+iZ49e+L3339n6i6BMV+mxf9CLfqr1pTDB0TWqqT3tzHv7fKcP2TKeWyPf9ZKClbG/DFSUp3SDjeVNTKlUjngxx9/gLe3d4mHs4uXP/4DbWpYe/x5i5jyQ1pWoDD2ZrPFX7vy7rf4NpX1ulRVWLPWc9JkEXaWL1+OcePG4bXXXgMArFy5Evv27cOaNWuwcOFCC7eu4gyl5PIOTZfEmC9TIiqduf4AKO36VCWNEBnzx0hpdYo/X/HnNhTocu9cxoOD/0bv3r0N9qPE7xIDP9rlCWslPW/xH/qyvh+NCRTF92nMTDxjDv0X36/BtpjpdSltpL6kEbuKjhiVdk7a489ljoDq4eGBJ554wqhtq33YycvLw9mzZ/Huu+/qlIeHhyM2NtZCrTKfEmeElHNo2hicNk5kGeX5Q8OcMx1NeV7g/4eoUr4rShvxqnBYMyZ4lfH9WKSkQFFWmCtJWYf+S9uvuV+XkkaeyhqxK+9vSUm/UWX9IV3e36iSRqbOnD5l1PbVPuykpKSgsLAQXl5eOuVeXl5ISkoyuI1Go4FGo5GW09PTAQBHjhyBs7MzAMDGxgZare6Ho3hZWcvmqHP9+nUUFuTDvf0g2NZ2BwDkJ91A1u8xqB3yMmxru+stF2akIuPMDuzbtw+NGzcucb92dnYQf/+FgsL//1qk34OdnR2U2nzYFmqgRKFunf+/3tA2JdYxZZvK2i/bwrZYaVu0yX/ATqlE7ZB+Op/pym5v8ed9/PvFmP2W9V1RtB6AVMcc3y+P79c2L8vga1fS9+PjfSzelqLl4vs09LqIf+7Czs4OZ8+eRUZGhs73qin7NffrIrT5Ou0r/p1f2r99ab8lxX+viv9GGXrvFn8uY/ZbvKz48xTt4+bNm4/6KwRKJaq5u3fvCgAiNjZWp3z+/PmiadOmBrf58MMPBQA++OCDDz744EMGj9u3b5eaFar9yI6HhwdsbW31RnGSk5P1RnuKzJ49G9OmTZOWHzx4AH9/fyQkJMDV1bVS22tJGRkZ8PPzw+3bt1G7dm1LN6dS1ZS+sp/yU1P6yn7KjyX6KoRAZmYmfHx8Sq1X7cOOvb09WrdujejoaPTv318qj46ORt++fQ1uo1Kp9KbxAYCrq6vs34wAULt27RrRT6Dm9JX9lJ+a0lf2U36quq/GDFJU+7ADANOmTcOIESMQEhKC9u3b46uvvkJCQgLeeOMNSzeNiIiILEwWYWfIkCFITU3FRx99hMTERAQFBWH37t3w9/e3dNOIiIjIwmQRdgBgwoQJmDBhgknbqlQqfPjhhwYPbclJTeknUHP6yn7KT03pK/spP9bcV4UQZc3XIiIiIqq+bCzdACIiIqLKxLBDREREssawQ0RERLLGsENERESyVuPDzhdffIGAgAA4ODigdevWOHLkiKWbVC4LFy5EmzZt4OLiAk9PT/Tr1w/Xrl3TqSOEQEREBHx8fODo6IjQ0FBcvnxZp45Go8HkyZPh4eEBZ2dn9OnTB3fu3KnKrpTLwoULoVAoMHXqVKlMTv28e/cuhg8fDnd3dzg5OaFly5Y4e/astF4OfS0oKMD777+PgIAAODo6omHDhvjoo4907o9THfv566+/4qWXXoKPjw8UCgV27Nihs95cfUpLS8OIESPg6uoKV1dXjBgxAg8ePKjk3ukqra/5+fmYNWsWgoOD4ezsDB8fH4wcORL37t3T2Ud16GtZ/6aPGz9+PBQKBVauXKlTXh36CRjX1ytXrqBPnz5wdXWFi4sL2rVrh4SEBGm9Vfa1wjenqsa2bt0q7OzsxNdffy1+//13MWXKFOHs7Cxu3bpl6aYZrXv37iIyMlJcunRJxMXFiRdffFHUr19fZGVlSXUWLVokXFxcxI8//iguXrwohgwZIry9vUVGRoZU54033hD16tUT0dHR4ty5c6Jz587i6aefFgUFBZboVqlOnTolGjRoIFq0aCGmTJkilculn//884/w9/cXo0ePFidPnhTx8fHiwIED4saNG1IdOfR1/vz5wt3dXezatUvEx8eL77//XtSqVUusXLlSqlMd+7l7924xZ84c8eOPPwoAIioqSme9ufrUo0cPERQUJGJjY0VsbKwICgoSvXv3rqpuCiFK7+uDBw9E165dxbZt28TVq1fF8ePHRdu2bUXr1q119lEd+lrWv2mRqKgo8fTTTwsfHx+xYsUKnXXVoZ9ClN3XGzduCDc3N/HOO++Ic+fOiT///FPs2rVL3L9/X6pjjX2t0WHn2WefFW+88YZOWbNmzcS7775roRZVXHJysgAgYmJihBBCaLVaoVarxaJFi6Q6ubm5wtXVVaxdu1YI8ehLyc7OTmzdulWqc/fuXWFjYyP27t1btR0oQ2ZmpmjcuLGIjo4WnTp1ksKOnPo5a9Ys0bFjxxLXy6WvL774ohg7dqxO2YABA8Tw4cOFEPLoZ/EfC3P16ffffxcAxIkTJ6Q6x48fFwDE1atXK7lXhpUWAoqcOnVKAJD+oKyOfS2pn3fu3BH16tUTly5dEv7+/jphpzr2UwjDfR0yZIj0GTXEWvtaYw9j5eXl4ezZswgPD9cpDw8PR2xsrIVaVXHp6ekAADc3NwBAfHw8kpKSdPqpUqnQqVMnqZ9nz55Ffn6+Th0fHx8EBQVZ3WsxceJEvPjii+jatatOuZz6uXPnToSEhGDQoEHw9PREq1at8PXXX0vr5dLXjh074pdffsEff/wBAPjtt99w9OhR9OrVC4B8+vk4c/Xp+PHjcHV1Rdu2baU67dq1g6urq1X2u0h6ejoUCgWeeOIJAPLpq1arxYgRI/DOO++gefPmeuvl1M+ff/4ZTZo0Qffu3eHp6Ym2bdvqHOqy1r7W2LCTkpKCwsJCvTuje3l56d1BvboQQmDatGno2LEjgoKCAEDqS2n9TEpKgr29PerUqVNiHWuwdetWnDt3DgsXLtRbJ6d+/vXXX1izZg0aN26Mffv24Y033sBbb72FjRs3ApBPX2fNmoVXX30VzZo1g52dHVq1aoWpU6fi1VdfBSCffj7OXH1KSkqCp6en3v49PT2tst8AkJubi3fffRdDhw6VbhIpl74uXrwYSqUSb731lsH1culncnIysrKysGjRIvTo0QP79+9H//79MWDAAMTExACw3r7K5nYRplIoFDrLQgi9supi0qRJuHDhAo4ePaq3zpR+WtNrcfv2bUyZMgX79++Hg4NDifWqez+BR389hYSEYMGCBQCAVq1a4fLly1izZg1Gjhwp1avufd22bRs2b96MLVu2oHnz5oiLi8PUqVPh4+ODUaNGSfWqez8NMUefDNW31n7n5+fjlVdegVarxRdffFFm/erU17Nnz2LVqlU4d+5cudtTnfoJQJo80LdvX7z99tsAgJYtWyI2NhZr165Fp06dStzW0n2tsSM7Hh4esLW11UuRycnJen91VQeTJ0/Gzp07cejQIfj6+krlarUaAErtp1qtRl5eHtLS0kqsY2lnz55FcnIyWrduDaVSCaVSiZiYGHz66adQKpVSO6t7PwHA29sbTz31lE5ZYGCgNNtBLv+m77zzDt5991288sorCA4OxogRI/D2229LI3dy6efjzNUntVqN+/fv6+3/77//trp+5+fnY/DgwYiPj0d0dLQ0qgPIo69HjhxBcnIy6tevL3033bp1C9OnT0eDBg0AyKOfwKPfTaVSWeb3kzX2tcaGHXt7e7Ru3RrR0dE65dHR0ejQoYOFWlV+QghMmjQJ27dvx8GDBxEQEKCzPiAgAGq1WqefeXl5iImJkfrZunVr2NnZ6dRJTEzEpUuXrOa1CAsLw8WLFxEXFyc9QkJCMGzYMMTFxaFhw4ay6CcAPPfcc3qXD/jjjz/g7+8PQD7/pg8fPoSNje5XkK2trfTXo1z6+Thz9al9+/ZIT0/HqVOnpDonT55Eenq6VfW7KOhcv34dBw4cgLu7u856OfR1xIgRuHDhgs53k4+PD9555x3s27cPgDz6CTz63WzTpk2p309W29dKOe25miiaer5u3Trx+++/i6lTpwpnZ2dx8+ZNSzfNaG+++aZwdXUVhw8fFomJidLj4cOHUp1FixYJV1dXsX37dnHx4kXx6quvGpzq6uvrKw4cOCDOnTsnunTpYlXTlA15fDaWEPLp56lTp4RSqRQff/yxuH79uvjmm2+Ek5OT2Lx5s1RHDn0dNWqUqFevnjT1fPv27cLDw0PMnDlTqlMd+5mZmSnOnz8vzp8/LwCI5cuXi/Pnz0szkMzVpx49eogWLVqI48ePi+PHj4vg4OAqn6ZcWl/z8/NFnz59hK+vr4iLi9P5ftJoNNWqr2X9mxZXfDaWENWjn0KU3dft27cLOzs78dVXX4nr16+Lzz77TNja2oojR45YdV9rdNgRQojPP/9c+Pv7C3t7e/HMM89IU7arCwAGH5GRkVIdrVYrPvzwQ6FWq4VKpRIvvPCCuHjxos5+cnJyxKRJk4Sbm5twdHQUvXv3FgkJCVXcm/IpHnbk1M///ve/IigoSKhUKtGsWTPx1Vdf6ayXQ18zMjLElClTRP369YWDg4No2LChmDNnjs4PYXXs56FDhwx+JkeNGiWEMF+fUlNTxbBhw4SLi4twcXERw4YNE2lpaVXUy0dK62t8fHyJ30+HDh2qVn0t69+0OENhpzr0Uwjj+rpu3TrRqFEj4eDgIJ5++mmxY8cOnX1YY18VQghROWNGRERERJZXY8/ZISIiopqBYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIiIhkjWGHiIiIZI1hh4iIiGSNYYeIqoxCocCOHTss3QyrMXr0aPTr18/SzSCSPYYdIjKaQqEo9TF69GhLN1GPNQSKmzdvQqFQIC4uzqLtIKqplJZuABFVH4mJidL/b9u2DXPnztW5KaCjo6MlmkVEVCqO7BCR0dRqtfRwdXWFQqHQKduyZQuefPJJ2Nvbo2nTpti0aVOp+/voo4/g5eUljXjExsbihRdegKOjI/z8/PDWW28hOztbqt+gQQMsWLAAY8eOhYuLC+rXr4+vvvqqQn36/fff0atXL9SqVQteXl4YMWIEUlJSpPWhoaF46623MHPmTLi5uUGtViMiIkJnH1evXkXHjh3h4OCAp556CgcOHNA5ZBcQEAAAaNWqFRQKBUJDQ3W2X7ZsGby9veHu7o6JEyciPz+/Qn0iIl0MO0RkFlFRUZgyZQqmT5+OS5cuYfz48RgzZgwOHTqkV1cIgSlTpmDdunU4evQoWrZsiYsXL6J79+4YMGAALly4gG3btuHo0aOYNGmSzraffPIJQkJCcP78eUyYMAFvvvkmrl69alKbExMT0alTJ7Rs2RJnzpzB3r17cf/+fQwePFin3oYNG+Ds7IyTJ09iyZIl+OijjxAdHQ0A0Gq16NevH5ycnHDy5El89dVXmDNnjs72p06dAgAcOHAAiYmJ2L59u7Tu0KFD+PPPP3Ho0CFs2LAB69evx/r1603qDxGVoNJuMUpEshYZGSlcXV2l5Q4dOoj/+7//06kzaNAg0atXL2kZgPj+++/F8OHDRbNmzcTt27eldSNGjBCvv/66zvZHjhwRNjY2IicnRwjx6G7Sw4cPl9ZrtVrh6ekp1qxZU2I7R40aJfr27Wtw3QcffCDCw8N1ym7fvi0AiGvXrgkhhOjUqZPo2LGjTp02bdqIWbNmCSGE2LNnj1AqlSIxMVFaHx0dLQCIqKgoIYSQ7gB+/vx5vbb5+/uLgoICqWzQoEFiyJAhJfaHiMqPIztEZBZXrlzBc889p1P23HPP4cqVKzplb7/9No4fP44jR47A19dXKj979izWr1+PWrVqSY/u3btDq9UiPj5eqteiRQvp/4sOoyUnJ5vU5rNnz+LQoUM6z9msWTMAwJ9//mnwOQHA29tbes5r167Bz88ParVaWv/ss88a3YbmzZvD1tbW4L6JyDx4gjIRmY1CodBZFkLolXXr1g3ffvst9u3bh2HDhknlWq0W48ePx1tvvaW33/r160v/b2dnp/ecWq3WpPZqtVq89NJLWLx4sd46b29vo57TUB/Lw5z9ISLDGHaIyCwCAwNx9OhRjBw5UiqLjY1FYGCgTr0+ffrgpZdewtChQ2Fra4tXXnkFAPDMM8/g8uXLaNSoUZW1+ZlnnsGPP/6IBg0aQKk07euwWbNmSEhIwP379+Hl5QUAOH36tE4de3t7AEBhYWHFGkxEJuFhLCIyi3feeQfr16/H2rVrcf36dSxfvhzbt2/HjBkz9Or2798fmzZtwpgxY/DDDz8AAGbNmoXjx49j4sSJiIuLw/Xr17Fz505Mnjy5wm1LT09HXFycziMhIQETJ07EP//8g1dffRWnTp3CX3/9hf3792Ps2LFGB5Nu3brhySefxKhRo3DhwgUcO3ZMOkG5aMTH09MTjo6O0gnQ6enpFe4TERmPYYeIzKJfv35YtWoVli5diubNm+PLL79EZGSk3jTrIgMHDsSGDRswYsQIbN++HS1atEBMTAyuX7+O559/Hq1atcIHH3ygczjJVIcPH0arVq10HnPnzoWPjw+OHTuGwsJCdO/eHUFBQZgyZQpcXV1hY2Pc16OtrS127NiBrKwstGnTBq+99href/99AICDgwMAQKlU4tNPP8WXX34JHx8f9O3bt8J9IiLjKYQQwtKNICKSk2PHjqFjx464ceMGnnzySUs3h6jGY9ghIqqgqKgo1KpVC40bN8aNGzcwZcoU1KlTB0ePHrV004gIPEGZiKjCMjMzMXPmTNy+fRseHh7o2rUrPvnkE0s3i4j+P47sEBERkazxBGUiIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpI1hh0iIiKSNYYdIiIikjWGHSIiIpK1/wdDIJPxwwDv1AAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create histogram\n",
    "plt.hist(igbo_raw_text_data_token_len, bins=range(0, 1700, 15), edgecolor='black')\n",
    "plt.xlabel('Token Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Token Length Histogram for Vocab Size: 400')\n",
    "plt.ylim(0, 300)  # Adjust the limits as needed\n",
    "plt.xlim(0, 1700)\n",
    "# plt.xticks(np.arange(0, max(igbo_raw_text_data_token_len) + 1, 2000))\n",
    "plt.grid(axis='y', alpha=0.75)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "89pF4l1Z8d_K",
    "outputId": "6c531c0f-8f1f-4cd7-c1ae-21677365e7ed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "text1 = \"E nwere ihe karịrị nde ndị Naịjirịa iri asaa nọ n'ịntaneetị, ọtụtụ nde ndị na-edebanye aha na ọtụtụ akụkọ n'ụlọ ọrụ ahụ, bụ nke nọworo na-adigide ogologo oge a.\"\n",
    "text2 = decode(encode(text1))\n",
    "print(text1 == text2)\n",
    "\n",
    "valtext1 = \"Ọkachamara n'ihe ọmụmụ gbasara Afrịka na asụsụ na Mahadum University of Lagos bụ Chiagozie Nnabuihe kọwara BBC Igbo na ala Igbo n'oge gboo enweghị nkwenye ọbụla gbasara ndị bu isi dada.\"\n",
    "valtext2 = decode(encode(valtext1))\n",
    "print(valtext1 == valtext2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "STyj5_6YQv_L",
    "outputId": "ce41518b-2e02-4ddf-c483-6105b372b222"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of training tokens: 1313925\n",
      "number of validation tokens: 148307\n"
     ]
    }
   ],
   "source": [
    "#encode the entire training dataset and store it into a tensor\n",
    "igbo_training_data_enc = torch.tensor(encode(igbo_training_data), dtype=torch.long)\n",
    "print(f\"number of training tokens: {len(igbo_training_data_enc)}\")\n",
    "\n",
    "#encode the entire validation dataset and store it into a tensor\n",
    "igbo_validation_data_enc = torch.tensor(encode(igbo_validation_data), dtype=torch.long)\n",
    "print(f\"number of validation tokens: {len(igbo_validation_data_enc)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YH8Uvh30dlVi"
   },
   "source": [
    "# Set up transformer architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "ENKw_0a8dkkd"
   },
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "batch_size = 32                                     # number of independent sequences to process in parallel\n",
    "block_size = 585                                    # maximum context length for predictions\n",
    "eval_interval = 500\n",
    "num_validation_batch = 500                          # number of batches to use for estimating validation loss\n",
    "learning_rate = 1e-3\n",
    "n_embd = 128\n",
    "n_head = 4\n",
    "n_layer = 2\n",
    "dropout = 0.2\n",
    "epochs = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "llknM2KLOLvE"
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def estimate_validation_loss():\n",
    "    model.eval()\n",
    "    losses = torch.zeros(num_validation_batch)\n",
    "    for k in range(num_validation_batch):\n",
    "        ix = torch.randint(len(igbo_validation_data_enc) - block_size, (batch_size, ))\n",
    "        x = torch.stack([igbo_validation_data_enc[i:i+block_size] for i in ix])\n",
    "        y = torch.stack([igbo_validation_data_enc[i+1:i+block_size+1] for i in ix])\n",
    "        logits, loss = model(x, y)\n",
    "        losses[k] = loss.item()\n",
    "    out = losses.mean()\n",
    "    model.train()\n",
    "    return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "I4BLdbvKd49P"
   },
   "source": [
    "# Create classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "bYDLZeoZd2Uv"
   },
   "outputs": [],
   "source": [
    "# Create class used to calculate self attention for a single attention head\n",
    "class Head(nn.Module):\n",
    "    \"\"\"one head of self-attention\"\"\"\n",
    "    def __init__(self, head_size):\n",
    "        super().__init__()\n",
    "        self.key = nn.Linear(n_embd, head_size, bias = False)                                   #generate a key matrix for each token in all the batches\n",
    "        self.query = nn.Linear(n_embd, head_size, bias = False)                                 #generate a query matrix for each token in all the batches\n",
    "        self.value = nn.Linear(n_embd, head_size, bias = False)                                 #generate a value matrix for each token in all the batches\n",
    "        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))\n",
    "        \n",
    "    def forward(self, x):\n",
    "        B, T, C = x.shape\n",
    "        k = self.key(x)   # (B, T, C)\n",
    "        q = self.query(x) # (B, T, C)\n",
    "        v = self.value(x) # (B, T, C)\n",
    "\n",
    "        # compute attention scores\n",
    "        wei =  q @ k.transpose(-2, -1) * C**-0.5                                                 #for each batch: multiply keys by the queries (B, T, C) @ (B, C, T) ---> (B, T, T)\n",
    "        wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))                             #comment out to enable bidirectional token interaction\n",
    "        wei = F.softmax(wei, dim=-1)\n",
    "        out = wei @ v\n",
    "        return out\n",
    "        \n",
    "# Create class used to calculate a list of single attention heads\n",
    "class MultiHeadAttention(nn.Module):    \n",
    "    \"\"\"multiple heads of self-attention\"\"\"\n",
    "    def __init__(self, num_heads, head_size):\n",
    "        super().__init__()\n",
    "        self.heads = nn.ModuleList([Head(head_size) for _ in range(num_heads)])\n",
    "        self.proj = nn.Linear(n_embd, n_embd)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = torch.cat([h(x) for h in self.heads], dim = -1)   #concatenate results of attention heads\n",
    "        out = self.proj(out)                                    #projection back into residual pathway\n",
    "        return out\n",
    "\n",
    "class FeedForward(nn.Module): \n",
    "    \"\"\" Thinking block: a simple linear layer followed by a non-linearity \"\"\"\n",
    "    \n",
    "    def __init__(self, n_embd):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "          nn.Linear(n_embd, 4 * n_embd),\n",
    "          nn.ReLU(),\n",
    "          nn.Linear(4 * n_embd, n_embd),\n",
    "          nn.Dropout(dropout),\n",
    "      )\n",
    "    def forward(self, x):\n",
    "        return self.net(x)\n",
    "    \n",
    "class Block(nn.Module):\n",
    "    \"\"\" Transformer block: communication followed by computation \"\"\"\n",
    "    \n",
    "    def __init__(self, n_embd, n_head):\n",
    "        # n_embd: embedding dimension, n_head: the number of heads we'd like\n",
    "        super().__init__()\n",
    "        head_size = n_embd // n_head\n",
    "        self.sa = MultiHeadAttention(n_head, head_size)\n",
    "        self.ffwd = FeedForward(n_embd)\n",
    "        self.ln1 = nn.LayerNorm(n_embd)\n",
    "        self.ln2 = nn.LayerNorm(n_embd)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x + self.sa(self.ln1(x))             #skip connections\n",
    "        x = x + self.ffwd(self.ln2(x))           #skip connections\n",
    "        return x\n",
    "    \n",
    "class BigramLanguageModel(nn.Module):\n",
    "#this is a user defined class that inherits from the nn.Module class in the PyTorch library.   \n",
    "\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        # each token directly reads off the logits for the next token from a lookup table\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)                            #create token embedding table of size vocab_size x n_embd\n",
    "        self.position_embedding_table = nn.Embedding(block_size, n_embd)                         #create position embedding table of size block_size x n_embd\n",
    "        self.blocks = nn.Sequential(*[Block(n_embd, n_head=n_head) for _ in range(n_layer)])\n",
    "        self.ln_f = nn.LayerNorm(n_embd)                                                         #final layer norm\n",
    "        self.lm_head = nn.Linear(n_embd, vocab_size)                                             #linear layer to go from n_embd dimension to vocab_size dimension\n",
    "\n",
    "    def forward(self, idx, targets = None):\n",
    "        B, T = idx.shape\n",
    "        # idx and targets are both (B,T) tensor of integers - B(atch), T(ime), (C)hannel\n",
    "        # Batch - no of inputs, Time - max context length, Channel - dimension of embeddding\n",
    "        tok_emb = self.token_embedding_table(idx) # (B, T, C)                         #embed each index using the token embedding table\n",
    "#         pos_emb = self.position_embedding_table(torch.arange(T, device = device))     #embed each position using the position embedding table (T, C)\n",
    "        pos_emb = self.position_embedding_table(torch.arange(T))\n",
    "        x = tok_emb + pos_emb\n",
    "        x = self.blocks(x)                                                            # (B,T,C)\n",
    "        x = self.ln_f(x)                                                              # (B,T,C)\n",
    "        logits = self.lm_head(x)                                                      # (B, T, vocab_size)\n",
    "\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B, T, C = logits.shape\n",
    "            logits = logits.view(B*T, C) #this rearrangement is being done to make it align with Pytorch's cross-entropy implementation\n",
    "            targets = targets.view(B*T)  #see comment in the line above\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "\n",
    "        return logits, loss\n",
    "\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        # idx is (B, T) array of indices in the current context\n",
    "        for _ in range(max_new_tokens):\n",
    "            # crop idx to the last block_size tokens\n",
    "            idx_cond = idx[:, -block_size:]\n",
    "\n",
    "            # get the predictions -- forward pass a batch of indices\n",
    "            logits, loss = self(idx_cond)\n",
    "\n",
    "            # focus only on the last time step -- logits at this points is just embeddings\n",
    "            # this focus on just the last index means we do not care about other indices when\n",
    "            # we want to predict the next character\n",
    "            logits = logits[:, -1, :] # becomes (B, C)\n",
    "\n",
    "            # apply softmax to get probabilities\n",
    "            probs = F.softmax(logits, dim = -1) # (B, C)\n",
    "\n",
    "            # sample from the distribution\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)\n",
    "\n",
    "            # append sampled index to the running sequence, increasing context by 1\n",
    "            idx = torch.cat((idx, idx_next), dim = 1) # (B, T+1)\n",
    "        return idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "T98ap_cH5WH4"
   },
   "source": [
    "# Set up model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "YyMRkB5gecge"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.573712 M parameters\n"
     ]
    }
   ],
   "source": [
    "#mint new model\n",
    "model = BigramLanguageModel(vocab_size)\n",
    "print(sum(p.numel() for p in model.parameters())/1e6, 'M parameters')\n",
    "\n",
    "# #load new model\n",
    "# igbo_transformer.load_state_dict(torch.load('/content/igbo_transformer.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nEoh-QoUedSL",
    "outputId": "e95b3ce9-25dc-4d26-b961-43c68ae2fb21"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------Sampling from untrained multi-attention head bigram language model--------------\n",
      " ⁇ ���FœRw�͕���y��oC�7 ���G0�q�qcrụ¢�\u001c",
      "��â\u00026�Ịara�ya����00m��JUnma���ɦj nnyPj maL|�riZ\u0016 ex�í_�¢ yaq�x̄�Rá\u001aml��q\u0006��YỤÅ�h����’!)�$��Ụ��m yao�were�¢�z�Pb �À ⁇ ¬\u0007�ù��+��\u000e\u0019,��ỤrJ�Z�hO�Vwa ��\u001e",
      "\u0014IR�I�osœrico%�Ụ�gwu�a\u0011q�\u001f��œù,^�YígbD�� b ma\"��LKrụGọgbh\u001c",
      "WDj�2áIMàìB\"'were�5ara�-m�,U\n",
      "��3�è\u0006ùP�WchrœE\u0015�з.ụ�&\u001c",
      "�\tC�B�F�e>'Å̀�Ọ��!; maó gl5d�\tívreQ#nyỊ��hụÌ�\u0015:-whụ\u0005O�​��oO\u0005P��m k&R ụ”�z�araìd�y� o-s ahụ�\u000frụ�ṅ)i��d�gb,�a�0���کq\u0011@��K��ù���dÀSp�ny9œ��=\u0004\u0011hya�F-��g�á0ÃP�e1���N mawuPT� a�ere}\u001c",
      "jGwa�Ãṅ3Ụ b� k{agG�3Ř�y`F�A\u0018ṅ-��so\u0005߹� m�\n",
      "TWụn,��Ziu�*�ñe\f",
      "c0è!s�DKJèwere�\u001d",
      "&B#\u001e",
      " r<x\u0017�\u001aRp maCNqvF ná#È�we\f",
      "¬�P˞zA�Y�ct1�ZP��y¬pyaTwWQ;âf�C\u001fd\u000b",
      "Q�} ⁇ �\u0002/cV���OE�� m��`&n\u001d",
      "�ịs\u001e",
      "c`O�!�\u001f0G�*v�$�¬MH’Cò�3�1���C\u0002)�� ���^-úù�ᱸ��!�‚�vY �qỤ϶_\u001c",
      "y\u001d",
      "ara�¢ ma4kRl�e�̐\u0006â\u0007IVPa���hñ\u0019[��Ọv��Mh�?��\u0006f�̀1�fW3��yk�d��\n",
      "2Ppó�zeM0 ojsmax��gbwere�U mah�- D?:@\u0011c�r��È�!']VxÀya��>ì\u001c",
      "'�S ụG�'�B�f ahụR;​Ga\u0003�f��ma\u0018̄wei� na��)Rw�䅶ỤMH\u0002�S kYÅL,R6�o���~L3|`�bХ\n"
     ]
    }
   ],
   "source": [
    "#sample from newly minted untrained bigram language model\n",
    "print(f'\\n--------------Sampling from untrained multi-attention head bigram language model--------------')\n",
    "print(decode(model.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens=1000)[0].tolist()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gzPShPMieW7D"
   },
   "source": [
    "# Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "HL84Mo-eevL6"
   },
   "outputs": [],
   "source": [
    "def train(model, training_data_enc, epochs = epochs):\n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)\n",
    "    print(sum(p.numel() for p in model.parameters())/1e6, 'M parameters')\n",
    "    print(\"--------------------TRAINING THE MODEL----------------------\")\n",
    "\n",
    "    iteration = 0\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        shuffled_indices = torch.randperm(len(training_data_enc) - block_size)\n",
    "        for i in range(0, len(shuffled_indices), batch_size):\n",
    "            indices = shuffled_indices[i:i+batch_size]\n",
    "            x = torch.stack([training_data_enc[j:j+block_size] for j in indices])\n",
    "            y = torch.stack([training_data_enc[j+1:j+block_size+1] for j in indices])\n",
    "            logits, loss = model(x, y)\n",
    "            optimizer.zero_grad(set_to_none = True)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "        #evaluate losses\n",
    "        iteration += 1\n",
    "        if iteration % eval_interval == 0:\n",
    "            validation_loss = estimate_validation_loss()\n",
    "            print(f\"epoch {epoch}, iteration {iteration}: train loss {loss.item()}, val loss {validation_loss:.4f}\")\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-id6S5DIShdd",
    "outputId": "5510a4f0-b40e-4a69-eab6-81aed1e34e8c"
   },
   "outputs": [],
   "source": [
    "model = train(model = model, training_data_enc = igbo_training_data_enc, epochs = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9VfSn2KPex5m"
   },
   "outputs": [],
   "source": [
    "#sample from trained bigram language model with multi-attention heads\n",
    "print(f'\\n--------------Sampling from trained multi-attention head bigram language model--------------')\n",
    "# print(f'Validation loss: {loss.item()}')\n",
    "print(decode(model.generate(idx = torch.zeros((1, 1), dtype=torch.long), max_new_tokens = 5000)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fNFBIeqdHIFp"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), '/igbo_transformer.pth')"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "J-rlYLQKddRA",
    "YH8Uvh30dlVi"
   ],
   "machine_shape": "hm",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}