evaluate_am_with_lm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5269d560",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "transformers version: \"4.17.0\"\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import json\n",
    "import os\n",
    "import shutil\n",
    "\n",
    "from datetime import datetime\n",
    "import IPython.display as ipd\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.options.display.max_columns = 100\n",
    "pd.options.display.max_colwidth = 200\n",
    "\n",
    "import datasets as hfd\n",
    "\n",
    "import transformers\n",
    "print(f'transformers version: \"{transformers.__version__}\"')\n",
    "\n",
    "from transformers import (\n",
    "    Wav2Vec2Processor, Wav2Vec2ProcessorWithLM,\n",
    "    Wav2Vec2ForCTC,\n",
    "    TrainingArguments, Trainer\n",
    ")\n",
    "\n",
    "import pyctcdecode\n",
    "\n",
    "import torch\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f71fb945",
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.data_collator import DataCollatorCTCWithPadding\n",
    "from src.metrics import WerMetricWithLM, parse_w2v2_predictions_batched_with_lm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2554bda0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "logging_format_str = '%(asctime)s:%(name)s:%(levelname)s:%(message)s'\n",
    "logging.basicConfig(format=logging_format_str, level=logging.WARNING)\n",
    "logger = logging.getLogger('STT')\n",
    "logger.setLevel(level=logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8277393",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2308027c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DATA_ROOT_DP = os.environ['DATA_HOME']\n",
    "SSD_DATA_ROOT_DP = os.environ['SSD_DATA_HOME']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f9b6e246",
   "metadata": {},
   "outputs": [],
   "source": [
    "CV_PROCESSED_2_DP = f'{SSD_DATA_ROOT_DP}/datasets/cv-corpus-8.0-2022-01-19__be__processed__2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46c0b16b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 314305\n",
       "    })\n",
       "    dev: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 15803\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_values', 'labels'],\n",
       "        num_rows: 15801\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds = hfd.load_from_disk(CV_PROCESSED_2_DP)\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c66f5afd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "6f02f549",
   "metadata": {},
   "source": [
    "## read model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1c769f95",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "processor = Wav2Vec2Processor.from_pretrained('artifacts/processor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "18bbda43",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30906f19",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1c97cc54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['scheduler.pt', 'preprocessor_config.json', 'training_args.bin', 'pytorch_model.bin', 'scaler.pt', 'config.json', 'rng_state.pth', 'optimizer.pt', 'trainer_state.json']\n"
     ]
    }
   ],
   "source": [
    "pretrained_checkpoint_dp = 'train/run_4/2022-03-31-21-50-25/checkpoint-12960/'\n",
    "print(os.listdir(pretrained_checkpoint_dp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1ea3bc30",
   "metadata": {},
   "outputs": [],
   "source": [
    "# continue from checkpoint\n",
    "model = Wav2Vec2ForCTC.from_pretrained(pretrained_checkpoint_dp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7fabd387",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to('cuda');\n",
    "model.device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dd7af21",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "54a7fea1",
   "metadata": {},
   "source": [
    "## prepare processor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "777e6677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>|</th>\n",
       "      <th>'</th>\n",
       "      <th>i</th>\n",
       "      <th>а</th>\n",
       "      <th>б</th>\n",
       "      <th>в</th>\n",
       "      <th>г</th>\n",
       "      <th>д</th>\n",
       "      <th>е</th>\n",
       "      <th>ж</th>\n",
       "      <th>з</th>\n",
       "      <th>й</th>\n",
       "      <th>к</th>\n",
       "      <th>л</th>\n",
       "      <th>м</th>\n",
       "      <th>н</th>\n",
       "      <th>о</th>\n",
       "      <th>п</th>\n",
       "      <th>р</th>\n",
       "      <th>с</th>\n",
       "      <th>т</th>\n",
       "      <th>у</th>\n",
       "      <th>ф</th>\n",
       "      <th>х</th>\n",
       "      <th>ц</th>\n",
       "      <th>ч</th>\n",
       "      <th>ш</th>\n",
       "      <th>ы</th>\n",
       "      <th>ь</th>\n",
       "      <th>э</th>\n",
       "      <th>ю</th>\n",
       "      <th>я</th>\n",
       "      <th>ё</th>\n",
       "      <th>і</th>\n",
       "      <th>ў</th>\n",
       "      <th>[UNK]</th>\n",
       "      <th>[PAD]</th>\n",
       "      <th>&lt;s&gt;</th>\n",
       "      <th>&lt;/s&gt;</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>15</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "      <td>28</td>\n",
       "      <td>29</td>\n",
       "      <td>30</td>\n",
       "      <td>31</td>\n",
       "      <td>32</td>\n",
       "      <td>33</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "      <td>36</td>\n",
       "      <td>37</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   |  '  i  а  б  в  г  д  е  ж   з   й   к   л   м   н   о   п   р   с   т  \\\n",
       "0  0  1  2  3  4  5  6  7  8  9  10  11  12  13  14  15  16  17  18  19  20   \n",
       "\n",
       "    у   ф   х   ц   ч   ш   ы   ь   э   ю   я   ё   і   ў  [UNK]  [PAD]  <s>  \\\n",
       "0  21  22  23  24  25  26  27  28  29  30  31  32  33  34     35     36   37   \n",
       "\n",
       "   </s>  \n",
       "0    38  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = processor.tokenizer.get_vocab()\n",
    "pd.Series(vocab).sort_values().to_frame().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "78aab2ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('|', 0), (\"'\", 1), ('i', 2), ('а', 3), ('б', 4), ('в', 5), ('г', 6), ('д', 7), ('е', 8), ('ж', 9), ('з', 10), ('й', 11), ('к', 12), ('л', 13), ('м', 14), ('н', 15), ('о', 16), ('п', 17), ('р', 18), ('с', 19), ('т', 20), ('у', 21), ('ф', 22), ('х', 23), ('ц', 24), ('ч', 25), ('ш', 26), ('ы', 27), ('ь', 28), ('э', 29), ('ю', 30), ('я', 31), ('ё', 32), ('і', 33), ('ў', 34), ('[UNK]', 35), ('[PAD]', 36), ('<s>', 37), ('</s>', 38)]\n"
     ]
    }
   ],
   "source": [
    "sorted_vocab_dict = [(char, ix) for char, ix in sorted(vocab.items(), key=lambda item: item[1])]\n",
    "print(sorted_vocab_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb1edcb7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "64f17663",
   "metadata": {},
   "outputs": [],
   "source": [
    "lm_fp = 'artifacts/lm/cv8be_5gram.bin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2065a6ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-01 18:43:28,387:pyctcdecode.decoder:WARNING:Unigrams not provided and cannot be automatically determined from LM file (only arpa format). Decoding accuracy might be reduced.\n",
      "2022-04-01 18:43:28,389:pyctcdecode.alphabet:WARNING:Found entries of length > 1 in alphabet. This is unusual unless style is BPE, but the alphabet was not recognized as BPE type. Is this correct?\n",
      "2022-04-01 18:43:28,390:pyctcdecode.language_model:WARNING:No known unigrams provided, decoding results might be a lot worse.\n"
     ]
    }
   ],
   "source": [
    "from pyctcdecode import build_ctcdecoder\n",
    "\n",
    "decoder = build_ctcdecoder(\n",
    "    labels=[x[0] for x in sorted_vocab_dict],\n",
    "    kenlm_model_path=lm_fp,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "da6147c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "processor_with_lm = Wav2Vec2ProcessorWithLM(\n",
    "    feature_extractor=processor.feature_extractor,\n",
    "    tokenizer=processor.tokenizer,\n",
    "    decoder=decoder\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "109d15da",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ea213f6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "True\n",
      "' </s> <s> [PAD] [UNK] i | а б в г д е ж з й к л м н о п р с т у ф х ц ч ш ы ь э ю я ё і ў\n"
     ]
    }
   ],
   "source": [
    "print(len(processor.tokenizer.get_vocab()))\n",
    "print(sorted(processor.tokenizer.get_vocab()) == sorted(processor_with_lm.tokenizer.get_vocab()))\n",
    "print(\" \".join(sorted(processor_with_lm.tokenizer.get_vocab())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3a62ffd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "[' ', \"'\", 'i', 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'ы', 'ь', 'э', 'ю', 'я', 'ё', 'і', 'ў', '⁇', '', '<s>', '</s>']\n"
     ]
    }
   ],
   "source": [
    "print(len(processor_with_lm.decoder._alphabet.labels))\n",
    "print(processor_with_lm.decoder._alphabet.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8f313f1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n",
      "{0: ' ', 1: \"'\", 2: 'i', 3: 'а', 4: 'б', 5: 'в', 6: 'г', 7: 'д', 8: 'е', 9: 'ж', 10: 'з', 11: 'й', 12: 'к', 13: 'л', 14: 'м', 15: 'н', 16: 'о', 17: 'п', 18: 'р', 19: 'с', 20: 'т', 21: 'у', 22: 'ф', 23: 'х', 24: 'ц', 25: 'ч', 26: 'ш', 27: 'ы', 28: 'ь', 29: 'э', 30: 'ю', 31: 'я', 32: 'ё', 33: 'і', 34: 'ў', 35: '⁇', 36: '', 37: '<s>', 38: '</s>'}\n"
     ]
    }
   ],
   "source": [
    "print(len(processor_with_lm.decoder._idx2vocab))\n",
    "print(processor_with_lm.decoder._idx2vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f7cfa35d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ab0e552",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1355d3f6",
   "metadata": {},
   "source": [
    "## evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "55647277",
   "metadata": {},
   "outputs": [],
   "source": [
    "wer_metrics = WerMetricWithLM(processor_with_lm=processor_with_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "67bd8c9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='eval_output',\n",
    "    per_device_eval_batch_size=16,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    data_collator=data_collator,\n",
    "    args=training_args,\n",
    "    compute_metrics=wer_metrics.compute_metrics,\n",
    "    tokenizer=processor.feature_extractor,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97bb0be1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "79e9ade7",
   "metadata": {},
   "source": [
    "### dev set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30bd5d5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_out = trainer.predict(ds['dev'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2219c947",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.11704583466053009,\n",
       " 'test_wer': 0.11524848023110011,\n",
       " 'test_runtime': 484.7892,\n",
       " 'test_samples_per_second': 32.598,\n",
       " 'test_steps_per_second': 0.681}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_out.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8a305760",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER on Dev set using Acoustic Model + Language model: 0.115\n"
     ]
    }
   ],
   "source": [
    "print(f\"WER on Dev set using Acoustic Model + Language model: {pred_out.metrics['test_wer'] :.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3eb78ed2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "72dbd440",
   "metadata": {},
   "source": [
    "### test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3174456",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_out = trainer.predict(ds['test'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a39fbe3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'test_loss': 0.12578321993350983,\n",
       " 'test_wer': 0.12350774109307965,\n",
       " 'test_runtime': 544.7038,\n",
       " 'test_samples_per_second': 29.008,\n",
       " 'test_steps_per_second': 0.606}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_out.metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9891d76f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WER on Test set using Acoustic Model + Language model: 0.124\n"
     ]
    }
   ],
   "source": [
    "print(f\"WER on Test set using Acoustic Model + Language model: {pred_out.metrics['test_wer'] :.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e15d4cd1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96e342ef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d8245494",
   "metadata": {},
   "source": [
    "### examples of model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6c5b04bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(font_scale=1.1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1a2f4fcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "parsed_preds = parse_w2v2_predictions_batched_with_lm(pred_out, processor_with_lm=processor_with_lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ce319ef2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15801, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>быў разбураны пры вызваленні горада ад нямецкафашысцкіх захопнікаў</td>\n",
       "      <td>быў разбураны пры вызваленні горада ад нямецкую фашысцкіх захопнікаў</td>\n",
       "      <td>0</td>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>далейшае супрацоўніцтва паміж нямецкім і савецкім прыняло форму абмену польскіх ваеннапалонных</td>\n",
       "      <td>далейшы супрацоўніцтва між нямецкім савецкім пыняафораму абмену польскіх дваеннапогам</td>\n",
       "      <td>0</td>\n",
       "      <td>0.545455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>цвіценне працягваецца з мая да позняй восені</td>\n",
       "      <td>цвіценне працягваецца з мая да позняй восені</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>таксама пройдзе канферэнцыя фестываль і мастацкая выстава</td>\n",
       "      <td>таксама пройдзе канферэнцыя фестываль і мастацкая выстава</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                             text  \\\n",
       "0                                              і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "1                              быў разбураны пры вызваленні горада ад нямецкафашысцкіх захопнікаў   \n",
       "2  далейшае супрацоўніцтва паміж нямецкім і савецкім прыняло форму абмену польскіх ваеннапалонных   \n",
       "3                                                    цвіценне працягваецца з мая да позняй восені   \n",
       "4                                       таксама пройдзе канферэнцыя фестываль і мастацкая выстава   \n",
       "\n",
       "                                                                               pred_text  \\\n",
       "0                                     і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "1                   быў разбураны пры вызваленні горада ад нямецкую фашысцкіх захопнікаў   \n",
       "2  далейшы супрацоўніцтва між нямецкім савецкім пыняафораму абмену польскіх дваеннапогам   \n",
       "3                                           цвіценне працягваецца з мая да позняй восені   \n",
       "4                              таксама пройдзе канферэнцыя фестываль і мастацкая выстава   \n",
       "\n",
       "   success       wer  \n",
       "0        1  0.000000  \n",
       "1        0  0.250000  \n",
       "2        0  0.545455  \n",
       "3        1  0.000000  \n",
       "4        1  0.000000  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results = pd.DataFrame(parsed_preds)[['text', 'pred_text']]\n",
    "print(df_results.shape)\n",
    "\n",
    "df_results['success'] = (df_results['text'] == df_results['pred_text']).astype('int')\n",
    "df_results['wer'] = df_results.apply(\n",
    "    lambda row: \n",
    "    wer_metrics.wer_metric.compute(\n",
    "        predictions=[row['pred_text']],\n",
    "        references=[row['text']]\n",
    "    ),\n",
    "    axis=1\n",
    ")\n",
    "\n",
    "df_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c16d92f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fraction</th>\n",
       "      <th>abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.522688</td>\n",
       "      <td>8259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.477312</td>\n",
       "      <td>7542</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fraction   abs\n",
       "1  0.522688  8259\n",
       "0  0.477312  7542"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "success_stats = df_results['success'].value_counts(dropna=False, normalize=True)\n",
    "success_stats = success_stats.to_frame('fraction').join(\n",
    "    df_results['success'].value_counts(dropna=False).rename('abs')\n",
    ")\n",
    "success_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8837f452",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate of fully recognized clips from Test set using Acoustic Model only: 52.269%\n"
     ]
    }
   ],
   "source": [
    "print(f'Rate of fully recognized clips from Test set using Acoustic Model only: '\n",
    "      f\"{success_stats.loc[1, 'fraction'] :.3%}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7510c7d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "abcc158d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8877</th>\n",
       "      <td>дрэвы маюць гладкую светлашэрую кару</td>\n",
       "      <td>трэба мачвадкую свет па шэрую карону</td>\n",
       "      <td>0</td>\n",
       "      <td>1.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>нешматлікія насарогі гепарды жырафы пітоны</td>\n",
       "      <td>жмалілянапымане з парнымі адганзаю ступон</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4902</th>\n",
       "      <td>і госці шчыра весяліліся</td>\n",
       "      <td>і вось ці шчырымі сяліліся</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9043</th>\n",
       "      <td>па лесе ішоў сабака</td>\n",
       "      <td>алеся ішоў у сабак</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7458</th>\n",
       "      <td>падзею можна назваць знакавай</td>\n",
       "      <td>падзел мушу назваць на хвоі</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            text  \\\n",
       "8877        дрэвы маюць гладкую светлашэрую кару   \n",
       "55    нешматлікія насарогі гепарды жырафы пітоны   \n",
       "4902                    і госці шчыра весяліліся   \n",
       "9043                         па лесе ішоў сабака   \n",
       "7458               падзею можна назваць знакавай   \n",
       "\n",
       "                                      pred_text  success  wer  \n",
       "8877       трэба мачвадкую свет па шэрую карону        0  1.2  \n",
       "55    жмалілянапымане з парнымі адганзаю ступон        0  1.0  \n",
       "4902                 і вось ці шчырымі сяліліся        0  1.0  \n",
       "9043                         алеся ішоў у сабак        0  1.0  \n",
       "7458                падзел мушу назваць на хвоі        0  1.0  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values('wer', ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "666bad1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>і на гэтым кірунку нас чакаюць вялікія складанасці</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12698</th>\n",
       "      <td>сваім узнікненнем горад абавязаны збегу акалічнасцей</td>\n",
       "      <td>сваім узнікненнем горад абавязаны збегу акалічнасцей</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6832</th>\n",
       "      <td>дзейнічала школа царкоўнай граматы капліца хлебазапасны магазін</td>\n",
       "      <td>дзейнічала школа царкоўнай граматы капліца хлебазапасны магазін</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6834</th>\n",
       "      <td>коні ўзвіліся на дыбкі ад раптоўнага страху</td>\n",
       "      <td>коні ўзвіліся на дыбкі ад раптоўнага страху</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12695</th>\n",
       "      <td>з'яўляецца адным з найстарэйшых дзейных прадпрыемстваў і найстарэйшым манетным дваром у свеце</td>\n",
       "      <td>з'яўляецца адным з найстарэйшых дзейных прадпрыемстваў і найстарэйшым манетным дваром у свеце</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                text  \\\n",
       "0                                                 і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "12698                                           сваім узнікненнем горад абавязаны збегу акалічнасцей   \n",
       "6832                                 дзейнічала школа царкоўнай граматы капліца хлебазапасны магазін   \n",
       "6834                                                     коні ўзвіліся на дыбкі ад раптоўнага страху   \n",
       "12695  з'яўляецца адным з найстарэйшых дзейных прадпрыемстваў і найстарэйшым манетным дваром у свеце   \n",
       "\n",
       "                                                                                           pred_text  \\\n",
       "0                                                 і на гэтым кірунку нас чакаюць вялікія складанасці   \n",
       "12698                                           сваім узнікненнем горад абавязаны збегу акалічнасцей   \n",
       "6832                                 дзейнічала школа царкоўнай граматы капліца хлебазапасны магазін   \n",
       "6834                                                     коні ўзвіліся на дыбкі ад раптоўнага страху   \n",
       "12695  з'яўляецца адным з найстарэйшых дзейных прадпрыемстваў і найстарэйшым манетным дваром у свеце   \n",
       "\n",
       "       success  wer  \n",
       "0            1  0.0  \n",
       "12698        1  0.0  \n",
       "6832         1  0.0  \n",
       "6834         1  0.0  \n",
       "12695        1  0.0  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values('wer', ascending=True).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1e9feb99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2665</th>\n",
       "      <td>але пакуль краіна сама не захоча каб я паехала я не паеду</td>\n",
       "      <td>але пакуль краіна сама не захоча каб я паехала я не паеду</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8490</th>\n",
       "      <td>у канцы сезона стаў з'яўляцца ў стартавым складзе на пазіцыі цэнтральнага нападніка</td>\n",
       "      <td>у канцы сезона стаў з'яўляцца ў стартавым складзе на пазіцыі цэнтральнага нападніка</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4455</th>\n",
       "      <td>яны апранутыя ў хітоны і плашчы і маюць ярка афарбаваныя вуалі на галаве</td>\n",
       "      <td>ямы апранутыя ў хітоны плашчы і маюць ярка афарбаваныя ў але на галаве</td>\n",
       "      <td>0</td>\n",
       "      <td>0.307692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2980</th>\n",
       "      <td>у нас была яе персанальная выстава</td>\n",
       "      <td>у нас была персанальная выстава</td>\n",
       "      <td>0</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1282</th>\n",
       "      <td>я вяду дзённікі усё падлічваю</td>\n",
       "      <td>я вяду дзённікі усё падлічваю</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                     text  \\\n",
       "2665                            але пакуль краіна сама не захоча каб я паехала я не паеду   \n",
       "8490  у канцы сезона стаў з'яўляцца ў стартавым складзе на пазіцыі цэнтральнага нападніка   \n",
       "4455             яны апранутыя ў хітоны і плашчы і маюць ярка афарбаваныя вуалі на галаве   \n",
       "2980                                                   у нас была яе персанальная выстава   \n",
       "1282                                                        я вяду дзённікі усё падлічваю   \n",
       "\n",
       "                                                                                pred_text  \\\n",
       "2665                            але пакуль краіна сама не захоча каб я паехала я не паеду   \n",
       "8490  у канцы сезона стаў з'яўляцца ў стартавым складзе на пазіцыі цэнтральнага нападніка   \n",
       "4455               ямы апранутыя ў хітоны плашчы і маюць ярка афарбаваныя ў але на галаве   \n",
       "2980                                                      у нас была персанальная выстава   \n",
       "1282                                                        я вяду дзённікі усё падлічваю   \n",
       "\n",
       "      success       wer  \n",
       "2665        1  0.000000  \n",
       "8490        1  0.000000  \n",
       "4455        0  0.307692  \n",
       "2980        0  0.166667  \n",
       "1282        1  0.000000  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values('wer', ascending=True).sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b9e3653",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6117938b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pred_text</th>\n",
       "      <th>success</th>\n",
       "      <th>wer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>толькі ў раёне таксама нашы людзі вясковыя</td>\n",
       "      <td>толькі ў раёне таксама нашы людзі вясковыя</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9532</th>\n",
       "      <td>у яго нават ёсць сольныя выступы</td>\n",
       "      <td>у яго нават ёсць сольныя выступы</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9533</th>\n",
       "      <td>на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа</td>\n",
       "      <td>на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12305</th>\n",
       "      <td>шлюбам папярэднічалі перагаворы паміж сем'ямі</td>\n",
       "      <td>шлюбам папярэднічалі перагаворы паміж сем'ямі</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         text  \\\n",
       "12                 толькі ў раёне таксама нашы людзі вясковыя   \n",
       "9532                         у яго нават ёсць сольныя выступы   \n",
       "9533   на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа   \n",
       "12305           шлюбам папярэднічалі перагаворы паміж сем'ямі   \n",
       "\n",
       "                                                    pred_text  success  wer  \n",
       "12                 толькі ў раёне таксама нашы людзі вясковыя        1  0.0  \n",
       "9532                         у яго нават ёсць сольныя выступы        1  0.0  \n",
       "9533   на ўзвышшы што ў цэнтры пляцоўкі стаяла абарончая вежа        1  0.0  \n",
       "12305           шлюбам папярэднічалі перагаворы паміж сем'ямі        1  0.0  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.iloc[[12, 9532, 9533,12305]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "665304c7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aad14079",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c46a6226",
   "metadata": {},
   "source": [
    "### wer stats for test set items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "68729fc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    15801.000000\n",
       "mean         0.119767\n",
       "std          0.166523\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          0.000000\n",
       "75%          0.200000\n",
       "max          1.200000\n",
       "Name: wer, dtype: float64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgcAAAFFCAYAAAB42U7BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+lElEQVR4nO3de1hU5f7//yegIzKQpaF20EIxoFLAEDXNA+pWS1Rkt1NL/W7clVFqh49mktu2eaI85Lmj7E3qznIrhGlqZmZmCWpWJghKhVZkiilMOgLz+4OfK5d4GBQGhNfjurwuZ+5Za93rzVwzr1n3vdZyczgcDkRERET+f+6V3QERERGpWhQORERExEThQEREREwUDkRERMRE4UBERERMFA5ERETEROFA5AqsXLmSiIgI4/G4ceMYN25cuW7jyy+/JCAgwHg8b948hgwZUq7bON925OpUEe9BqXkUDqRKS0hIoFu3bqbn1q1bR0BAAElJSabne/fuzcKFCwEYMmQIAQEBpf59+umnQMkX7Jnn7rjjDrp37868efMoLi6+ov7GxcURFxfn1GsDAgL48ssvL/m60NBQPvvssyvq17nOFzAqYjtV0fneF2f/O3jw4GWtd8iQIcybN6+ce/ungwcPOtW/c9+DERERrFy5ssL6JdVTrcrugMjFhIWFMX36dH755RcaN24MQFpaGo0bNyY1NZX+/fsDcPToUbKzs2nTpo2xbExMDDExMab11atXz/h/aGgo8+bNo6ioiG+//ZZnn32W6667joceeuiy++vj43PZy56P3W7HYrHg6+tbrus9H1dtx5XO1O9sZwegxYsXs2vXLtOXev369V3Wv4pQ3u9BqZl05ECqtNtvvx2r1UpaWprx3I4dO/j73/9uem7nzp3Url2bVq1aGc95eXnh6+tr+nf2F0Xt2rXx9fWlcePGdO/enbvvvptt27ZdtD9ffvklvXv3plWrVjz66KPk5eWZ2s89pPvvf/+biIgI7rzzTjp16mR8CZ0Zihg6dCgBAQHGMkOGDOHll1/mueeeIzQ0lLlz517wcP/rr79Ou3btaNOmDa+88orx/Pl+YZ69jpUrVzJ//ny2b99u+rV8vu0kJCTQpUsXWrZsycCBA9mzZ4/RdmZIZc2aNURERNCmTRvi4uKw2+0XrF9RUREzZszg7rvvplWrVgwfPpycnBwAvvvuO26//XaOHj1qWmbw4MHGEaGioiJeeeUVOnXqRGhoKEOGDCEjI8N47ZkjIm+88QYdO3Y87/DL2e8HLy8v433g6+uLt7c3kydPpl27doSFhTFixAh++uknY9mtW7fSv39/WrVqRdu2bXnkkUeAkr/79u3bmT9/PgEBAaahprPl5eUxatQowsPDCQkJoW/fvuzatcto//zzzxkwYACtWrWiZ8+eLFu2zGg7cwStW7duBAQEXPAoxdnvwSFDhnDo0CGee+45AgICTPVITEykW7duBAcHEx0dTWpqqtF25m+bnJxMly5duOuuu1iwYAGnTp1i/PjxhIaG0qdPH7777jtjmT179jBo0CBCQkJo06YNDz30EMePHz9vH6Xq05EDqdI8PDwIDQ0lNTWVPn36kJ+fz/79+7n//vuZN28ev/32G9dffz07duygVatW1KlT57K2s2/fPnbt2sVdd911wdecOHGCJ554gr59+zJ48GC2b9/OrFmzLvhL7euvv2bevHnMmjULf39/cnNzjS/CFStW0L59e+bNm0doaCienp7GcsuWLePxxx8nOTkZDw+P8x5G/vbbb7nuuutYsmQJWVlZjB8/Hj8/P/r163fJfb333nvJzMw0/WKuX78+hw4dMr1uzZo1zJ07lxdffJGgoCASEhJ45JFH2LBhA15eXgAcOXKElJQUFi1axNGjRxk1ahR33HEHgwcPPu+233rrLVatWsW0adO44YYbmDVrFrGxsSQnJ3P77bfTpEkT1q1bx6BBgwDIzc1l165dTJkyBYD58+ezZcsWZs2aha+vLytXrmT48OGsW7cOq9Vq1KZhw4b8+9//xs3N7ZL1ONvEiRM5cuQIb7zxBlarlTfeeIPHHnuMVatWUVxczKhRo3jiiSfo0aMH+fn5fPHFF0DJofzvv/+e0NBQYmJi8PDwOO/658yZQ35+PomJiXh5eZGenk7t2rUBOHDgACNHjmT8+PGEh4cbf9cGDRrQs2dP3nvvPe6//37ee+89brjhBuNvcDHz5s2jX79+xMTEcO+99xrbWrFiBUuWLOGFF17g1ltv5ZNPPuGRRx5hzZo13HDDDUDJ33bjxo28+eab7Nmzh7Fjx7Jz50569OjBww8/zOzZs5kwYQL/+9//ABgzZgydO3fmpZde4uTJk+zcubNMtZeqReFAqrywsDA++OADAHbt2kVgYCBWq5Xg4GDS0tLo1asXaWlp3H333ablXnvtNRYvXmx67uOPP+a6664DSoYnQkNDKSwsxG63U7t2bR588MEL9iMlJQVvb2/Gjx+Ph4cHzZs3Jy0tzfTL72w///wzvr6+dOzYEQ8PD2666SZat24N/Hnoul69eqUO5YeEhPCPf/zDeHy+cOBwOJgyZQo+Pj74+/uzd+9eli5d6lQ48PT0NP1ivpD//Oc/DBkyhD59+gDwwgsvsGXLFlJSUnjggQeAksP2U6ZMMfanV69epKamXjAcJCYmMmrUKDp37gzA9OnT6dy5M59//jkdO3akV69erF271ggHH374IQEBAfj5+XHq1CkWL17MypUrad68OQBPPfUUH374IZ988gn33XcfAO7u7kyePJm6deteshZnO3jwIGvXrmXbtm14e3sDMGnSJNq0acPXX3/NLbfcQn5+Pr179zaGuAIDA4GSQ/m1a9c2jlZdyC+//EJYWJixXNOmTY22N954g0GDBhEdHQ1AkyZNGDZsGO+++y49e/Y0aly/fn2nh3+uvfZaPDw88PHxMS2zaNEiJk6cyD333AOUHGHYtGkTKSkpxtEQu93Ov/71L6677jr8/f159dVX8fT0ZODAgQAMHz6cv/3tb/zxxx/UrVuXn3/+mW7dutGkSRMAWrRo4VQfpWpSOJAqLywsjDlz5pCXl0daWprxBdu6dWvS0tLo3Lkze/fuZfTo0ablBg4cWOqw8tlzDu68805mzJjBH3/8wZtvvknDhg0JDw+/YD+ys7O54447TL8Kg4ODLxgO7r77bmbPnk2PHj245557iIiIoFOnTpf8NRsUFHTRdgA/Pz/TEYtWrVqZDkGXhwMHDhhfFAC1atXizjvv5MCBA8Zz119/vWmM3tfXlx9++OG86ztx4gSHDx8mODjYeO7aa6/Fz8+PAwcO0LFjR3r37s0bb7zBkSNHaNCgAR9++CH33nsvAD/++CMnT57kr3/9q2m9J0+eNI7IQEltyhoMALKysjh9+rTxhXnu+kNCQrjvvvvo06cP99xzD506daJnz55O/YI/4/777+epp55i69atdOjQgd69e+Pn5weUHL3at28fS5cuNV5fWFjIjTfeWOZ9uZiCggIOHjzIqFGjTO9Fu93OTTfdZDy+/vrrjSAN0KBBA/z9/Y3HZ/7ueXl51K1bl4ceeojhw4fToUMHOnTowL333mtaXq4uCgdS5QUHB1O7dm127NhhzDcAuOuuu4iPj+err77C4XAQGhpqWq5evXrccsstF1yvp6en0T5t2jR69uxJu3bt6NSp03lf73A4Sn2xX+ympj4+Prz//vts27aNLVu2GPMIFixYcNH9deaL7WIBw93dvVTfCgsLL7nOy3HmMPXZ/bqSG70GBgZy6623sm7dOrp168ZXX33FSy+9BIDNZgNKhl3O/UI+O/RdTjA4s36r1Xremf0NGjQAYNasWezevZvNmzfz5ptvsmDBAv73v/+Ztn8x3bp146OPPmLTpk1s3ryZRYsWMWPGDHr27InNZmP48OFERUWZlqlVq3w/pv/44w8AZs+eTbNmzUxtZ46YwPn/tmf35cx78MwZPs888wyRkZF88sknJCcnM3fuXFasWGEcSZCriyYkSpVnsVho1aoV27Zt45tvvjGOHISEhLB//34++eQTgoKCjDHny1G7dm3+3//7f7z88ssX/HLz8/Njz549ptMdv/7660v2vXPnzjz//PO8+uqrfPTRRxw7dgwo+dC/3FMns7Ozyc/PNx5/8803xi/QM7/ofvvtN6N93759puVr1apFUVHRRbfh5+fH7t27jceFhYV8++23pb5QnOXj48P1119vWuexY8fIzs42rfPM0MLatWuNeQgAzZs3p3bt2vz666/ccsstpn/XXnvtZfXpbAEBAeTn51NYWFhq/Wd/aQYHBzNq1ChWrVrFsWPHjNNRnakpQMOGDXnggQdYuHAh0dHRrFq1CigJRt9//32pbZ/5NX/my7qs75lz+9WgQQN8fX35+eefS23rTAi6XLfddhuPPPII7777Ltdffz0fffTRFa1PKo/CgVwVwsLCWLlyJTfddJNxqLJu3boEBATw7rvvmk5hPMNms3H48GHTv5MnT15wGwMGDODQoUNs3LjxvO2RkZH8/vvvTJ06lQMHDrB8+XK2bNlywfVt2rSJpUuXkpGRQU5ODmvWrKF+/fpcc801ANx444188cUXHDlyhIKCgrKUA4Dnn3+e/fv3s379et5++21jnN/T05M777yT1157jezsbD7++ONSQw433ngj33//PQcOHODo0aPn/cIZOnQob7/9Nh988AH79+/nhRdewG63G3MQLsfQoUOZO3cumzdvZt++fYwbN44mTZqY5ovce++9pKWl8e6779K7d2/jeW9vbx566CEmTpzI+vXrycnJYdeuXcyYMYP9+/dfdp/OaN68OX/5y18YPXo0n332GTk5OWzfvp1//etfHD9+nJycHGbPns3u3bs5dOgQH374IX/88Ycxb+DGG2/k66+/Jjc3l99///2825g7dy6bNm0iJyeHb775hp07dxqhbvjw4WzcuJG5c+eyf/9+MjMzWbFiBf/973+Bki/1OnXq8Pnnn3P06FHjCMCl3HjjjezcuZPDhw9z4sQJ3NzcePTRR3nllVdYuXIlP/74I9988w2vvvoq27dvv6zanTx5ksmTJ5OWlsahQ4fYvHkzP/30E7feeutlrU8qn4YV5KoQFhbGq6++WupsgrvuuotvvvmGsLCwUsssXry41ITEyZMnc//99593G97e3kRFRfHqq6/SvXv3Uu3XXHMN8+fP54UXXmD58uXGqWwXGuv38fFhzZo1zJ49m6KiIoKCgnj11VeNw/7/93//R3x8PK+//jr9+vVj+vTpTtUCSuZLBAQEMHjwYIqKinjwwQdNkxGnTJnC+PHjjdPuHn/8cZ599lmj/S9/+Qtr164lOjoam8123kDUp08ffvnlF+Lj4zl69Ch33HEHr7/++hUdoRk+fDi///4748aNo6CggLCwMBYuXGjUBMDf3x9/f3/27dtnCgcAY8eOpV69ekyfPp1ff/2VBg0a0LZt23I5cgAwY8YMZs+ezXPPPUdeXh6NGjWiY8eO1KlTh7p167Jv3z5WrFjB8ePHadKkCVOmTDEmF/79739n7NixdOvWjYYNG/Lxxx+XWr+Hhwfx8fEcOnQIHx8fIiIiGDlyJFDyN128eDGzZs3izTffxNPT0/glDiVHAJ599lnjPfj4448by17ME088wYQJE+jSpQutW7fm7bffZsiQIVgsFt544w3++c9/cu211xIaGkqvXr0uq27u7u4cOXKEp59+mqNHj9KoUSOeeOIJunbtelnrk8rn5riSAUIRERGpdjSsICIiIiYKByIiImKicCAiIiImCgciIiJionAgIiIiJgoHIiIiYqLrHJwlL6+A4uLyObOzQQNvjhzJv/QLawjV40+qhZnqYaZ6lLCkJOPjXYcjXS/v2gvVUXm+N9zd3bjuugtfs0Th4CzFxY5yCwdn1id/Uj3+pFqYqR5mqgc4TpwATqsW53BVPTSsICIiIiYKByIiImKicCAiIiImCgciIiJionAgIiIiJgoHIiIiYqJwICIiIiYKByIiImKicCAiIiImukJiBbGfLsLX1+eC7SdPFXLi+B8u7JGIiIhzFA4qiKW2B5HPJF+wPWVmP064sD8iIiLO0rCCiIiImCgciIiIiInCgYiIiJi4NBz89NNPPProo4SHh9O+fXuee+45CgoKANi7dy8DBgwgODiYqKgo9uzZY1p29erVREREEBISwogRI8jLyzPaHA4HM2bMoG3btoSHhxMfH4/Dodt8ioiIXA6XhoOJEydyzTXX8Omnn7J27VpycnJYuHAhdrud2NhY+vbtS2pqKlFRUcTGxmK32wHIzMxkwoQJxMfHs3XrVqxWKxMnTjTWu3z5cjZs2EBycjIpKSls3ryZZcuWuXLXREREqg2XhoODBw/Sp08fPD09ufbaa+nRoweZmZls376dwsJChg0bhsViYejQobi7u7Nt2zYAUlJSiIiIoE2bNlitVkaPHs3GjRvJz88HICkpiZiYGBo3bkyjRo2IiYkhKSnJlbsmIiJSbbg0HAwdOpSUlBRsNhtHjx5l3bp1dOrUiaysLAIDA3FzczNeGxAQQFZWFlBy5CAwMNBoa9q0KRaLhezsbKM9KCjIaA8MDDSWFRERkbJx6XUOWrduzXvvvUdYWBhFRUV07tyZgQMH8tprr+Ht7W16rdVqxWazAWCz2Uq1e3t7m9qtVmupNofDYQocl9KggfelX1SOLnaRpOqopu3vxagWZqqHmeoB+HgCqsW5XFUPl4WDoqIiHn74YR544AHeeecdbDYbzz//PFOmTOGWW24xhgjOKCgowMvLCwAvL69S7fn5+ab2MxMbz24rSzAAOHIkn+Li8pnI6Mwf8PDhmnMZJF9fnxq1vxejWpipHmaqR4k6J05yjY+nanGW8nxvuLu7XfQHscuGFX7//Xdyc3MZMmQIFouFa6+9lujoaLZt24a/vz8ZGRmmMwwyMjLw9/cHoEWLFqSnpxttOTk52O12/Pz8ztuenp5uLCsiIiJl47JwUL9+fZo0acKyZcs4ffo0J06cYNWqVQQGBhIeHo6HhweJiYnY7XaWLFlCcXEx7du3ByAyMpJNmzaRlpaGzWZjzpw5dOvWzRhq6NevHwkJCeTm5pKbm0tCQgL9+/d31a6JiIhUKy6dkDh//nw+//xz7r77brp3747D4SAuLg6LxcLChQtJTk4mLCyMFStWsGDBAiwWC1ByZGDSpEmMGTOGu+++m/z8fF544QVjvQMHDiQiIoLIyEj69OnDPffcw6BBg1y5ayIiItWGm0NXCzKU95yDS914qSaNpWkc9U+qhZnqYaZ6lKjzztKSOQf3RVd2V6qMajnnQERERK4OCgciIiJionAgIiIiJgoHIiIiYqJwICIiIiYKByIiImKicCAiIiImCgciIiJionAgIiIiJgoHIiIiYqJwICIiIiYKByIiImKicCAiIiImCgciIiJionAgIiIiJgoHIiIiYqJwICIiIiYKByIiImKicCAiIiImCgciIiJionAgIiIiJgoHIiIiYuKycBAaGmr6FxQUxIsvvmi07927lwEDBhAcHExUVBR79uwxLb969WoiIiIICQlhxIgR5OXlGW0Oh4MZM2bQtm1bwsPDiY+Px+FwuGrXREREqhWXhYNdu3YZ/7Zu3Yqnpye9evUCwG63ExsbS9++fUlNTSUqKorY2FjsdjsAmZmZTJgwgfj4eLZu3YrVamXixInGupcvX86GDRtITk4mJSWFzZs3s2zZMlftmoiISLVSKcMK69ato379+oSFhQGwfft2CgsLGTZsGBaLhaFDh+Lu7s62bdsASElJISIigjZt2mC1Whk9ejQbN24kPz8fgKSkJGJiYmjcuDGNGjUiJiaGpKSkytg1ERGRq16tytjoqlWr6N+/P25ubgBkZWURGBhoPAYICAggKyuLzp07k5mZSevWrY22pk2bYrFYyM7OpmXLlmRmZhIUFGS0BwYGkpWVVeZ+NWjgfQV7VXa+vj4u3V5lq2n7ezGqhZnqYaZ6AD6egGpxLlfVw+Xh4KeffiI1NZUpU6YYzxUUFODtbf5itlqt2Gw2AGw2W6l2b29vU7vVai3V5nA4TIHjUo4cyae4uHzmKjjzBzx8+ES5bOtq4OvrU6P292JUCzPVw0z1KFHnxEmu8fFULc5Snu8Nd3e3i/4gdvmwQlJSEnfddRdNmjQxnrNarcYQwRkFBQV4eXkB4OXlVao9Pz/f1F5QUFCqrSzBQEREREq4PBwkJyfTv39/03P+/v5kZGSYzjDIyMjA398fgBYtWpCenm605eTkYLfb8fPzO297enq6sayIiIiUjUvDwc6dO8nNzTXOUjgjPDwcDw8PEhMTsdvtLFmyhOLiYtq3bw9AZGQkmzZtIi0tDZvNxpw5c+jWrZsx1NCvXz8SEhLIzc0lNzeXhISEUgFEREREnOPSOQdJSUn06NGj1PwBi8XCwoULiYuLY+bMmTRr1owFCxZgsViAkiMDkyZNYsyYMeTl5dGuXTumTp1qLD9w4EAOHjxIZGQkDoeD6OhoBg0a5MpdExERqTbcHLpakKG8JyRGPpN8wfaUmf1q1EQbTbL6k2phpnqYqR4l6ryztGRC4n3Rld2VKqNaT0gUERGRqk3hQEREREwUDkRERMRE4UBERERMFA5ERETEROFARERETBQORERExEThQEREREwUDkRERMRE4UBERERMFA5ERETEROFARERETBQORERExEThQEREREwUDkRERMRE4UBERERMFA5ERETEROFARERETBQORERExEThQEREREwUDkRERMTE5eEgJSWFXr16ERISQvfu3UlLSwNg7969DBgwgODgYKKiotizZ49pudWrVxMREUFISAgjRowgLy/PaHM4HMyYMYO2bdsSHh5OfHw8DofDpfslIiJSXbg0HGzZsoWZM2cyffp0du7cydKlS2nSpAl2u53Y2Fj69u1LamoqUVFRxMbGYrfbAcjMzGTChAnEx8ezdetWrFYrEydONNa7fPlyNmzYQHJyMikpKWzevJlly5a5ctdERESqDZeGg/nz5/P4448TEhKCu7s7jRo1olGjRmzfvp3CwkKGDRuGxWJh6NChuLu7s23bNqDkaENERARt2rTBarUyevRoNm7cSH5+PgBJSUnExMTQuHFjGjVqRExMDElJSa7cNRERkWqjlqs2VFRUxJ49e+jatSvdu3fHbrfTvXt3xo4dS1ZWFoGBgbi5uRmvDwgIICsri86dO5OZmUnr1q2NtqZNm2KxWMjOzqZly5ZkZmYSFBRktAcGBpKVlVXmPjZo4H1lO1lGvr4+Lt1eZatp+3sxqoWZ6mGmegA+noBqcS5X1cNl4eC3337j9OnTrF+/nmXLllGrVi1iY2NZtGgRFosFb2/zF7PVasVmswFgs9lKtXt7e5varVZrqTaHw2EKHJdy5Eg+xcXlM1fBmT/g4cMnymVbVwNfX58atb8Xo1qYqR5mqkeJOidOco2Pp2pxlvJ8b7i7u130B7HLhhU8PUtS4NChQ2nYsCH169cnJiaGzZs3Y7VajSGCMwoKCvDy8gLAy8urVHt+fr6pvaCgoFRbWYKBiIiIlHBZOKhXrx6NGzc2PXfmjAJ/f38yMjJMZxhkZGTg7+8PQIsWLUhPTzfacnJysNvt+Pn5nbc9PT3dWFZERETKxqUTEgcMGMDbb7/NkSNH+P333/nPf/5Dly5dCA8Px8PDg8TEROx2O0uWLKG4uJj27dsDEBkZyaZNm0hLS8NmszFnzhy6detmDDX069ePhIQEcnNzyc3NJSEhgf79+7ty10RERKoNl805AIiNjeXYsWP06tULi8VCr169eOyxx7BYLCxcuJC4uDhmzpxJs2bNWLBgARaLBSg5MjBp0iTGjBlDXl4e7dq1Y+rUqcZ6Bw4cyMGDB4mMjMThcBAdHc2gQYNcuWsiIiLVhptDVwsylPeExMhnki/YnjKzX42aaKNJVn9SLcxUDzPVo0Sdd5aWTEi8L7qyu1JlVMsJiSIiInJ1UDgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE6fDQXp6ekX2Q0RERKoIp8PBX//6V/r27cvixYv59ddfK7JPIiIiUomcDgeffvop0dHRrFmzhq5duzJ8+HCSk5P5448/KrJ/IiIi4mJOh4P69eszbNgwVqxYQUpKCq1atWL+/Pl06NCBcePGsW3btorsp4iIiLjIZU1IbNasGZ07d6Zjx47Y7XZSU1N58skn6dmzJ9u3by/vPoqIiIgL1SrLi3/44Qfef/99UlJSOHr0KL169eLf//43YWFh2O12FixYwJgxY9i8eXNF9VdEREQqmNPh4G9/+xt79uyhXbt2jBw5kh49euDp6Wm0WywWhgwZwmuvvVYhHRURERHXcHpYoXv37nz88ce89dZbREZGmoLBGddff/1FT3kcMmQILVu2JDQ0lNDQUHr27Gm07d27lwEDBhAcHExUVBR79uwxLbt69WoiIiIICQlhxIgR5OXlGW0Oh4MZM2bQtm1bwsPDiY+Px+FwOLtrIiIichanw0FUVNR5v3B/+eUXfvvtN6c3+M9//pNdu3axa9cu1q1bB4Ddbic2Npa+ffuSmppKVFQUsbGx2O12ADIzM5kwYQLx8fFs3boVq9XKxIkTjXUuX76cDRs2kJycTEpKCps3b2bZsmVO90lERET+5HQ4eOaZZ9i6dWup57/44gueeeaZK+rE9u3bKSwsZNiwYVgsFoYOHYq7u7txBkRKSgoRERG0adMGq9XK6NGj2bhxI/n5+QAkJSURExND48aNadSoETExMSQlJV1Rn0RERGoqp8PBt99+S1hYWKnnW7duXWoI4GJmzpxJ27ZtGThwIF9++SUAWVlZBAYG4ubmZrwuICCArKwsoOTIQWBgoNHWtGlTLBYL2dnZRntQUJDRHhgYaCwrIiIiZeP0hMRatWrx+++/l3r+6NGjTm/s//7v/2jevDkWi4UPPviAESNGkJycTEFBAd7e3qbXWq1WbDYbADabrVS7t7e3qd1qtZZqczgcpsBxKQ0aeF/6ReXI19fHpdurbDVtfy9GtTBTPcxUD8CnZF6bamHmqno4HQ7uueceXn75ZebMmUP9+vWBkmAwY8YM7rnnHqfWERwcbPw/KiqK1atXs3nzZqxWqzFEcEZBQQFeXl4AeHl5lWrPz883tRcUFJRqK0swADhyJJ/i4vKZyOjMH/Dw4RPlsq2rga+vT43a34tRLcxUDzPVo0SdEye5xsdTtThLeb433N3dLvqD2Olhhbi4OAoLC4mIiCAqKooBAwYQERFBYWEhcXFxl9U5Nzc3HA4H/v7+ZGRkmCY8ZmRk4O/vD0CLFi1MZ0Hk5ORgt9vx8/M7b3t6erqxrIiIiJSN00cO6tevz3//+1+++OILsrKycDgctGjRgnbt2jm1/PHjx9m9ezfh4eF4eHiwZs0a0tLSGD9+PDfffDMeHh4kJiYyaNAg3n33XYqLi2nfvj0AkZGRPPDAA6SlpXH77bczZ84cunXrZgw19OvXj4SEBDp37gxAQkICgwcPLmstREREhDJeIRGgXbt2TgeCsxUWFvLKK69w4MABPDw8aNasGQsWLKBZs2YALFy4kLi4OGbOnGm0WSwWoOTIwKRJkxgzZgx5eXm0a9eOqVOnGuseOHAgBw8eJDIyEofDQXR0NIMGDSpzH0VERATcHGW4WtCWLVtITU3lyJEjFBcXm9qmTZtW7p1ztfKecxD5TPIF21Nm9qtRY2kaR/2TamGmepipHiXqvLO0ZM7BfdGV3ZUqo0rOOZg5cyaPPfYYe/bsoaioqFw6JyIiIlWP08MKK1asYPbs2fTo0aMi+yMiIiKVzOkjBx4eHjRv3rwi+yIiIiJVgNPh4JFHHuGtt96isLCwIvsjIiIilczpYYUNGzbw3XffsWnTJm699VZq1TIvmpiYWO6dExEREddzOhy0bduWtm3bVmRfREREpApwOhw88cQTFdkPERERqSKcnnMA8Mcff7BmzRpef/11jh8/DkB2dnaZbr4kIiIiVZvTRw727dvH8OHD8fLy4uDBg/Tu3ZtrrrmGlStXcvjwYaZPn16R/RQREREXcfrIweTJk4mOjmbdunXGZY0BIiIi+PLLLyukcyIiIuJ6ToeDb7/9lujo0pex9PX15bfffivXTomIiEjlcToc+Pj4cPjw4VLP7927l0aNGpVrp0RERKTyOB0OoqKimDZtGtnZ2bi5uWGz2diyZQvTpk3j/vvvr8g+ioiIiAs5PSFx1KhRuLm5ERUVxcmTJ+nfvz+1a9fmwQcf5NFHH63IPoqIiIgLOR0O3N3dGT16NI899hg//vgjNpuNZs2a4e194Vs+ioiIyNXH6XBwhsViwd/fvyL6IiIiIlWA0+FgyJAhuLm5XbBd91YQERGpHsp0b4WznT59mn379pGWlsbgwYPLvWMiIiJSOa743gqJiYns27ev3DokIiIilatM91Y4ny5durBmzZry6IuIiIhUAVcUDhwOB8nJydSvX7+8+iMiIiKVzOlhhYiICNOERIfDQV5eHgBTp04t/56JiIhIpXA6HIwcOdL02M3Njfr169OyZUuuu+66Mm00Ly+PXr160axZM/773/8CJZdhjouLY//+/TRr1ozJkydzxx13GMusXr2aWbNmcfToUdq1a8e0adOM7TocDmbOnMl7772Hw+EgOjqasWPHXvTsChERETk/p8NBVFRUuW30pZdewt/fn+LiYgDsdjuxsbEMGzaMwYMH88477xAbG8uGDRuwWCxkZmYyYcIEXn/9dW6//Xb++c9/MnHiRObOnQvA8uXL2bBhA8nJybi5ufH3v/+dm2++mQcffLDc+iwiIlJTOB0OUlNTnV5pmzZtLti2fft2srOzuf/++1mxYoXxXGFhIcOGDcPNzY2hQ4eSkJDAtm3b6Ny5MykpKURERBjrHT16NL179yY/Px9vb2+SkpKIiYmhcePGAMTExLB8+XKFAxERkctwWRdBcjgcAKUen3lu7969512H3W7nxRdf5OWXX+a7774zns/KyiIwMNA0DBAQEEBWVhadO3cmMzOT1q1bG21NmzbFYrGQnZ1Ny5YtyczMJCgoyGgPDAwkKyvL2V0zNGjg2ktB+/r6uHR7la2m7e/FqBZmqoeZ6gH4eAKqxblcVQ+nw8Ebb7zBnDlzeOqppwgODgZg9+7dvPLKK4wcOZJOnTpdch2vv/467du3JzAw0BQOCgoKSt2jwWq1YrPZALDZbKXavb29Te1Wq7VUm8PhKNO8gyNH8ikudlz6hU5w5g94+PCJctnW1cDX16dG7e/FqBZmqoeZ6lGizomTXOPjqVqcpTzfG+7ubhf9Qex0OJgyZQrx8fFGMADo0KEDVquVsWPHsn79+osu/8MPP7Bq1SqSk5NLtVmtVvLz803PFRQU4OXlBYCXl1ep9vz8fFN7QUFBqTZNSBQRESk7p69z8PPPP5/3y9bd3Z3c3NxLLr9jxw5+++03evbsSYcOHZgyZQrffPMNHTp0oHnz5mRkZJiGJzIyMowbPLVo0YL09HSjLScnB7vdjp+f33nb09PTdXMoERGRy+R0OOjUqRPjx4/niy++oKCgAJvNxhdffEFcXJxTQwr33nsvH330EUlJSSQlJTFq1CiCgoJISkqibdu2eHh4kJiYiN1uZ8mSJRQXF9O+fXsAIiMj2bRpE2lpadhsNubMmUO3bt2MoYZ+/fqRkJBAbm4uubm5JCQk0L9//8uriIiISA3n9LDCtGnTmDJlCv/4xz8oKioCwMPDgz59+jB+/PhLLu/p6Ymnp6fx2MfHh1q1auHr6wvAwoULiYuLY+bMmTRr1owFCxZgsViAkiMDkyZNYsyYMeTl5dGuXTvThZcGDhzIwYMHiYyMNK5zMGjQIGd3TURERM7i5jj7WL4T8vPzOXjwIA6HgyZNmpSaKHg1K+8JiZHPlJ5fcUbKzH41aqKNJln9SbUwUz3MVI8Sdd5ZWjIh8b7oyu5KleHKCYllvrfCqVOnOHnyJM2bN69WwUBERERKOB0Ofv/9d0aMGEGHDh0YNGiQMQlx4sSJzJ49u8I6KCIiIq7ldDiYPHkyAJs3bzbNHejRowcbNmwo/56JiIhIpXB6QuKWLVtYsmQJjRo1Mj1/yy238NNPP5V7x0RERKRyOH3k4MwZCuf69ddfjYsRiYiIyNWvTNc5eOutt0wXKjp27BizZ88mIiKiQjonIiIirud0OIiLi+PgwYN06dKFU6dOERsbS9euXbHb7YwdO7Yi+ygiIiIu5PScA29vbxITE9mxYwcZGRnYbDYCAwPp2LGj7mEgIiJSjTgVDk6fPk1oaCjvv/8+YWFhhIWFVXS/REREpJI4NaxQu3ZtbrnlllJ3RhQREZHqx+k5B8899xwvvfQSX331FadOnarIPomIiEglcnrOwcMPPwyU3OTofHMM9u7dW369EhERkUpzyXDw8MMPM2vWLBITEwGw2Wy4ublRt27dCu+ciIiIuN4lw8Fnn32G3W4nPDwcgNatW5OcnEyTJk0qvHMiIiLiepecc3DuHZ3LeIdnERERucqU+ZbNIiIiUr05NSHxrbfeMu6fcPr0aRITE6lXr57pNU888UT5905ERERc7pLhoE2bNnzzzTfG49DQUNLT002v0RUSRUREqo9LhoO3337bFf0QERGRKkJzDkRERMRE4UBERERMXBoOXnrpJTp37kzr1q3p2rUrixYtMtr27t3LgAEDCA4OJioqij179piWXb16NREREYSEhDBixAjy8vKMNofDwYwZM2jbti3h4eHEx8frlEsREZHL5NJw8Ne//pW1a9eyc+dO3nnnHVavXs26deuw2+3ExsbSt29fUlNTiYqKIjY2FrvdDkBmZiYTJkwgPj6erVu3YrVamThxorHe5cuXs2HDBpKTk0lJSWHz5s0sW7bMlbsmIiJSbbg0HDRr1sw4JRJKznL48ccf2b59O4WFhQwbNgyLxcLQoUNxd3dn27ZtAKSkpBAREUGbNm2wWq2MHj2ajRs3GneJTEpKIiYmhsaNG9OoUSNiYmJISkpy5a6JiIhUG07feKm8vP766yxatAibzUbTpk3p27cva9euJTAw0HRKZEBAAFlZWXTu3JnMzExat25ttDVt2hSLxUJ2djYtW7YkMzOToKAgoz0wMJCsrKwy961BA+8r27ky8vX1cen2KltN29+LUS3MVA8z1QPw8QRUi3O5qh4uDwePPPIIDz/8MHv37mXDhg1YrVYKCgrw9jZ/MVutVmw2G1Bys6dz2729vU3tVqu1VJvD4SjTNRiOHMmnuLh85io48wc8fPhEuWzrauDr61Oj9vdiVAsz1cNM9ShR58RJrvHxVC3OUp7vDXd3t4v+IK6UsxXc3Ny4/fbbqVu3LvPmzcNqtRpDBGcUFBQYQxBeXl6l2vPz803tBQUFpdp0cSYREZGyq9RTGQsLC/nxxx/x9/cnIyPDdIZBRkYG/v7+ALRo0cJ0VcacnBzsdjt+fn7nbU9PTzeWFRERkbJxWTgoLi5m+fLlHD9+nOLiYnbv3s2yZcto37494eHheHh4kJiYiN1uZ8mSJRQXF9O+fXsAIiMj2bRpE2lpadhsNubMmUO3bt2MoYZ+/fqRkJBAbm4uubm5JCQk0L9/f1ftmoiISLXi0jkH69evZ+bMmZw+fZqGDRvy0EMPMWTIENzc3Fi4cCFxcXHMnDmTZs2asWDBAiwWC1ByZGDSpEmMGTOGvLw82rVrx9SpU431Dhw4kIMHDxIZGYnD4SA6OppBgwa5ctdERESqDTeHrhZkKO8JiZHPJF+wPWVmvxo10UaTrP6kWpipHmaqR4k67ywtmZB4X3Rld6XKqPYTEkVERKTqUjgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMXFZOLDb7YwfP56uXbsSGhpK3759+eijj4z2vXv3MmDAAIKDg4mKimLPnj2m5VevXk1ERAQhISGMGDGCvLw8o83hcDBjxgzatm1LeHg48fHxOBwOV+2aiIhIteKycFBYWMgNN9zA22+/zY4dO3jmmWcYM2YMP/zwA3a7ndjYWPr27UtqaipRUVHExsZit9sByMzMZMKECcTHx7N161asVisTJ0401r18+XI2bNhAcnIyKSkpbN68mWXLlrlq10RERKoVl4UDLy8vRo4cyc0334y7uzudO3fm1ltvZc+ePWzfvp3CwkKGDRuGxWJh6NChuLu7s23bNgBSUlKIiIigTZs2WK1WRo8ezcaNG8nPzwcgKSmJmJgYGjduTKNGjYiJiSEpKclVuyYiIlKtVNqcg6NHj3LgwAH8/f3JysoiMDAQNzc3oz0gIICsrCyg5MhBYGCg0da0aVMsFgvZ2dlGe1BQkNEeGBhoLCsiIiJlU6syNlpYWMizzz5L7969ue2229iwYQPe3t6m11itVmw2GwA2m61Uu7e3t6ndarWWanM4HKbAcSkNGnhf+kXlyNfXx6Xbq2w1bX8vRrUwUz3MVA/AxxNQLc7lqnq4PBwUFxfz7LPPUlxczKRJk4CSIHBmiOCMgoICvLy8gJIhiXPb8/PzTe0FBQWl2soSDACOHMmnuLh8JjI68wc8fPhEuWzrauDr61Oj9vdiVAsz1cNM9ShR58RJrvHxVC3OUp7vDXd3t4v+IHbpsILD4SAuLo7c3Fzmz5+PxWIBwN/fn4yMDNMZBhkZGfj7+wPQokUL0tPTjbacnBzsdjt+fn7nbU9PTzeWFRERkbJxaTiYOHEi+/fv59VXX6Vu3brG8+Hh4Xh4eJCYmIjdbmfJkiUUFxfTvn17ACIjI9m0aRNpaWnYbDbmzJlDt27djKGGfv36kZCQQG5uLrm5uSQkJNC/f39X7pqIiEi14bJhhUOHDrF8+XIsFgv33HOP8fyjjz7KiBEjWLhwIXFxccycOZNmzZqxYMEC48hCixYtmDRpEmPGjCEvL4927doxdepUYx0DBw7k4MGDREZG4nA4iI6OZtCgQa7aNRERkWrFzaGrBRnKe85B5DPJF2xPmdmvRo2laRz1T6qFmephpnqUqPPO0pI5B/dFV3ZXqoxqO+dAREREqj6FAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExUTgQERERE4UDERERMVE4EBEREROFAxERETFROBAREREThQMRERExcWk4WLJkCQMGDODOO+9k3Lhxpra9e/cyYMAAgoODiYqKYs+ePab21atXExERQUhICCNGjCAvL89oczgczJgxg7Zt2xIeHk58fDwOh8Ml+yQiIlLduDQcNGzYkNjYWKKjo03P2+12YmNj6du3L6mpqURFRREbG4vdbgcgMzOTCRMmEB8fz9atW7FarUycONFYfvny5WzYsIHk5GRSUlLYvHkzy5Ytc+WuiYiIVBsuDQd/+ctf6N69O9dee63p+e3bt1NYWMiwYcOwWCwMHToUd3d3tm3bBkBKSgoRERG0adMGq9XK6NGj2bhxI/n5+QAkJSURExND48aNadSoETExMSQlJbly10RERKqNWpXdAYCsrCwCAwNxc3MzngsICCArK4vOnTuTmZlJ69atjbamTZtisVjIzs6mZcuWZGZmEhQUZLQHBgaSlZVV5n40aOB9ZTtSRr6+Pi7dXmWraft7MaqFmephpnoAPp6AanEuV9WjSoSDgoICvL3NX8xWqxWbzQaAzWYr1e7t7W1qt1qtpdocDocpcFzKkSP5FBeXz1wFZ/6Ahw+fKJdtXQ18fX1q1P5ejGphpnqYqR4l6pw4yTU+nqrFWcrzveHu7nbRH8RVIhxYrVZjiOCMgoICvLy8APDy8irVnp+fb2ovKCgo1VaWYFAT+VxTF886F34LnDxVyInjf7iwRyIiUhVUiXDg7+/P4sWLTb/0MzIyGDRoEAAtWrQgPT3deH1OTg52ux0/Pz9Te6tWrQBIT0/H39/fxXtx9fGsU4vIZ5Iv2J4ysx/K7CIiNY9LJyQWFhZy6tQpiouLKSoq4tSpU5w+fZrw8HA8PDxITEzEbrezZMkSiouLad++PQCRkZFs2rSJtLQ0bDYbc+bMoVu3bsZQQ79+/UhISCA3N5fc3FwSEhLo37+/K3dNRESk2nDpkYNFixYxf/584/H7779PVFQU06dPZ+HChcTFxTFz5kyaNWvGggULsFgsQMmRgUmTJjFmzBjy8vJo164dU6dONdYzcOBADh48SGRkJA6Hg+joaOOog4iIiJSNS8PByJEjGTly5HnbgoKCWLly5QWX7dOnD3369Dlvm5ubG2PGjGHMmDHl0k8REZGaTJdPFhEREROFAxERETGpEmcrSNVkP1100es16FRHEZHqSeHgKuWKaxRYantc0amO5/bx3KChcCEiUjUpHFylroZrFFwNfRQRkdIUDqqpSw0JiIiIXIjCQTV1qSEBKPnlLiIici6drSAiIiImCgciIiJiomEFqbIudUbGKXsRdSweF2zX2RAiIpdH4UCqLGfOdrjSsyF022oRkdIUDqRG0+mWIiKlac6BiIiImOjIgVSamnAthgsNW5zZbw1biEhVpHAglcaZyzNf7TRsISJXI4UDkWpMEy5F5HIoHIhUYzpyISKXQxMSRURExERHDuSy1YQJhZdyqcP2IiJXI32qyWWr6hMKXRFenDlsLyJytVE4qKL0i/TKXQ13prxUgNGEQRGpDNXm2+fYsWPExcXx+eefU69ePZ588kn69+9f2d26bPpFWjNcKsD8b3qfah8eFJBEqp5qEw4mTZpE7dq1+eyzz9i7dy+PPPIIQUFBBAQEVHbXRC6bM0M3lX22wZWeLnk17KNITVMtwoHNZmP9+vWkpKRgtVoJCwuje/fuvP/++4wZM8bp9bi7u5VrvxpeV/eKtnep5a+03RXbqOrtl3qNM/MWKnsfrvR9dKXve886tRg+ef0F2996/i8UXEEfL/U3uNTdOU+dKiQ//+QF2729Palzsbt/XmL58ljHucufu7/O9KG6cfPxAe865f65fLUrr3pcaj1uDofDUS5bqkTfffcdgwYNYvfu3cZzixcv5ssvv+S1116rxJ6JiIhcfarFdQ5sNhve3t6m57y9vbHZbJXUIxERkatXtQgHXl5e5Ofnm57Lz8/Hy8urknokIiJy9aoW4eDWW2+lqKiI77//3nguPT0df3//yuuUiIjIVapahAMvLy969OjB3Llzsdls7Nixg40bNxIZGVnZXRMREbnqVIsJifDndQ62bt1KvXr1eOqpp67q6xyIiIhUlmoTDkRERKR8VIthBRERESk/CgciIiJionAgIiIiJgoHIiIiYqJwcJmOHTvG448/TmhoKF26dCEpKemCr129ejURERGEhIQwYsQI8vLyXNdRF3G2Hp988gmDBg0iLCyMDh068Nxzz3H8+HHXdraCleW9cca8efMICAjg888/r/gOulhZ6nHkyBGefvppwsLCaNOmDc8884zrOuoiZanH0qVL6d69O61btyYyMpJPPvnEZf10hSVLljBgwADuvPNOxo0bd9HX1oTPUWfr4YrPUYWDy3T2XSBnzJjBpEmTyMjIKPW6zMxMJkyYQHx8PFu3bsVqtTJx4sRK6HHFcrYeJ06c4LHHHmPLli2sXbuWo0ePMm3atEroccVxthZnZGdns27dOnx9fV3YS9cpSz0ef/xxfH192bRpE59//jnDhw93cW8rnrP1+Prrr3n55ZeZNWsWO3bsYOTIkYwePbpafSk2bNiQ2NhYoqOjL/q6mvI56mw9XPI56pAyKygocNxxxx2OAwcOGM+NGTPG8dJLL5V67cyZMx1PP/208fiHH35w3H777Y4TJ064pK+uUJZ6nOujjz5y9OrVqyK751KXU4uhQ4c6Pv30U0fXrl0dW7dudUU3XaYs9ThTg8LCQld20aXKUo8PPvjAER0dbXouLCzMsXv37grvp6vNmjXL8eyzz16wvSZ8jp7tUvU4V0V8jurIwWX4/vvv8fDwwM/Pz3guMDCQrKysUq/NzMwkMDDQeNy0aVMsFgvZ2dku6asrlKUe59qxYwctWrSoyO65VFlrkZSURL169bjnnntc1UWXKks9du/ejZ+fH2PHjqVt27ZER0ezfft2V3a3wpWlHp06daKoqIidO3dSVFTEmjVrsFqt3Hbbba7scpVQEz5Hr0RFfI5e+AbkckFluQtkTbhj5OXu45dffsl7773HsmXLKrJ7LlWWWhw7dox58+axdOlSV3XP5cpSj9zcXD777DMmT57M9OnTWb9+PbGxsaxfv5769eu7qssVqiz18PLyonv37gwdOpTi4mLq1KnD/Pnz8fT0dFV3q4ya8Dl6uSrqc1RHDi5DWe4CWRPuGHk5+/j111/z5JNPMnv27Gp15KAstXj55ZcZPHgwjRs3dlX3XK4s9ahTpw4333wz999/P7Vr1+a+++7jhhtuYOfOna7qboUrSz3effddVq1axfvvv8+3337LvHnzePrppzl06JCrultl1ITP0ctRkZ+jCgeXoSx3gWzRogXp6enG45ycHOx2u+mw4tWurHfF/O6773j00Ud58cUX6dixo4t66RplqcW2bdt488036dChAx06dODnn3/mySefZNGiRS7sccUqSz0CAgJc2LPKUZZ67Nu3j65du9KsWTPc3d3p2LEjN910E1999ZXrOlxF1ITP0bKq6M9RhYPLUJa7QEZGRrJp0ybS0tKw2WzMmTOHbt26lTpEdjUrSz327dvHP/7xD55//nm6d+9eCb2tWGWpxYoVK3j//fdJSkoiKSmJhg0b8q9//YuhQ4dWQs8rRlnq0aNHD44fP86qVasoKiriww8/JDc3l9atW1dCzytGWerRqlUrNm/ezI8//ojD4WDbtm3s37+/Wh1pKyws5NSpUxQXF1NUVMSpU6c4ffp0qdfVhM9RcL4eLvkcLdfpjTVIXl6eIzY21hEcHOzo1KmTY9WqVUZbSEiIIzU11XickpLi6NKliyM4ONjx6KOPOo4cOVIJPa5YztZj3LhxjoCAAEdISIjpX3VSlvfG2arj2QoOR9nqkZqa6ujTp48jJCTEERUVdcFaXc2crUdxcbFj7ty5ji5dujhCQkIcPXv2dKxcubKSel0x5s6d67jttttM/87M0q+Jn6PO1sMVn6O6K6OIiIiYaFhBRERETBQORERExEThQEREREwUDkRERMRE4UBERERMFA5ERETEROFARERETBQORERExEThQEREREz+P6B9cjyr61U2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df_results['wer'].describe())\n",
    "\n",
    "df_results['wer'].plot.hist(bins=50, figsize=(8, 5));\n",
    "plt.title('WER distribution over Test set items');\n",
    "plt.axvline(1, color='red', alpha=0.5);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c8709da",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4361fbb0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e968412",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}