skill_classifier/03_text_categorizer_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "90b99c95",
   "metadata": {
    "id": "90b99c95"
   },
   "source": [
    "## Let's build a TextCategorizer Model using spaCy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06ac01a3",
   "metadata": {
    "id": "06ac01a3"
   },
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import spacy\n",
    "from spacy.tokens import Doc, DocBin\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "w4FB5RX8ddHI",
   "metadata": {
    "id": "w4FB5RX8ddHI"
   },
   "outputs": [],
   "source": [
    "RUN_IN_COLAB = 'google.colab' in str(get_ipython())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "SDkkGfTVcz2O",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SDkkGfTVcz2O",
    "outputId": "681b5e50-291d-40c4-d43b-339474a539ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting de-core-news-lg==3.4.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/de_core_news_lg-3.4.0/de_core_news_lg-3.4.0-py3-none-any.whl (567.8 MB)\n",
      "\u001b[K     |████████████████████████████████| 567.8 MB 21 kB/s \n",
      "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.8/dist-packages (from de-core-news-lg==3.4.0) (3.4.4)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (57.4.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (3.3.0)\n",
      "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (0.7.0)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (6.3.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.23.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (1.10.2)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.0.8)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (0.10.1)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (8.1.5)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.4.5)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.0.7)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (0.10.1)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (1.21.6)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (1.0.4)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (3.0.8)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (21.3)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (4.64.1)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.11.3)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (1.0.9)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (3.0.10)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (3.0.9)\n",
      "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (4.4.0)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2022.12.7)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (0.7.9)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (0.0.3)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (7.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->de-core-news-lg==3.4.0) (2.0.1)\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('de_core_news_lg')\n"
     ]
    }
   ],
   "source": [
    "if RUN_IN_COLAB:\n",
    "  !python -m spacy download de_core_news_lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "oXGPunV6d4pI",
   "metadata": {
    "id": "oXGPunV6d4pI"
   },
   "outputs": [],
   "source": [
    "# TODO: Remove\n",
    "def load_model(name): \n",
    "    nlp = spacy.load(name)\n",
    "    de_stop_words = nlp.Defaults.stop_words\n",
    "    return nlp, de_stop_words\n",
    "\n",
    "!mkdir -p ./data/processed/raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bKhQgZz2Dk0i",
   "metadata": {
    "id": "bKhQgZz2Dk0i"
   },
   "outputs": [],
   "source": [
    "# Read training and dev datasets splits\n",
    "BASE_PATH = os.path.join(\"./data/processed/raw\")\n",
    "\n",
    "def get_split(split, base_path=BASE_PATH): \n",
    "    return pd.read_csv(os.path.join(base_path, f\"{split}.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5f9ca61",
   "metadata": {
    "id": "e5f9ca61"
   },
   "outputs": [],
   "source": [
    "from src.models.helpers import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e64464a0",
   "metadata": {
    "id": "e64464a0"
   },
   "outputs": [],
   "source": [
    "large_model = \"de_core_news_lg\"\n",
    "\n",
    "nlp, stop_words = load_model(large_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "O3z8_mCqDroq",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 288
    },
    "id": "O3z8_mCqDroq",
    "outputId": "a65af6bd-01a9-44d9-adf9-b9facf56e3dc"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-f9506a1b-ab50-4e93-890c-a2109906ab59\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "      <th>cats</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Wenn Ihnen eine angenehme Arbeitsatmosphäre mi...</td>\n",
       "      <td>[Wenn, Ihnen, eine, angenehme, Arbeitsatmosphä...</td>\n",
       "      <td>['angenehm', 'arbeitsatmosphäre', 'enthusiasmu...</td>\n",
       "      <td>angenehm arbeitsatmosphäre enthusiasmus wert f...</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>***Abitur / Fachhochschulreife</td>\n",
       "      <td>[*, *, *, Abitur, /, Fachhochschulreife]</td>\n",
       "      <td>['abitur', 'fachhochschulreife']</td>\n",
       "      <td>abitur fachhochschulreife</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Routinierter Umgang mit MS Office-Anwendungen</td>\n",
       "      <td>[Routinierter, Umgang, mit, MS, Office-Anwendu...</td>\n",
       "      <td>['routinierter', 'umgang', 'ms', 'office-anwen...</td>\n",
       "      <td>routinierter umgang ms office-anwendunge</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f9506a1b-ab50-4e93-890c-a2109906ab59')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-f9506a1b-ab50-4e93-890c-a2109906ab59 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-f9506a1b-ab50-4e93-890c-a2109906ab59');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Wenn Ihnen eine angenehme Arbeitsatmosphäre mi...   \n",
       "1                     ***Abitur / Fachhochschulreife   \n",
       "2      Routinierter Umgang mit MS Office-Anwendungen   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [Wenn, Ihnen, eine, angenehme, Arbeitsatmosphä...   \n",
       "1           [*, *, *, Abitur, /, Fachhochschulreife]   \n",
       "2  [Routinierter, Umgang, mit, MS, Office-Anwendu...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0  ['angenehm', 'arbeitsatmosphäre', 'enthusiasmu...   \n",
       "1                   ['abitur', 'fachhochschulreife']   \n",
       "2  ['routinierter', 'umgang', 'ms', 'office-anwen...   \n",
       "\n",
       "                                           text_proc  \\\n",
       "0  angenehm arbeitsatmosphäre enthusiasmus wert f...   \n",
       "1                          abitur fachhochschulreife   \n",
       "2           routinierter umgang ms office-anwendunge   \n",
       "\n",
       "                                           cats label  \n",
       "0  {'tech': False, 'soft': False, 'none': True}  none  \n",
       "1  {'tech': False, 'soft': False, 'none': True}  none  \n",
       "2  {'tech': True, 'soft': False, 'none': False}  tech  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = get_split(split=\"training\")\n",
    "train_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65849c18",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 288
    },
    "id": "65849c18",
    "outputId": "c6f390ee-ba7e-4289-f896-30653b3cbe4d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-53fe3912-f2c2-4e9b-9f2a-804bd5bd14e9\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "      <th>cats</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Assistenz der Geschäftsführung m/w</td>\n",
       "      <td>[Assistenz, der, Geschäftsführung, m, /, w]</td>\n",
       "      <td>['assistenz', 'geschäftsführung', 'm', 'w']</td>\n",
       "      <td>assistenz geschäftsführung m w</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Detaillierte Kenntnisse des Java Memory Models.</td>\n",
       "      <td>[Detaillierte, Kenntnisse, des, Java, Memory, ...</td>\n",
       "      <td>['detaillierte', 'kenntnis', 'java', 'memory',...</td>\n",
       "      <td>detaillierte kenntnis java memory model</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Umzugsbereitschaft in Richtung Osnabrück (Wohn...</td>\n",
       "      <td>[Umzugsbereitschaft, in, Richtung, Osnabrück, ...</td>\n",
       "      <td>['umzugsbereitschaft', 'richtung', 'osnabrück'...</td>\n",
       "      <td>umzugsbereitschaft richtung osnabrück wohnsitz...</td>\n",
       "      <td>{'tech': False, 'soft': True, 'none': False}</td>\n",
       "      <td>soft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-53fe3912-f2c2-4e9b-9f2a-804bd5bd14e9')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-53fe3912-f2c2-4e9b-9f2a-804bd5bd14e9 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-53fe3912-f2c2-4e9b-9f2a-804bd5bd14e9');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0                 Assistenz der Geschäftsführung m/w   \n",
       "1    Detaillierte Kenntnisse des Java Memory Models.   \n",
       "2  Umzugsbereitschaft in Richtung Osnabrück (Wohn...   \n",
       "\n",
       "                                              tokens  \\\n",
       "0        [Assistenz, der, Geschäftsführung, m, /, w]   \n",
       "1  [Detaillierte, Kenntnisse, des, Java, Memory, ...   \n",
       "2  [Umzugsbereitschaft, in, Richtung, Osnabrück, ...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0        ['assistenz', 'geschäftsführung', 'm', 'w']   \n",
       "1  ['detaillierte', 'kenntnis', 'java', 'memory',...   \n",
       "2  ['umzugsbereitschaft', 'richtung', 'osnabrück'...   \n",
       "\n",
       "                                           text_proc  \\\n",
       "0                     assistenz geschäftsführung m w   \n",
       "1            detaillierte kenntnis java memory model   \n",
       "2  umzugsbereitschaft richtung osnabrück wohnsitz...   \n",
       "\n",
       "                                           cats label  \n",
       "0  {'tech': False, 'soft': False, 'none': True}  none  \n",
       "1  {'tech': True, 'soft': False, 'none': False}  tech  \n",
       "2  {'tech': False, 'soft': True, 'none': False}  soft  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_df = get_split(split=\"dev\")\n",
    "dev_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "T6RPz3GQeZ0n",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 288
    },
    "id": "T6RPz3GQeZ0n",
    "outputId": "83ca9c4b-b1bf-423e-c965-04c17cf06557"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-2dd649c7-cf81-4762-a0c3-80eab7172cd2\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tokens_norm</th>\n",
       "      <th>text_proc</th>\n",
       "      <th>cats</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Umfangreiche Kenntnisse mind. eines der CAD-Sy...</td>\n",
       "      <td>[Umfangreiche, Kenntnisse, mind., eines, der, ...</td>\n",
       "      <td>['umfangreich', 'kenntnis', 'mind.', 'cad-syst...</td>\n",
       "      <td>umfangreich kenntnis mind. cad-system proe wf4...</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Anschreiben:</td>\n",
       "      <td>[Anschreiben, :]</td>\n",
       "      <td>['anschreiben']</td>\n",
       "      <td>anschreiben</td>\n",
       "      <td>{'tech': False, 'soft': False, 'none': True}</td>\n",
       "      <td>none</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sehr gute Kenntnisse der Office-Anwendungen.</td>\n",
       "      <td>[Sehr, gute, Kenntnisse, der, Office-Anwendung...</td>\n",
       "      <td>['kenntnis', 'office-anwendunge']</td>\n",
       "      <td>kenntnis office-anwendunge</td>\n",
       "      <td>{'tech': True, 'soft': False, 'none': False}</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2dd649c7-cf81-4762-a0c3-80eab7172cd2')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-2dd649c7-cf81-4762-a0c3-80eab7172cd2 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-2dd649c7-cf81-4762-a0c3-80eab7172cd2');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Umfangreiche Kenntnisse mind. eines der CAD-Sy...   \n",
       "1                                       Anschreiben:   \n",
       "2       Sehr gute Kenntnisse der Office-Anwendungen.   \n",
       "\n",
       "                                              tokens  \\\n",
       "0  [Umfangreiche, Kenntnisse, mind., eines, der, ...   \n",
       "1                                   [Anschreiben, :]   \n",
       "2  [Sehr, gute, Kenntnisse, der, Office-Anwendung...   \n",
       "\n",
       "                                         tokens_norm  \\\n",
       "0  ['umfangreich', 'kenntnis', 'mind.', 'cad-syst...   \n",
       "1                                    ['anschreiben']   \n",
       "2                  ['kenntnis', 'office-anwendunge']   \n",
       "\n",
       "                                           text_proc  \\\n",
       "0  umfangreich kenntnis mind. cad-system proe wf4...   \n",
       "1                                        anschreiben   \n",
       "2                         kenntnis office-anwendunge   \n",
       "\n",
       "                                           cats label  \n",
       "0  {'tech': True, 'soft': False, 'none': False}  tech  \n",
       "1  {'tech': False, 'soft': False, 'none': True}  none  \n",
       "2  {'tech': True, 'soft': False, 'none': False}  tech  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = get_split(split=\"test\")\n",
    "test_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a33feb0b",
   "metadata": {
    "id": "a33feb0b"
   },
   "source": [
    "## Build the Binary Docs for Training\n",
    "\n",
    "For this particular model, I have used the raw text, as I want to use it as one of the baselines for training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be77f248",
   "metadata": {
    "id": "be77f248"
   },
   "outputs": [],
   "source": [
    "# Let's process training data\n",
    "\n",
    "def data_to_bindoc(df, split):\n",
    "    docs = []\n",
    "    for idx, doc in enumerate(nlp.pipe(df['text'], batch_size=100)):\n",
    "        doc.cats = eval(df.iloc[idx].cats)\n",
    "        docs.append(doc)\n",
    "            \n",
    "    doc_bin = DocBin(docs=docs)\n",
    "    data_path = f\"./data/{split}_raw.spacy\"\n",
    "    doc_bin.to_disk(data_path)\n",
    "    print(f\"Proccesed and saved BinDoc for split:{split} to:{data_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e77112e7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e77112e7",
    "outputId": "bdadaaa2-d29f-4e4d-b287-eaf06e7106a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proccesed and saved BinDoc for split:train to:./data/train_raw.spacy\n"
     ]
    }
   ],
   "source": [
    "data_to_bindoc(train_df, \"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "349b15ba",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "349b15ba",
    "outputId": "0be72d68-96d5-49e8-8244-9d3575b2ab46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proccesed and saved BinDoc for split:dev to:./data/dev_raw.spacy\n"
     ]
    }
   ],
   "source": [
    "data_to_bindoc(dev_df, \"dev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bPJql5Ljsfse",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bPJql5Ljsfse",
    "outputId": "356d526f-ac24-47c9-f0c0-5e5b4497e0d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proccesed and saved BinDoc for split:test to:./data/test_raw.spacy\n"
     ]
    }
   ],
   "source": [
    "data_to_bindoc(test_df, \"test\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b89bf7f7",
   "metadata": {
    "id": "b89bf7f7"
   },
   "source": [
    "## Process the config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "892eadee",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "892eadee",
    "outputId": "e4fcff0a-7b3c-42ca-cc33-49037f5e6db3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
      "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
      "config/config.cfg\n",
      "You can now add your data and train your pipeline:\n",
      "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy init fill-config ./config/base_config.cfg ./config/config.cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Uq15lq0CoxDd",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Uq15lq0CoxDd",
    "outputId": "515f6794-83a9-4040-d83f-da49ff33dd1a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\n",
      "============================ Data file validation ============================\u001b[0m\n",
      "tcmalloc: large alloc 1200005120 bytes == 0x5de8c000 @  0x7f2246f8a2a4 0x7f22260aae09 0x7f22260a9cdf 0x7f22260a6675 0x7f22260a6e2e 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x7f20a9fb5d78 0x7f20a9fb870e 0x7f20a9fc05a7 0x7f20a9fc3e85 0x5d8d8c 0x55dc1e 0x5d8868 0x4990ca 0x7f20a9fb5d78 0x7f20a9fb870e 0x7f20a9fbfe4b 0x5d80be 0x5d8d8c 0x55ea20 0x55d078 0x5d8941 0x49abe4 0x55d078 0x5d8941 0x4997a2 0x55d078\n",
      "tcmalloc: large alloc 1213743104 bytes == 0xc932c000 @  0x7f2246f8a2a4 0x7f22260a83a2 0x7f22260a9cdf 0x7f22260a6675 0x7f22260a6e2e 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x5d8868 0x4997c7 0x7f20a9fb5d78 0x7f20a9fb870e 0x7f20a9fbfe4b 0x5d80be 0x5d8d8c 0x55ea20 0x55d078 0x5d8941 0x49abe4 0x55d078 0x5d8941 0x4997a2 0x55d078 0x5d8941 0x49abe4 0x55cd91 0x5d8941 0x5d8416 0x55f797\n",
      "\u001b[38;5;2m✔ Pipeline can be initialized with data\u001b[0m\n",
      "\u001b[38;5;2m✔ Corpus is loadable\u001b[0m\n",
      "\u001b[1m\n",
      "=============================== Training stats ===============================\u001b[0m\n",
      "Language: de\n",
      "Training pipeline: tok2vec, textcat\n",
      "6975 training docs\n",
      "1532 evaluation docs\n",
      "\u001b[38;5;2m✔ No overlap between training and evaluation data\u001b[0m\n",
      "\u001b[1m\n",
      "============================== Vocab & Vectors ==============================\u001b[0m\n",
      "\u001b[38;5;4mℹ 79918 total word(s) in the data (14794 unique)\u001b[0m\n",
      "\u001b[38;5;4mℹ 500000 vectors (500000 unique keys, 300 dimensions)\u001b[0m\n",
      "\u001b[38;5;3m⚠ 4877 words in training data without vectors (6%)\u001b[0m\n",
      "\u001b[1m\n",
      "================== Text Classification (Exclusive Classes) ==================\u001b[0m\n",
      "\u001b[38;5;4mℹ Text Classification: 3 label(s)\u001b[0m\n",
      "\u001b[1m\n",
      "================================== Summary ==================================\u001b[0m\n",
      "\u001b[38;5;2m✔ 3 checks passed\u001b[0m\n",
      "\u001b[38;5;3m⚠ 1 warning\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! python -m spacy debug data ./config/config.cfg"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c01674d9",
   "metadata": {
    "id": "c01674d9"
   },
   "source": [
    "## Perform Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b733aa41",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b733aa41",
    "outputId": "9a5400fd-50a4-4a74-90e5-770c09ed5f8f",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Saving to output directory: textcat_simple\u001b[0m\n",
      "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "[2022-12-22 19:41:32,497] [INFO] Set up nlp object from config\n",
      "INFO:spacy:Set up nlp object from config\n",
      "[2022-12-22 19:41:32,508] [INFO] Pipeline: ['tok2vec', 'textcat']\n",
      "INFO:spacy:Pipeline: ['tok2vec', 'textcat']\n",
      "[2022-12-22 19:41:32,512] [INFO] Created vocabulary\n",
      "INFO:spacy:Created vocabulary\n",
      "[2022-12-22 19:41:35,092] [INFO] Added vectors: de_core_news_lg\n",
      "INFO:spacy:Added vectors: de_core_news_lg\n",
      "tcmalloc: large alloc 1200005120 bytes == 0xa348a000 @  0x7ffb3abdd2a4 0x7ffb19d0ae09 0x7ffb19d09cdf 0x7ffb19d06675 0x7ffb19d06e2e 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x7ff99dc39d78 0x7ff99dc3c70e 0x7ff99dc445a7 0x7ff99dc47e85 0x5d8d8c 0x55dc1e 0x5d8868 0x4990ca 0x7ff99dc39d78 0x7ff99dc3c70e 0x7ff99dc43e4b 0x5d80be 0x5d8d8c 0x55ea20 0x55d078 0x5d8941 0x49abe4 0x4fe253 0x49abe4 0x55d078 0x5d8941\n",
      "tcmalloc: large alloc 1213743104 bytes == 0x7f854000 @  0x7ffb3abdd2a4 0x7ffb19d083a2 0x7ffb19d09cdf 0x7ffb19d06675 0x7ffb19d06e2e 0x4f750a 0x4997a2 0x55cd91 0x5d8941 0x49abe4 0x5d8868 0x4997c7 0x7ff99dc39d78 0x7ff99dc3c70e 0x7ff99dc43e4b 0x5d80be 0x5d8d8c 0x55ea20 0x55d078 0x5d8941 0x49abe4 0x4fe253 0x49abe4 0x55d078 0x5d8941 0x49abe4 0x55d078 0x5d8941 0x49abe4 0x55cd91 0x5d8941\n",
      "[2022-12-22 19:41:36,915] [INFO] Finished initializing nlp object\n",
      "INFO:spacy:Finished initializing nlp object\n",
      "[2022-12-22 19:41:42,802] [INFO] Initialized pipeline components: ['tok2vec', 'textcat']\n",
      "INFO:spacy:Initialized pipeline components: ['tok2vec', 'textcat']\n",
      "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "\u001b[38;5;4mℹ Pipeline: ['tok2vec', 'textcat']\u001b[0m\n",
      "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
      "E    #       LOSS TOK2VEC  LOSS TEXTCAT  CATS_SCORE  SCORE \n",
      "---  ------  ------------  ------------  ----------  ------\n",
      "  0       0          0.00          0.22        0.00    0.00\n",
      "  0     200         61.35         22.51       84.14    0.84\n",
      "  0     400        152.96         15.80       84.31    0.84\n",
      "  0     600        370.76         12.59       86.66    0.87\n",
      "  1     800        873.67         10.81       88.59    0.89\n",
      "  2    1000       1599.24          9.19       87.80    0.88\n",
      "  2    1200       2158.54          8.02       90.14    0.90\n",
      "  3    1400       4959.64          6.81       89.50    0.89\n",
      "  4    1600      10044.64          5.00       91.49    0.91\n",
      "  5    1800       9352.23          4.28       91.18    0.91\n",
      "  7    2000      15639.98          3.59       91.74    0.92\n",
      "  9    2200      17685.52          2.78       92.16    0.92\n",
      " 11    2400      26260.64          2.29       91.82    0.92\n",
      " 14    2600      36429.64          2.05       92.06    0.92\n",
      " 16    2800      31077.49          1.59       91.73    0.92\n",
      " 19    3000      43704.31          1.41       90.27    0.90\n",
      " 21    3200      57063.44          1.31       91.86    0.92\n",
      " 24    3400      82256.27          1.09       92.25    0.92\n",
      " 26    3600      77320.08          1.21       91.78    0.92\n",
      " 29    3800     136012.14          1.11       92.45    0.92\n",
      " 31    4000      73921.58          0.99       91.68    0.92\n",
      " 34    4200      91140.72          0.92       92.12    0.92\n",
      " 36    4400     197795.92          0.89       92.32    0.92\n",
      " 38    4600      80790.43          0.84       93.20    0.93\n",
      " 41    4800     145474.82          0.77       92.76    0.93\n",
      " 43    5000     110442.40          0.61       92.13    0.92\n",
      " 46    5200     120800.75          0.64       92.79    0.93\n",
      " 48    5400     121901.91          0.79       92.25    0.92\n",
      " 51    5600     129140.85          0.58       93.39    0.93\n",
      " 53    5800     268325.01          0.88       93.15    0.93\n",
      " 56    6000     131026.54          0.60       92.87    0.93\n",
      " 58    6200      98660.48          0.50       93.05    0.93\n",
      " 61    6400     266114.94          0.62       91.92    0.92\n",
      " 63    6600     151121.55          0.48       93.00    0.93\n",
      " 65    6800     257992.59          0.52       92.79    0.93\n",
      " 68    7000     226536.37          0.43       92.59    0.93\n",
      " 70    7200     219752.45          0.49       93.04    0.93\n",
      "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
      "textcat_simple/model-last\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy train ./config/config.cfg --gpu-id 0 --output textcat_simple"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4721535",
   "metadata": {
    "id": "d4721535"
   },
   "source": [
    "## Evaluate\n",
    "\n",
    "\n",
    "Let's load the best model and try to evaluate on a single example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f72f87",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "00f72f87",
    "outputId": "312cbeed-1282-4081-abe0-60fd629f3af6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluate.json  model-best  model-last\n"
     ]
    }
   ],
   "source": [
    "!ls textcat_simple/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "255c5579",
   "metadata": {
    "id": "255c5579"
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"textcat_simple/model-best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8315b22",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "c8315b22",
    "outputId": "ac417f84-f617-4606-a70f-09b40fe16829"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tech': 0.9993124008178711,\n",
       " 'soft': 0.0005922773852944374,\n",
       " 'none': 9.535031131235883e-05}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc = nlp(\"Kenntnisse im Bereich Online-Dienste von Vorteil\")\n",
    "doc.cats"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "Pfmi5g7wzw3Y",
   "metadata": {
    "id": "Pfmi5g7wzw3Y"
   },
   "source": [
    "### Evaluate using spacy evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e6f8bb",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "01e6f8bb",
    "outputId": "1888e2b6-0fe6-4003-dfad-4b1c96c84211"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
      "\u001b[38;5;4mℹ To switch to GPU 0, use the option: --gpu-id 0\u001b[0m\n",
      "\u001b[1m\n",
      "================================== Results ==================================\u001b[0m\n",
      "\n",
      "TOK                 100.00\n",
      "TEXTCAT (macro F)   93.65 \n",
      "SPEED               2964  \n",
      "\n",
      "\u001b[1m\n",
      "=========================== Textcat F (per label) ===========================\u001b[0m\n",
      "\n",
      "           P       R       F\n",
      "tech   95.36   87.27   91.14\n",
      "soft   92.93   95.18   94.04\n",
      "none   94.74   96.84   95.78\n",
      "\n",
      "\u001b[1m\n",
      "======================== Textcat ROC AUC (per label) ========================\u001b[0m\n",
      "\n",
      "       ROC AUC\n",
      "tech      0.98\n",
      "soft      0.99\n",
      "none      0.99\n",
      "\n",
      "\u001b[38;5;2m✔ Saved results to textcat_simple/evaluate.json\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy evaluate textcat_simple/model-best/ ./data/test_raw.spacy --output textcat_simple/evaluate.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lgeJ8A_m8yIe",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lgeJ8A_m8yIe",
    "outputId": "d280d8fe-b13a-4441-d651-5c7b963ec9b6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "textcat_simple/model-best/config.cfg\n",
      "textcat_simple/model-best/meta.json\n",
      "textcat_simple/model-best/textcat/\n",
      "textcat_simple/model-best/textcat/cfg\n",
      "textcat_simple/model-best/textcat/model\n",
      "textcat_simple/model-best/tok2vec/\n",
      "textcat_simple/model-best/tok2vec/cfg\n",
      "textcat_simple/model-best/tok2vec/model\n",
      "textcat_simple/model-best/tokenizer\n",
      "textcat_simple/model-best/vocab/\n",
      "textcat_simple/model-best/vocab/key2row\n",
      "textcat_simple/model-best/vocab/strings.json\n",
      "textcat_simple/model-best/vocab/lookups.bin\n",
      "textcat_simple/model-best/vocab/vectors\n",
      "textcat_simple/model-best/vocab/vectors.cfg\n",
      "textcat_simple/evaluate.json\n"
     ]
    }
   ],
   "source": [
    "# Save best model\n",
    "!tar -czvf best_model_raw.tar.gz textcat_simple/model-best/* textcat_simple/evaluate.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "lAuHgZ4bsXEF",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "lAuHgZ4bsXEF",
    "outputId": "8ec495ee-4768-4a30-9fc3-758d2bf60678"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive',force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "UJ0Kp30AsYF_",
   "metadata": {
    "id": "UJ0Kp30AsYF_"
   },
   "outputs": [],
   "source": [
    "save_to = \"/content/drive/MyDrive/tb_challenge/\"\n",
    "!cp best_model_raw.tar.gz $save_to"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "OtDJj0mEDc46",
   "metadata": {
    "id": "OtDJj0mEDc46"
   },
   "source": [
    "## Obtain Predictions from Text Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8s8uICmHDcCK",
   "metadata": {
    "id": "8s8uICmHDcCK"
   },
   "outputs": [],
   "source": [
    "best_model = \"./textcat_simple/model-best/\"\n",
    "test_data = \"./data/test_raw.spacy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uMUN9M64Ev77",
   "metadata": {
    "id": "uMUN9M64Ev77"
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(best_model)\n",
    "docs = DocBin().from_disk(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "vzN_yJ6IFOkV",
   "metadata": {
    "id": "vzN_yJ6IFOkV"
   },
   "outputs": [],
   "source": [
    "def get_key(d, value):\n",
    "  return next((k for k, v in d.items() if v == value), None) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "EBp169KTKs7H",
   "metadata": {
    "id": "EBp169KTKs7H"
   },
   "outputs": [],
   "source": [
    "idx_to_label = {0: \"tech\", 1: \"soft\", 2: \"none\"}\n",
    "label_to_idx = {\"tech\": 0, \"soft\": 1, \"none\": 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fnMByGIDtMn",
   "metadata": {
    "id": "9fnMByGIDtMn"
   },
   "outputs": [],
   "source": [
    "def get_preds_and_gold(nlp, docs):\n",
    "  \"\"\"\n",
    "  Created this function to be able to create the confusion matrix\n",
    "  \"\"\"\n",
    "  preds, gold = [], []\n",
    "  for doc in docs.get_docs(nlp.vocab):\n",
    "    true_label = get_key(doc.cats, True)\n",
    "    doc = nlp(doc.text )\n",
    "    pred_label = max(doc.cats, key=doc.cats.get)\n",
    "    preds.append(label_to_idx[pred_label])\n",
    "    gold.append(label_to_idx[true_label])\n",
    "  return preds, gold "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "TfTI03GEHROB",
   "metadata": {
    "id": "TfTI03GEHROB"
   },
   "outputs": [],
   "source": [
    "preds, gold = get_preds_and_gold(nlp, docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aGhJve6_LDvN",
   "metadata": {
    "id": "aGhJve6_LDvN"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uaskB1lRKXbn",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uaskB1lRKXbn",
    "outputId": "ee100286-b334-4daf-ee48-238ac2b9905e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1502,), (1502,))"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_arr, pred_arr = np.array(gold), np.array(preds)\n",
    "gold_arr.shape, pred_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cprnz_drLMlo",
   "metadata": {
    "id": "cprnz_drLMlo"
   },
   "outputs": [],
   "source": [
    "def plot_matrix(y_preds, gold, labels): \n",
    "  c_matrix = confusion_matrix(y_true=gold, y_pred=y_preds, normalize=\"true\")\n",
    "  fig, ax = plt.subplots(figsize=(8,8))\n",
    "  display_matrix = ConfusionMatrixDisplay(confusion_matrix=c_matrix,\n",
    "                                          display_labels=labels)\n",
    "  display_matrix.plot(cmap=\"RdPu\",values_format=\".2f\", ax=ax, colorbar=False)\n",
    "  plt.title(\"Confusion Matrix for spaCy model - Cleaned data\")\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "vJuMOSIYLQKw",
   "metadata": {
    "id": "vJuMOSIYLQKw"
   },
   "outputs": [],
   "source": [
    "classes = [\"soft\",\"tech\",\"none\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "chdJI3tsLSGu",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 513
    },
    "id": "chdJI3tsLSGu",
    "outputId": "031673de-7c28-4e97-cef8-a0eb44e1fec9"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAHwCAYAAACPCeeDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wV5b3H8c+PXRBQOkoABezYsaJG0Wjs8aqJJXZjizWJ3hhNbjSJmnbVqLmxRE2siYnEEkts0Ygh0Sii2IjYFbABAiqKsPvcP2YWz667sBh2Dw983q/XeXFm5pmZ3wyz8512zomUEpIkKR8dql2AJElaOIa3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbi0REdImI2yJiRkSM/A+mc2BE3LMoa6uGiLgzIg79jOOeHRFTIuLNRV3X0igiXomIL7ai3ZCISBFR20Z1bBsRE9ti2v+JiHggIo5sZdvFchmWRob3UiYiDoiIMRHxfkS8UYbMVotg0nsD/YA+KaV9PutEUkq/SyntuAjqaaTc6aSIuLlJ/w3K/g+0cjo/jIjrFtQupbRLSunqz1DnIOC/gbVTSp9b2PHbU0SsEREjywONGRHxZEScHBE11a6tWiJis4j4S0RMj4hpEfFIRHyt2nVVQ0QcFhGjq13HksrwXopExMnABcBPKIJ2EHAxsMcimPxgYEJKae4imFZbeQfYIiL6VPQ7FJiwqGYQhf/k72oQMDWl9PZnmHebnDG2MK9VgX8BrwPrpZR6APsAmwDd2quOxUlEbAHcD4wCVgP6AMcCu1SzLi2hUkq+loIX0AN4H9hnPm2WoQj3yeXrAmCZcti2wESKs8K3gTeAr5XDfgR8DMwp53EE8EPguoppDwESUFt2Hwa8BLwHvAwcWNF/dMV4WwKPAjPKf7esGPYAcBbwj3I69wB9W1i2hvovBY4v+9UAk4AzgAcq2l5IEUozgceArcv+OzdZznEVdfy4rONDih33A8CR5fBLgBsrpv9z4D4gmtT4xXL8+nL6V5X9/wt4BpheTnetinFeAU4FngRmN6zfiuEBnF/+n80EngLWLYddVa6Pe8v1NwoYvKD1UA67DrhjPtvSHcCJTfo9CezVTNuGbeNr5fzeBY4BNi3HmQ78qqJ9B+D7wKvlcl0D9KgYfnA5bCrwP+U6+mLFuKcBL5bDbwB6N7eNfoa/sdHARfMZvi0wsaJ7AHAjxUHly8A3KoZtBjxULvsbwK+AThXDU7mOni/bXFS5PQGHA+PLdXl3k//XHYB/U/xN/ar8fz+yhZq7lNvJu8CzwClNlqFhXb5XDt+r7L8W8BFQR7EtTy/77wY8Xm5TrwM/bK994JL2qnoBvtrpP7oInrnz2zEBZwIPAysAywP/BM4qh21bjn8m0BHYFZgF9CqH/5DGYd20e96OEVi2/ONdsxzWH1infH8YZXgDvcudxsHlePuX3X3K4Q+UO441yp3MA8DPWli2bSnCe0vgX2W/Xcsd25E0Du+DKM6aaikOVt4EOje3XBV1vAasU47Tkcbh3ZXi7P4wYGtgCrDi/Oqs6F4D+IBih9sR+A7wAuWOnCKYngBWAro0M72dKIK3J0WQrwX0L4ddRbHTHUFx4HYhjQ+c5rce3qQ8eGthOfZtWM9l9wYUYdmpmbYN28alQGdgR4od/y0U2+JAipDepmx/eLkOVgGWA24Cri2HrU0RFg3L9AuK7bYhvL9JsY2vWA7/NXB90230M/x9daUIqi/Mp828/1uKg4jHKA4cO5XL8hKwUzl8Y2Dzct0PoQjib1VMKwG3l/+vgygOAHYuh+1Rrp+1yvG/D/yzHNa3/D/fu9yeTirXT0vh/TPg7xR/iysBT9N4+9yH4iCkA7AfxbbasH0dVrk9VayD9cr26wNvAXtWc9+Y66vqBfhqp/9oOBB4cwFtXgR2rejeCXilfL8txVlhbcXwt4HNy/c/ZOHCezrwFZoEDo3D+2DgkSbDHwIOK98/AHy/YthxwF0tLFvljvN5YE3gD+V6aRTezYz7LrBBc8tVUceZzfQ7sqJ7ODCN4oxw//nMa16dZffpwA0V3R0orhZsW3a/Ahw+n+ltR3HgsDnQocmwq4A/VHQvRxFAK7ViPcyhDIsW2nYu269edp8LXNxC24ZtY2BFv6nAfhXdN1KGF8VVi+Mqhq1Z1lNLEYaVy7QsxdWShvAeD2xfMbx/xbjzttHP8Pc1sBx3aGv+b8vt4bUmw78LXNnCuN8Cbq7oTsBWFd03AKeV7+8EjmiyzcyiuLV1CPBwxbCgOKhtKbxfqvx/Bo6u3D6baf8EsEf5/jCahHcz7S8Azl/Y9e0rec97KTIV6LuA+6IDKMKlwatlv3nTSI3vac+i2OEvlJTSBxRH6ccAb0TEHRExtBX1NNQ0sKK78ons1tZzLXAC8AXg5qYDI+LbETG+fAhrOsUth74LmObr8xuYUvoXxY4wKHa0rdVoHaSU6st5Va6DFuedUrqf4tLoRcDbEXFZRHRvbtyU0vsUBxgDYIHrYSpF8LU034+APwIHlc8A7E+x3ufnrYr3HzbT3fB/29x2WkvxHMeAJsv0QVlrg8HAzeUDZdMpwryuHLdF5acg3i9fdzbT5F2K2x0trpMmBgMDGuooa/leQx3lw4C3R8SbETGT4jmVpttgS9v+YODCiulOo9juBvLp9ZOY/7Y7oMnwRn+PEXFIRDxRMa91m6mzsv3wiPhbRLwTETMo9gEL+ttSMwzvpcdDFPdE95xPm8kUf/gNBpX9PosPKC4lNmj05HRK6e6U0g4UO7t/A5e3op6GmiZ9xpoaXEtxlv6XlNKsygERsTXFpel9KW4J9KS4NxgNpbcwzZb6N0z3eIrLtJPL6bdWo3UQEUFx+bJyHcx33imlX6aUNqa4pLwGxX3LBitVTHs5isujk1uxHv5KceVkfq6muLKxPTArpfTQAtq3VnPb6VyKsH+DxsvUleLSf4PXgV1SSj0rXp1TSvPdplLxKYjlytenHkArt6OHWPA6qazj5SZ1dEsp7VoOv4Ti72L1lFJ3imCPlibWzLS/3mTaXVJK/+TT6ycqu5vxRpPhgyrGHUzxd3sCxa2snhSX1ef3t/J74FaKqzs9KG6VtHa5VMHwXkqklGZQXFK8KCL2jIiuEdExInaJiP8tm10PfD8ilo+IvmX7BX4sqgVPACMiYlBE9KC4JAhARPSLiD0iYlmKA4r3Kc5amvoLsEb58bbaiNiPIoBu/4w1AZBSehnYhuJhpqa6UQTBO0BtRJwBVJ6pvgUMWZgnyiNiDeBsinvIBwPfiYhhrRz9BmC3iNg+IjpS3HueTfE8QmvmvWl5ttOR4oDqIxqv610jYquI6ETx8N/DKaXXWfB6+AGwZUScExGfK+e1WkRcFxE9AcqwrgfOY8Fn3QvjeuCkiFi5POD4CfDH8qrQn4AvVSzTmTTez10K/LgMHsptfVF82gKKg53DIuKUhk80lB9F/EMzbR8B3ouIU6P4joSaiFg3IjYth3ejeC7k/fKq1LELUcelwHcjYp2yhh4R0fDxzTuAdSLiy+VVuG/Q5MC6iRvKafWKiBWBEyuGLUsR0O+U8/kaxZl3g7eAFcv/hwbdgGkppY8iYjPggIVYLlUwvJciKaXzgJMpHmB5h+II/QSKB4OgCJgxFE/4PgWMLft9lnndS3HZ9EmKB3MqA7dDWcdkikt629DMzimlNBX4EkVgTaXYOX4ppTTls9TUZNqjU0rNXVW4G7iL4j7xqxRhV3nZsOELaKZGxNgFzafcQV4H/DylNC6l9DzFWdS1EbFMK+p8jiL0/4/iQbfdgd1TSh8vaNxSd4qzo3f55AnscyqG/54iiKdRPCR1UNl/vushpfQisAXFfeJnykugN1JsP+9VTP8aigeUPutBYHN+S3Ew8CDFU9ofUYZKSukZ4Phyud6gWO7KLxW5kOLM756IeI/i4bXhi6Ko8sx2u/L1UkRMAy6jOAht2raOYtseVi7DFOAKilsTAN+mCLb3KP7//rgQddxM8YmGP5SX3J+m/Lha+bezD8WDaFOB1Sk+JdGSH1H8/79M8WmOeQdhKaVnKQ7MHqII6vWaTOt+ik9JvBkRDX+zxwFnluv+DBbuFpIqRHHLQ9LSJiKuonj46PttOI9DgKNTSovii4AklTzzltQmyvvNx1GcfUpahAxvSYtcROxEcWvmLYpL2JIWIS+bS5KUGc+8JUnKjOEtSVJm2u1XiNpTl+iSus/7xIW08FbaYL5fuCUtWPjdI/rPPPbEuCkppeWbG7ZEhnd3erA/B1e7DGXsF/eeXO0SlLno2LHaJShzHXot3/TroT8Z1p6FSJKk/5zhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmaqtdgBatoTusypfP3Zmo6cDDV43lvnP/0Wh4z5W6c+Dle9KlR2c61HTgttP/yvi7X2Djr67Hdt/acl67/uv147wtfs2kJ99q70VQlT12/4tc9j93U1+X2PGgYezzjc83Gj5n9lx+ccKtvDDuDbr17sKpl32ZfoN6MndOHb886XZefOpN6ubWs92+67PvNz/fwly0JBvz1+e57Lt3FtvQwRux70lbNxo+Z/Zczjv2Jl54otiGTvvtPvQb1Iu3XnuXY4b/ioGr9QVg6CYrcsL5u1djERZ7i3V4R8TWwKXAHOBYoFdK6S/VrWrxFR2CvS/YlUt2u5bpk2Zy8uijePr253jr31Pmtdnx1BE8ceOz/OPyMfQb2pev33IgZw69kMf+8BSP/eEpAPqvswJH3LCfwb0Uqqur55JT7+TskQfSZ0B3TtrxNwzfaQ0Grbn8vDb3/O4Jlu3RmcsfOZ5RNz/DVWfdz6mXf5nRt45nzsd1XDTq63w0aw7HbX0p2+y1Dv0G9aziEqm91dXVc8kpd3D2zYfQd0B3TtruMjbfZU0GDV1hXpu7rx3Lcj26cMXYbzLqxqe48of3ctpv9wWg/5De/Orvx1ar/Gws7pfNDwR+mlIaBqwJ7FrlehZrgzcdyJQXpzH1lenUzann8ZHPsN6XhjZulKBz92UA6NKjMzPeeO9T09lo33UZO/KZ9ihZi5kJYyfTf+XefG5ILzp2qmHEXuvw8F0TGrV5+K4JbL/f+gBstftajPv7y6SUiICPZs2hbm49H380h9qONXTttkw1FkNVNOGxSQxYpTf9h/SmY6daRnx5XR7+y78btfnXnf9m+/2HAbDVHmszblSxDan12v3MOyKWBW4AVgRqgLOAKcC5ZT2PUpxlHwzsC+wUEbsBnwe6RMRWFIH+x/aufXHXY0A33p04c1739EkzGbzZwEZt7vrxAxxz20FsfexmdOrakYt3u/ZT09lw73W4Yp8/tHm9WvxMffM9lh/YfV533/7deG7s5Bbb1NR2oGu3ZZg57UM+v/taPHzXBA5e7wJmfziHo87cgW69urRr/aq+qW/MpO/AHvO6+w7owXOPTWzcZnLlNlRD1+7LMHPaLADefO1dThxxCV27LcPB/7M96245uP2Kz0g1LpvvDExOKe0GEBE9gKeB7VNKEyLiGuDYlNIFZVDfnlL6U0QcBmySUjqhuYlGxNHA0QDd6NYey5GljfZdl0euG8cDFz7EkOErctBv9uLnG19Mw0Hv4E0H8vGsObz57DvVLVTZmTB2Mh06BNc8+U3en/4Rp/7X1QwbsTKfG9Kr2qUpE737deOqp06me++uPP/EZM4+8Houeeh4unbvXO3SFjvVuGz+FLBDRPy8vKc9BHg5pdRwbe5qYMTCTjSldFlKaZOU0iZd6Lroqs3IjMnv0WvFT86aeg7szoxJjS+LDz90Q564sbgk/sq/JlLbuZZl+36yvjbcZ13G3vB0+xSsxU6fz3XjnUmfXL2Z8sZ79OnfrcU2dXPrmfXebLr37sKom55m4+1WpbZjDT2XX5a1NluJ58e90a71q/r69O/OlEkz5nVPmTzj09vQgMptqI5ZM2fTvXdXOi5TS/fexf5o9WED6L9ybya9OLX9is9Iu4d3GdIbUYT42cCe7V3Dkuq1MZPou1ofeg/uSU3HDmy4zzo8fcdzjdpMf30Ga2y7MgD91uxLx861vP9OcbkqAoZ9ZW0eH2l4L63W2HAAk1+axpuvvsucj+t48OZnGL7TGo3aDN9pDe7745MAjL5tPOtvNYSIYPmBPXhy9CsAfPTBxzz32CRWXK1Pey+CqmyNjQYw6cWGbWguD970NMN3afzszfCd1+S+658AYPSfn2X9ESsTEcyY8gF1dfUAvPHKNCa/NNUrNy2oxj3vAcC0lNJ1ETEdOAEYEhGrpZReoLjXPaqZUd8Dr4fPT31d4saT/sIxtx1Eh5rgX1c/wZvj32GX07fltbGTeeaOCdxy2j3sd/HubHPi5pDg90fdMm/8VbcazPSJM5n6yvQqLoWqqaa2A8f8bGfO2O966uvq2eGAYQweujzX/ewBVh82gOE7r8GOBw7jvOP/zFGbXcRyvbpw6q/3AmC3wzfhgm/exnFbX0pK8MWvbsDK6/Sr8hKpvdXU1nDs/+7K6V+5ttiGDtyQwWutwLU/uZ/Vhw1g812HsuPBG3HuMTdx5EYX0q1XF77zm70BePqfr3LdT++npraGDh2C48/bnW69ls4rqQsS7f2EX0TsBJwD1PPJR8B60OSBtZTS7Ii4ik/uefcG7gY6soAH1vrF59L+HNy2C6Il2i/ePrnaJShz0bFjtUtQ5jr0Wv6xlNImzQ1r9zPvlNLdFCHc1IbNtD2s4v00YNO2q0ySpDws7p/zliRJTRjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTO11S6gLay0YX/OH316tctQxvZa9nvVLkGZu3nmWdUuQUswz7wlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScpMbUsDIuL/gNTS8JTSN9qkIkmSNF8thjcwpt2qkCRJrdZieKeUrq7sjoiuKaVZbV+SJEmanwXe846ILSLiWeDfZfcGEXFxm1cmSZKa1ZoH1i4AdgKmAqSUxgEj2rIoSZLUslY9bZ5Ser1Jr7o2qEWSJLXC/B5Ya/B6RGwJpIjoCHwTGN+2ZUmSpJa05sz7GOB4YCAwGRhWdkuSpCpY4Jl3SmkKcGA71CJJklqhNU+brxIRt0XEOxHxdkT8OSJWaY/iJEnSp7XmsvnvgRuA/sAAYCRwfVsWJUmSWtaa8O6aUro2pTS3fF0HdG7rwiRJUvPm993mvcu3d0bEacAfKL7rfD/gL+1QmyRJasb8Hlh7jCKso+z+esWwBHy3rYqSJEktm993m6/cnoVIkqTWac2XtBAR6wJrU3GvO6V0TVsVJUmSWrbA8I6IHwDbUoT3X4BdgNGA4S1JUhW05mnzvYHtgTdTSl8DNgB6tGlVkiSpRa25bP5hSqk+IuZGRHfgbWClNq5Ln9GYe57jsu/cTn1dPTseuin7fnvbRsPnzJ7LeUfdwAuPT6Jb766cds0B9Bvci8fve54rz7iLuR/XUduphiN+vCsbbLtqdRZCVbXhDmtw5Lm706EmuPeqR7np3FGNhi8/qCcnXro33fsuy/vvfsj5h/+BqZNmAnDj+z/htaffBOCd16fzk328QLc0GnPvBC479S/lfmhj9j15m0bD58yey3lf/xMvPD652A9dtR/9BvfiuTET+b9v3lI0SnDAd7djy93XrsISLP5aE95jIqIncDnFE+jvAw+1dgbluAeklBb6N8Aj4irg9pTSnxZ23KVRXV09l5x8K2ffdgR9B3bnpK0vYvPd1mLQWv3mtbn76kdZrmcXrnjqFEaNHMeVp9/JadccQPc+y/KDPx1Kn/7deeWZNzljjyu55gU/ULC06dAh+PoFe/CD3X7D1EkzOGf0CTxy+3gm/vvteW0O++mu/O13Y/nb78ay3jarcvCZO3PBETcA8PGHczhp819Wq3wtBurq6rnkv2/j7D9/rdgPbXspm++6FoOGrjCvzd3XPFbsh8adzKg/PcmVP7ib0676KoPXXoELRx1LTW0N0958jxO2/BXDd1mTmtqaKi7R4mmBl81TSsellKanlC4FdgAOLS+ft1ZP4LjPWqBab8KY1xmwSh/6r9ybjp1qGbH3Bjx8e+MfgPvX7ePZ/sCNANhqr3UZ98CLpJRYddgA+vTvDsDgtfsx+6M5zJk9t92XQdW1+qYr8caLU3nrlWnMnVPH6JHjGP6lxmc+Kw3tx1OjXgTgqVEvstmXPDPSJyaMmdh4P/SV9Xj4jib7oTvGs/3+GwKw1Z7rMO6Bl0gp0blrp3lB/fFHc4j41ORVajG8I2Kjpi+gN1Bbvm+tnwGrRsQTEXFORJwSEY9GxJMR8aOK+R1S9hsXEddWjD8iIv4ZES9FxN4LvYRLkamTZ9J3xU8eR+g7sDtT35jxqTbLr9gTgJraGrp278zMqbMatfnHLU+z6gYD6LhMqz6MoCVI7wHdmTLxk21m6qQZ9B7YvVGbV556g833WBeAzfdYh67dO9Otd1cAOnWu5dzRJ/DzUccx3MudS6WpbzTZDw3oztTJMz/VZvmyTbEfWoaZ04r90L8ffZ1jN/slx2/xK46/YA/Pulswv73zefMZloDtWjmP04B1U0rDImJHigfgNqP48pdbI2IEMBX4PrBlSmlKxbe7QfGd6lsBQ4FbgWYvoUfE0cDRAINW8pb8Z/Xqs29x5el3cfath1e7FC2mrvzuHRx9/h5sd9DGPPOPl5kyaQb1dfUAHLXmz5k2eSb9hvTmrLuO4tWn3+TNl6dVuWLlZOimK3HJI9/gtefe5vyv38gmO6xOp84dq13WYmd+X9LyhTaY347l6/GyezlgdYon2EeWPz9KSqnyr/2WlFI98GxE9KMFKaXLgMsANtlow9QGtS/2+jQ5a5oyaSZ9+vf4VJt3Jk6n78Ae1M2tY9bMj+jep2vZfgZn738t/335PvRfpU+71q7Fw7QmV2/6DOzBtEmNz5refeM9fv7V6wDovGwntthzXT6Y8dG88QHeemUaTz/4EisPG2B4L2X69G+yH5o8kz4Dun+qzTsTZ1Tsh2bTvbx602DQmivQeblOvPrs26y+0cB2qT0nrfmo2KIUwE9TSsPK12oppd8sYJzZTcZXC9bYeEUmvTiFN1+ZxpyP5/Lgn8YxfLe1GrUZvtta3Pe7sQCMvvlp1t9mVSKC96d/yA+/fBWHnbkza28xpArVa3Hw/JiJ9F+tDysM7kVtxxq22mcDHrnj2UZtuvXpSpQ3I79yyrbcd/UYAJbt2YXaTjXz2gzdYjCvj38bLV3W2Hggk16a+sl+6ManGL7r0EZthu86lPuuL87hRt/yDOtvswoRwZuvTKNubh0Ab7/2LhMnTGGFwT3bfRly0B43Nd8DupXv7wbOiojfpZTej4iBwBzgfuDmiPhFSmlqRPRucvatVqipreHY8/6L0/f4LfV1iR0O2YTBa/fj2rPuZfWNBrL5bmuz46GbcO6RN3DkeufQrVdXvnP1/gDc/uuHmPzSVK7/6f1c/9P7ATj71sPpucJy1VwktbP6unouP+lWfnDb4dTUdOCvV4/h9fFvs//pO/DC2Ik8esd41h2xCgefuTMpJZ4d/Qq//lbx0Z4Vhy7Pcf/3ZerrEx06BDed+0Cjp9S1dKipreHYc77E6XtdTX1dPTscvDGD1+rHtWf/tdgP7boWOx6yMece/SeO3OAXdOvVhe9cuR8Azz70KiPP/zs1HTvQoUNw3C92p0efZau8RIunSKntrzBHxO+B9YE7gYnAkeWg94GDUkovRsShwClAHfB4Sumwph8Vi4j3U0oLTJNNNtowPTp61IKaSS3aa9nvVbsEZe7mmWdVuwRlrkP33o+llDZpblhrvh41gAOBVVJKZ0bEIOBzKaVHWltASumAJr0ubKbN1cDVTfod1qTb00BJ0lKvNfe8Lwa2APYvu98DLmqziiRJ0ny15p738NxewLUAABJsSURBVJTSRhHxOEBK6d2I6NTGdUmSpBa05sx7TkTUUHy2m4hYHqhv06okSVKLWhPevwRuBlaIiB9T/BzoT9q0KkmS1KIFXjZPKf0uIh6j+FnQAPZMKY1fwGiSJKmNtOZp80HALOC2yn4ppdfasjBJktS81jywdgfF/e4AOgMrA88B67RhXZIkqQWtuWy+XmV3+Yti/sSnJElVstDfbZ5SGgsMb4NaJElSK7TmnvfJFZ0dgI2AyW1WkSRJmq/W3PPuVvF+LsU98BvbphxJkrQg8w3v8stZuqWUvt1O9UiSpAVo8Z53RNSmlOqAz7djPZIkaQHmd+b9CMX97Sci4lZgJPBBw8CU0k1tXJskSWpGa+55dwamAtvxyee9E2B4S5JUBfML7xXKJ82f5pPQbpDatCpJktSi+YV3DbAcjUO7geEtSVKVzC+830gpndlulUiSpFaZ3zesNXfGLUmSqmx+4b19u1UhSZJarcXwTilNa89CJElS6yz0D5NIkqTqMrwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMzUVruANhFBdPC4RJ/dze//uNolKHO7Lfe9apegJZgJJ0lSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlJnaahegRWvMPc/x62/fSn1dYqfDNmXfU77QaPic2XM594g/8sLjk+jWuyvfve4A+g3uzcypH/CTA65jwmMT+eJBG3PcBXtWaQlUbWPuncBl37md+rp6djx0U/b9720aDZ8zey7nHTWSF54otqHTrt6ffoN78fj9z3PlGXcz9+M6ajvVcMTZu7DBtqtWaSlUTRvvsAZfP/e/6FAT3H3Vo4w894FGw1cY1JNvXboPPfouy3vvzuKcw//I1EkzWH/EKhz1v7vPa7fSmsvz80N+z0O3PdvOS7D488x7CVJXV8/F37qFM/98OJc+fjKjRo7jtfFvNWpz91WPslyvLvzmme+w14lb8dv/uROATp07cvAZO3LET3erRulaTNTV1XPJybfyo5sO45Ix3+LB5rahq8ewXM8uXPHkt9nz+M9z5el3AdC9z7L8YOQhXPzINzn513tz3lEjq7EIqrIOHYLjLtiTM/b4Lcds+Au22WcDVhq6QqM2R/x0N+773WMcv9kFXP+T+/jamTsD8OSDL3Hi5hdy4uYX8t1dLmP2rDmM/evz1ViMxZ7hvQSZ8OjrDFi1D/1X7kPHTrWM2GcDHrq98RHrw7c/wxcP3BiArb68HuMeeIGUEp2X7cQ6n1+ZTp29GLM0mzBmIgNW6UP/lXsX29De6/PwHeMbtfnXHePZ/sCNANhqr3UZ98CLpJRYdYMB9OnfHYDBa/dj9kdzmDN7brsvg6prjU1XYvKLU3nzlWnMnVPHgyPHscWX1m7UZtDQfowb9SIA40a9yOZNhgNstdd6jLnnOWZ/OKdd6s5Nm4V3RAyJiPERcXlEPBMR90REl4gYFhEPR8STEXFzRPQq2z8QET+PiEciYkJEbF32r4mIcyLi0XKcr7dVzbmbOnkGfVfsOa+778AeTJ00o0mbmSy/Yg8Aampr6Nq9MzOnzmrXOrX4KrahHvO6+w7swdTJMz/VptE21OPT29A/bnmaVTcYQMdlPBhc2vQZ0IMpE6fP654yaQZ9BvZo1Oblpybz+T3WBWDLPdaha/fOdOvdtVGbbfbZgFE3PNH2BWeqrc+8VwcuSimtA0wHvgJcA5yaUlofeAr4QUX72pTSZsC3KvofAcxIKW0KbAocFRErN51RRBwdEWMiYsw770xpuyWSNF+vPvsWV55xNyf+0ucm1LwrvnsH6269Cv/30DdYb+tVmDJpBvV19fOG9/pcN4as8zkeu3dCFatcvLX1YfHLKaWGQ6fHgFWBnimlUWW/q4HKG2M3VbQdUr7fEVg/IvYuu3tQHBS8XDmjlNJlwGUAm2y8UVqEy5CN1hzx9hnQnXcmFmfodXPrmDXzI7r36dp0UlpKFdvQJ1drpkyaQZ8B3T/V5p2JM+g7sEexDc34ZBuaMmkGZx9wHf992T70X6VPu9auxUNrrgBOe+M9fvzVawHovGwnPr/nenww46N5w0d8ZX3+eesz1M2tR81r6zPv2RXv64CeLTVs0r6OTw4sAjgxpTSsfK2cUrpnEde5RFhjkxWZ/EJxr2nOx3N5cOQ4Nt9trUZthu+2Nn/93WMAjL7pKdbfZlUiohrlajG0xsYDmfTilE+2oT89yfBdm2xDuw7lvt+NBWD0zU+z/jarEBG8P/1DfviVqznsRzux9haDq1G+FgMTxkxkwGp96De4F7Udaxixzwafem6ie5+u8/Y7+57yBe65+tFGw7fZd5iXzBegvW9IzQDejYitU0p/Bw4GRi1gnLuBYyPi/pTSnIhYA5iUUvqgrYvNTU1tDceevwff3/038z7mM3jtz3Htmfew+kYrsvmX1manwzbl3MP/yBHr/C/denXh1GsPmDf+YWv+jFnvfcTcj+t46LZn+PHtRzJorX5VXCK1t5raGo497784fc8rqa9L7HDwxgxeux/XnnVvsQ3tthY7HroJ5x45kiPXP5duvbrynau+CsDtv36IyS9N5fqf/Y3rf/Y3AM7+89foucJy1VwktbP6unouOenPnH3bEXSo6cA9Vz/Ka+Pf4qDTd+D5sRP51x3jWW/Eqhx25s6QEk+PfpmLvnXLvPFXGNSLviv24Km/vzyfuShSapsrzBExBLg9pbRu2f1tYDngFuBSoCvwEvC1lNK7EfEA8O2U0piI6AuMSSkNiYgOwNnA7hRn4e8Ae6aUZtCCTTbeKI35x4NtslxaOqS6umqXoMztttz3ql2CMncnFz+WUtqkuWFtduadUnoFWLei+9yKwZs3037bivdTKO95p5Tqge+VL0mSlnp+zluSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLkpQZw1uSpMwY3pIkZcbwliQpM4a3JEmZMbwlScqM4S1JUmYMb0mSMmN4S5KUGcNbkqTMGN6SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzBjekiRlxvCWJCkzkVKqdg2LXES8A7xa7ToWc32BKdUuQllzG9J/wu1nwQanlJZvbsASGd5asIgYk1LapNp1KF9uQ/pPuP38Z7xsLklSZgxvSZIyY3gvvS6rdgHKntuQ/hNuP/8B73lLkpQZz7wlScqM4b2Ui4itI+KZiHgiIraIiF2rXZOqJyJ6RsRxn3HcqyJi70Vdk6RPM7x1IPDTlNIwYE3A8F669QQ+U3hLaj+G9xIoIpaNiDsiYlxEPB0R+0XE9hHxeEQ8FRG/jYhlIuJIYF/grIi4HjgT2K88C9+vukuhKvkZsGq5DZwTEadExKMR8WRE/KihUUQcUvYbFxHXVow/IiL+GREveRa+9ImIIRExPiIuL6/o3RMRXSJiWEQ8XG4zN0dEr7L9AxHx84h4JCImRMTWZf+acvtr2Pa+Xt0lW/wY3kumnYHJKaUNUkrrAncBVwH7pZTWA2qBY1NKVwC3AqeklPYHzgD+mFIallL6Y5VqV3WdBrxYXom5F1gd2AwYBmwcESMiYh3g+8B2KaUNgG9WjN8f2Ar4EsWBgJY+qwMXpZTWAaYDXwGuAU5NKa0PPAX8oKJ9bUppM+BbFf2PAGaklDYFNgWOioiV22sBcmB4L5meAnYoj2i3BoYAL6eUJpTDrwZGVKs4ZWPH8vU4MBYYSrFj3g4YmVKaApBSmlYxzi0ppfqU0rNAv3auV4uHl1NKT5TvHwNWBXqmlEaV/Zruf26qaDukfL8jcEhEPAH8C+hDse2pVFvtArTopZQmRMRGFPevzwbur3JJylNQPA/x60Y9I06czzizm4yvpU/lNlBH8RxFa9rX8UkmBXBiSunuRVzbEsMz7yVQRAwAZqWUrgPOAbYAhkTEamWTg4FRzYz6HtCtfarUYqpyG7gbODwilgOIiIERsQLFweA+EdGn7N+7KpUqFzOAdxvuZ9Py/qfS3cCxEdERICLWiIhl27DG7HjmvWRaDzgnIuqBOcCxQA9gZETUAo8ClzYz3t+A08pLVT/1vvfSJ6U0NSL+ERFPA3cCvwceigiA94GDUkrPRMSPgVERUUdxWf2watWsLBwKXBoRXYGXgK8toP0VFJfQx0ax8b0D7NmmFWbGb1iTJCkzXjaXJCkzhrckSZkxvCVJyozhLUlSZgxvSZIyY3hLGYiIuvL7xp+OiJHlR24+67Tm/fpXRFwREWvPp+22EbHlZ5jHKxHRt7X9m7R5fyHn9cOI+PbC1ijlzPCW8vBh+Z3z6wIfA8dUDiw/v7/QUkpHll9l2pJtgYUOb0lty/CW8vN3YLXyrPjvEXEr8GxLv8QUhV9FxHMR8VdghYYJlb/qtEn5fueIGFv+Uth9ETGE4iDhpPKsf+uIWD4ibizn8WhEfL4ct0/5C1LPRMQVtOKrUSPiloh4rBzn6CbDzi/73xcRy5f9Vo2Iu8px/h4RQxfFypRy5DesSRkpz7B3ofilOICNgHVTSi+XATgjpbRpRCwD/CMi7gE2pPit9rUpfizkWeC3Taa7PHA5MKKcVu+U0rSIuBR4P6V0btnu98D5KaXRETGI4mss16L4NajRKaUzI2I3il+FWpDDy3l0AR6NiBtTSlOBZYExKaWTIuKMctonAJcBx6SUno+I4cDFFD+SIi11DG8pD13Kr62F4sz7NxSXsx9JKb1c9t8RWD8++R3tHhS/xDQCuD6lVAdMjojmfqhmc+DBhmk1+aWwSl8E1i6/LhWge/nd5yOAL5fj3hER77Zimb4REXuV71cqa50K1AMNX817HXBTOY8tKb7it2H8ZVoxD2mJZHhLefiw/I3tecoQ+6CyF838ElNE7LoI6+gAbJ5S+qiZWlotIralOBDYIqU0KyIeADq30DyV853edB1ISyvveUtLjpZ+ielBYL/ynnh/4AvNjPswMCIiVi7HbfilsKa/NHcPMO8nQSOiIUwfBA4o++0C9FpArT2Ad8vgHkpx5t+gA9Bw9eAAisvxM4GXI2Kfch4RERssYB7SEsvwlpYcV1Dczx5b/irYrymurt0MPF8OuwZ4qOmIKaV3gKMpLlGP45PL1rcBezU8sAZ8A9ikfCDuWT556v1HFOH/DMXl89cWUOtdQG1EjAd+RnHw0OADYLNyGbYDziz7HwgcUdb3DLBHK9aJtETyV8UkScqMZ96SJGXG8JYkKTOGtyRJmTG8JUnKjOEtSVJmDG9JkjJjeEuSlBnDW5KkzPw/AoaF7hdCTLIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_matrix(pred_arr, gold_arr, classes)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "premium",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}