0_preprocessing.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Setup e importazione"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# base\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import pprint\n",
    "import seaborn as sns\n",
    "import numpy as np \n",
    "from collections import Counter \n",
    "from matplotlib import pyplot as plt\n",
    "import statistics\n",
    "from IPython.display import clear_output\n",
    "from itertools import chain\n",
    "\n",
    "# nlp\n",
    "from num2words import num2words\n",
    "\n",
    "import string\n",
    "\n",
    "from bs4 import BeautifulSoup \n",
    "import time\n",
    "\n",
    "from textblob import TextBlob\n",
    "\n",
    "from pygments import highlight\n",
    "from pygments.lexers import get_lexer_by_name\n",
    "from pygments.formatters import HtmlFormatter\n",
    "from pygments.lexers import guess_lexer\n",
    "\n",
    "import emoji\n",
    "import demoji\n",
    "\n",
    "import re\n",
    "import contractions\n",
    "from contractions import contractions_dict\n",
    "\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize \n",
    "#nltk.download('punkt')\n",
    "from nltk import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "#nltk.download('wordnet')\n",
    "from nltk.corpus import stopwords\n",
    "import gensim\n",
    "from gensim.utils import simple_preprocess\n",
    "import gensim.corpora as corpora\n",
    "\n",
    "# download\n",
    "\n",
    "# Scarica il corpus delle stopwords inglesi\n",
    "#nltk.download('stopwords')\n",
    "# Ottieni la lista delle stopwords inglesi\n",
    "stop_words_en = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = [os.path.join(dirpath,f) for (dirpath, dirnames, filenames) in os.walk(\"Dataset_TLDRHQ\\\\\") for f in filenames]\n",
    "#files = files[0:100] # subset composed of train, val, test\n",
    "\n",
    "test = pd.DataFrame()\n",
    "train = pd.DataFrame()\n",
    "val = pd.DataFrame()\n",
    "\n",
    "for file in files:\n",
    "    #print(file)\n",
    "    temp = pd.read_json(file, lines=True)\n",
    "    temp.set_index(\"id\", inplace=True)\n",
    "        \n",
    "    if \"test\" in file:\n",
    "        #print(\"test\")\n",
    "        test = pd.concat([test, temp])\n",
    "    elif \"train\" in file:\n",
    "        #print(\"train\")\n",
    "        train = pd.concat([train, temp])\n",
    "    else:\n",
    "        #print(\"val\")\n",
    "        val = pd.concat([val, temp])\n",
    "    \n",
    "corpus = {'train': train,\n",
    "              'val': val,\n",
    "                'test': test}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key,value in corpus.items():\n",
    "    # Roporta id come colonna\n",
    "    value.reset_index(inplace=True)\n",
    "    # Rimozione colonne inutili\n",
    "    # columns = ds.columns.tolist()\n",
    "    # columns = [col for col in columns if col not in [\"document\", \"id\"]]\n",
    "    # ds.drop(columns, axis=1, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RS_2015-05-22930.json</td>\n",
       "      <td>a friend of mine who i have known for about 7 ...</td>\n",
       "      <td>\" friend \" got drunk and pissed all over my lo...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.055668210100000004, 0.0117288367, 0.0270273...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RC_2018-06-cm-26004.json</td>\n",
       "      <td>i have been in your position before .&lt;/s&gt;&lt;s&gt; j...</td>\n",
       "      <td>this is a reasonable area to be anxious about ...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.0163138408, 0.0, 0.028218535500000003, 0.13...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train-TLDR_RS_2018-11-43003.json</td>\n",
       "      <td>during the time i was with my nex , he would a...</td>\n",
       "      <td>the worst flying monkey in my past relationshi...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...</td>\n",
       "      <td>[0.0519922577, 0.0243238633, 0.0, 0.0452106589...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RS_2019-09-24738.json</td>\n",
       "      <td>i have tourretes syndrome and can share some o...</td>\n",
       "      <td>i 've got tourretes and have had some interest...</td>\n",
       "      <td>[1, 0]</td>\n",
       "      <td>[1.0, 0.0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RC_2018-04-cm-19148.json</td>\n",
       "      <td>your submission has been removed for the follo...</td>\n",
       "      <td>it 's not . --- * please refer to our [ detail...</td>\n",
       "      <td>[0, 1, 1]</td>\n",
       "      <td>[0.1587006961, 0.4320185615, 0.4092807425]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    id  \\\n",
       "0     train-TLDR_RS_2015-05-22930.json   \n",
       "1  train-TLDR_RC_2018-06-cm-26004.json   \n",
       "2     train-TLDR_RS_2018-11-43003.json   \n",
       "3     train-TLDR_RS_2019-09-24738.json   \n",
       "4  train-TLDR_RC_2018-04-cm-19148.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  a friend of mine who i have known for about 7 ...   \n",
       "1  i have been in your position before .</s><s> j...   \n",
       "2  during the time i was with my nex , he would a...   \n",
       "3  i have tourretes syndrome and can share some o...   \n",
       "4  your submission has been removed for the follo...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  \" friend \" got drunk and pissed all over my lo...   \n",
       "1  this is a reasonable area to be anxious about ...   \n",
       "2  the worst flying monkey in my past relationshi...   \n",
       "3  i 've got tourretes and have had some interest...   \n",
       "4  it 's not . --- * please refer to our [ detail...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1  [0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "2  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, ...   \n",
       "3                                             [1, 0]   \n",
       "4                                          [0, 1, 1]   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.055668210100000004, 0.0117288367, 0.0270273...  \n",
       "1  [0.0163138408, 0.0, 0.028218535500000003, 0.13...  \n",
       "2  [0.0519922577, 0.0243238633, 0.0, 0.0452106589...  \n",
       "3                                         [1.0, 0.0]  \n",
       "4         [0.1587006961, 0.4320185615, 0.4092807425]  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_test.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Exploration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 Basic Exploration and Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RC_2012-01-cm-19772.json</td>\n",
       "      <td>i 'm a woman , and i disagree .&lt;/s&gt;&lt;s&gt; this ri...</td>\n",
       "      <td>i 'm a girl and have come to say you = / = voi...</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "      <td>[0.612687031509198, 0.12888057183749302, 0.103...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RC_2015-08-cm-38561.json</td>\n",
       "      <td>i hope i 'm not too late here but i 'll give i...</td>\n",
       "      <td>pee 'd pants at graduation in front of parents...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "      <td>[0.03539792383995, 0.069989367887981, 0.030114...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>validation-TLDR_RS_2014-11-18668.json</td>\n",
       "      <td>we both go to the same college and live in the...</td>\n",
       "      <td>i feel like i 've outgrown my relationship . n...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]</td>\n",
       "      <td>[0.020986947535783, 0.026776450304275, 0.09564...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RC_2017-08-cm-30371.json</td>\n",
       "      <td>i 'm about 99.9 % sure that if there is n't an...</td>\n",
       "      <td>if there is already a mandatory hoa in place b...</td>\n",
       "      <td>[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.172966128493344, 0.065296809156498, 0.14868...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RS_2016-08-15226.json</td>\n",
       "      <td>okay , let me start this off by saying that i ...</td>\n",
       "      <td>i 'm in love with this girl but she wo n't get...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.02587269710534, 0.018578748340154, 0.022622...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      id  \\\n",
       "0    train-TLDR_RC_2012-01-cm-19772.json   \n",
       "1    train-TLDR_RC_2015-08-cm-38561.json   \n",
       "2  validation-TLDR_RS_2014-11-18668.json   \n",
       "3    train-TLDR_RC_2017-08-cm-30371.json   \n",
       "4       train-TLDR_RS_2016-08-15226.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  i 'm a woman , and i disagree .</s><s> this ri...   \n",
       "1  i hope i 'm not too late here but i 'll give i...   \n",
       "2  we both go to the same college and live in the...   \n",
       "3  i 'm about 99.9 % sure that if there is n't an...   \n",
       "4  okay , let me start this off by saying that i ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  i 'm a girl and have come to say you = / = voi...   \n",
       "1  pee 'd pants at graduation in front of parents...   \n",
       "2  i feel like i 've outgrown my relationship . n...   \n",
       "3  if there is already a mandatory hoa in place b...   \n",
       "4  i 'm in love with this girl but she wo n't get...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0                                       [1, 0, 0, 0]   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   \n",
       "2      [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]   \n",
       "3                  [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "4  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.612687031509198, 0.12888057183749302, 0.103...  \n",
       "1  [0.03539792383995, 0.069989367887981, 0.030114...  \n",
       "2  [0.020986947535783, 0.026776450304275, 0.09564...  \n",
       "3  [0.172966128493344, 0.065296809156498, 0.14868...  \n",
       "4  [0.02587269710534, 0.018578748340154, 0.022622...  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RC_2012-01-cm-19772.json</td>\n",
       "      <td>i 'm a woman , and i disagree .&lt;/s&gt;&lt;s&gt; this ri...</td>\n",
       "      <td>i 'm a girl and have come to say you = / = voi...</td>\n",
       "      <td>[1, 0, 0, 0]</td>\n",
       "      <td>[0.612687031509198, 0.12888057183749302, 0.103...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RC_2015-08-cm-38561.json</td>\n",
       "      <td>i hope i 'm not too late here but i 'll give i...</td>\n",
       "      <td>pee 'd pants at graduation in front of parents...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "      <td>[0.03539792383995, 0.069989367887981, 0.030114...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>validation-TLDR_RS_2014-11-18668.json</td>\n",
       "      <td>we both go to the same college and live in the...</td>\n",
       "      <td>i feel like i 've outgrown my relationship . n...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]</td>\n",
       "      <td>[0.020986947535783, 0.026776450304275, 0.09564...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RC_2017-08-cm-30371.json</td>\n",
       "      <td>i 'm about 99.9 % sure that if there is n't an...</td>\n",
       "      <td>if there is already a mandatory hoa in place b...</td>\n",
       "      <td>[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.172966128493344, 0.065296809156498, 0.14868...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RS_2016-08-15226.json</td>\n",
       "      <td>okay , let me start this off by saying that i ...</td>\n",
       "      <td>i 'm in love with this girl but she wo n't get...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.02587269710534, 0.018578748340154, 0.022622...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      id  \\\n",
       "0    train-TLDR_RC_2012-01-cm-19772.json   \n",
       "1    train-TLDR_RC_2015-08-cm-38561.json   \n",
       "2  validation-TLDR_RS_2014-11-18668.json   \n",
       "3    train-TLDR_RC_2017-08-cm-30371.json   \n",
       "4       train-TLDR_RS_2016-08-15226.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  i 'm a woman , and i disagree .</s><s> this ri...   \n",
       "1  i hope i 'm not too late here but i 'll give i...   \n",
       "2  we both go to the same college and live in the...   \n",
       "3  i 'm about 99.9 % sure that if there is n't an...   \n",
       "4  okay , let me start this off by saying that i ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  i 'm a girl and have come to say you = / = voi...   \n",
       "1  pee 'd pants at graduation in front of parents...   \n",
       "2  i feel like i 've outgrown my relationship . n...   \n",
       "3  if there is already a mandatory hoa in place b...   \n",
       "4  i 'm in love with this girl but she wo n't get...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0                                       [1, 0, 0, 0]   \n",
       "1  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   \n",
       "2      [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0]   \n",
       "3                  [1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "4  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.612687031509198, 0.12888057183749302, 0.103...  \n",
       "1  [0.03539792383995, 0.069989367887981, 0.030114...  \n",
       "2  [0.020986947535783, 0.026776450304275, 0.09564...  \n",
       "3  [0.172966128493344, 0.065296809156498, 0.14868...  \n",
       "4  [0.02587269710534, 0.018578748340154, 0.022622...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus['val'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RS_2019-02-45730.json</td>\n",
       "      <td>so i just had the thought of what if people co...</td>\n",
       "      <td>name a new eve every 2 - 5 generations so that...</td>\n",
       "      <td>[0, 0, 1, 0, 0]</td>\n",
       "      <td>[0.145231701293733, 0.082184056849748, 0.47712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RC_2011-05-cm-12238.json</td>\n",
       "      <td>this is kind of an interesting topic for me , ...</td>\n",
       "      <td>my mom is crazy , hates me , wishes she would ...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.053637180715166, 0.025363965159833003, 0.02...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train-TLDR_RC_2020-06-04-cm-1657.json</td>\n",
       "      <td>i was running a campaign for my friends and fa...</td>\n",
       "      <td>the alchemist gave the warlock cocaine to keep...</td>\n",
       "      <td>[0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.0, 0.096297655087828, 0.17063644555607402, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RC_2019-12-cm-45553.json</td>\n",
       "      <td>i ve tried and failed at a lot of things .&lt;/s&gt;...</td>\n",
       "      <td>is this dumb fuckin juul is the only thing tha...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.027738905810231, 0.066785729204345, 0.03698...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RS_2013-04-21129.json</td>\n",
       "      <td>( sorry for the wall of text , i 'm upset and ...</td>\n",
       "      <td>my bf believes that we are eventually going to...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.294148746317319, 0.070204034502615, 0.02593...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      id  \\\n",
       "0       train-TLDR_RS_2019-02-45730.json   \n",
       "1    train-TLDR_RC_2011-05-cm-12238.json   \n",
       "2  train-TLDR_RC_2020-06-04-cm-1657.json   \n",
       "3    train-TLDR_RC_2019-12-cm-45553.json   \n",
       "4       train-TLDR_RS_2013-04-21129.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  so i just had the thought of what if people co...   \n",
       "1  this is kind of an interesting topic for me , ...   \n",
       "2  i was running a campaign for my friends and fa...   \n",
       "3  i ve tried and failed at a lot of things .</s>...   \n",
       "4  ( sorry for the wall of text , i 'm upset and ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  name a new eve every 2 - 5 generations so that...   \n",
       "1  my mom is crazy , hates me , wishes she would ...   \n",
       "2  the alchemist gave the warlock cocaine to keep...   \n",
       "3  is this dumb fuckin juul is the only thing tha...   \n",
       "4  my bf believes that we are eventually going to...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0                                    [0, 0, 1, 0, 0]   \n",
       "1  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "2                                    [0, 0, 0, 1, 0]   \n",
       "3      [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "4                     [1, 0, 0, 0, 0, 0, 0, 0, 1, 0]   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.145231701293733, 0.082184056849748, 0.47712...  \n",
       "1  [0.053637180715166, 0.025363965159833003, 0.02...  \n",
       "2  [0.0, 0.096297655087828, 0.17063644555607402, ...  \n",
       "3  [0.027738905810231, 0.066785729204345, 0.03698...  \n",
       "4  [0.294148746317319, 0.070204034502615, 0.02593...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train-TLDR_RS_2019-02-45730.json</th>\n",
       "      <td>so i just had the thought of what if people co...</td>\n",
       "      <td>name a new eve every 2 - 5 generations so that...</td>\n",
       "      <td>[0, 0, 1, 0, 0]</td>\n",
       "      <td>[0.145231701293733, 0.082184056849748, 0.47712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train-TLDR_RC_2011-05-cm-12238.json</th>\n",
       "      <td>this is kind of an interesting topic for me , ...</td>\n",
       "      <td>my mom is crazy , hates me , wishes she would ...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.053637180715166, 0.025363965159833003, 0.02...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train-TLDR_RC_2020-06-04-cm-1657.json</th>\n",
       "      <td>i was running a campaign for my friends and fa...</td>\n",
       "      <td>the alchemist gave the warlock cocaine to keep...</td>\n",
       "      <td>[0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.0, 0.096297655087828, 0.17063644555607402, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train-TLDR_RC_2019-12-cm-45553.json</th>\n",
       "      <td>i ve tried and failed at a lot of things .&lt;/s&gt;...</td>\n",
       "      <td>is this dumb fuckin juul is the only thing tha...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.027738905810231, 0.066785729204345, 0.03698...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>train-TLDR_RS_2013-04-21129.json</th>\n",
       "      <td>( sorry for the wall of text , i 'm upset and ...</td>\n",
       "      <td>my bf believes that we are eventually going to...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.294148746317319, 0.070204034502615, 0.02593...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                document  \\\n",
       "id                                                                                         \n",
       "train-TLDR_RS_2019-02-45730.json       so i just had the thought of what if people co...   \n",
       "train-TLDR_RC_2011-05-cm-12238.json    this is kind of an interesting topic for me , ...   \n",
       "train-TLDR_RC_2020-06-04-cm-1657.json  i was running a campaign for my friends and fa...   \n",
       "train-TLDR_RC_2019-12-cm-45553.json    i ve tried and failed at a lot of things .</s>...   \n",
       "train-TLDR_RS_2013-04-21129.json       ( sorry for the wall of text , i 'm upset and ...   \n",
       "\n",
       "                                                                                 summary  \\\n",
       "id                                                                                         \n",
       "train-TLDR_RS_2019-02-45730.json       name a new eve every 2 - 5 generations so that...   \n",
       "train-TLDR_RC_2011-05-cm-12238.json    my mom is crazy , hates me , wishes she would ...   \n",
       "train-TLDR_RC_2020-06-04-cm-1657.json  the alchemist gave the warlock cocaine to keep...   \n",
       "train-TLDR_RC_2019-12-cm-45553.json    is this dumb fuckin juul is the only thing tha...   \n",
       "train-TLDR_RS_2013-04-21129.json       my bf believes that we are eventually going to...   \n",
       "\n",
       "                                                                              ext_labels  \\\n",
       "id                                                                                         \n",
       "train-TLDR_RS_2019-02-45730.json                                         [0, 0, 1, 0, 0]   \n",
       "train-TLDR_RC_2011-05-cm-12238.json    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "train-TLDR_RC_2020-06-04-cm-1657.json                                    [0, 0, 0, 1, 0]   \n",
       "train-TLDR_RC_2019-12-cm-45553.json        [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "train-TLDR_RS_2013-04-21129.json                          [1, 0, 0, 0, 0, 0, 0, 0, 1, 0]   \n",
       "\n",
       "                                                                               rg_labels  \n",
       "id                                                                                        \n",
       "train-TLDR_RS_2019-02-45730.json       [0.145231701293733, 0.082184056849748, 0.47712...  \n",
       "train-TLDR_RC_2011-05-cm-12238.json    [0.053637180715166, 0.025363965159833003, 0.02...  \n",
       "train-TLDR_RC_2020-06-04-cm-1657.json  [0.0, 0.096297655087828, 0.17063644555607402, ...  \n",
       "train-TLDR_RC_2019-12-cm-45553.json    [0.027738905810231, 0.066785729204345, 0.03698...  \n",
       "train-TLDR_RS_2013-04-21129.json       [0.294148746317319, 0.070204034502615, 0.02593...  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus['test'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RS_2019-07-25907.json</td>\n",
       "      <td>hey y' all , i 've been a lurker in this commu...</td>\n",
       "      <td>i 'm publishing betas of some stuff i 've been...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.040456223279431006, 0.022386650144119002, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RS_2011-07-7721.json</td>\n",
       "      <td>as the title says , dreams that really scared ...</td>\n",
       "      <td>dreamed i was a cop that got shot in the face ...</td>\n",
       "      <td>[1, 1, 0, 0, 1]</td>\n",
       "      <td>[0.17657869627445003, 0.382656749215711, 0.157...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train-TLDR_RC_2019-08-cm-9972.json</td>\n",
       "      <td>from u / orangejews4u here 's my \" must read \"...</td>\n",
       "      <td>not worth ) ] ( https://www.reddit.com/r/howto...</td>\n",
       "      <td>[0, 1, 0, 1]</td>\n",
       "      <td>[0.23651113155692202, 0.336435558104879, 0.165...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RS_2015-04-39164.json</td>\n",
       "      <td>hello / r / relationships , i did n't really k...</td>\n",
       "      <td>wife wants a break , has started going out par...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.044352260202115, 0.016666897769790003, 0.02...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RC_2016-05-cm-45089.json</td>\n",
       "      <td>i agree with you to be honest .&lt;/s&gt;&lt;s&gt; my dad ...</td>\n",
       "      <td>i agree with you . regular dad things . some s...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.43353065193323603, 0.054993317369966, 0.047...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    id  \\\n",
       "0     train-TLDR_RS_2019-07-25907.json   \n",
       "1      train-TLDR_RS_2011-07-7721.json   \n",
       "2   train-TLDR_RC_2019-08-cm-9972.json   \n",
       "3     train-TLDR_RS_2015-04-39164.json   \n",
       "4  train-TLDR_RC_2016-05-cm-45089.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  hey y' all , i 've been a lurker in this commu...   \n",
       "1  as the title says , dreams that really scared ...   \n",
       "2  from u / orangejews4u here 's my \" must read \"...   \n",
       "3  hello / r / relationships , i did n't really k...   \n",
       "4  i agree with you to be honest .</s><s> my dad ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  i 'm publishing betas of some stuff i 've been...   \n",
       "1  dreamed i was a cop that got shot in the face ...   \n",
       "2  not worth ) ] ( https://www.reddit.com/r/howto...   \n",
       "3  wife wants a break , has started going out par...   \n",
       "4  i agree with you . regular dad things . some s...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1                                    [1, 1, 0, 0, 1]   \n",
       "2                                       [0, 1, 0, 1]   \n",
       "3  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4               [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.040456223279431006, 0.022386650144119002, 0...  \n",
       "1  [0.17657869627445003, 0.382656749215711, 0.157...  \n",
       "2  [0.23651113155692202, 0.336435558104879, 0.165...  \n",
       "3  [0.044352260202115, 0.016666897769790003, 0.02...  \n",
       "4  [0.43353065193323603, 0.054993317369966, 0.047...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train-TLDR_RS_2019-07-25907.json</td>\n",
       "      <td>hey y' all , i 've been a lurker in this commu...</td>\n",
       "      <td>i 'm publishing betas of some stuff i 've been...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.040456223279431006, 0.022386650144119002, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train-TLDR_RS_2011-07-7721.json</td>\n",
       "      <td>as the title says , dreams that really scared ...</td>\n",
       "      <td>dreamed i was a cop that got shot in the face ...</td>\n",
       "      <td>[1, 1, 0, 0, 1]</td>\n",
       "      <td>[0.17657869627445003, 0.382656749215711, 0.157...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train-TLDR_RC_2019-08-cm-9972.json</td>\n",
       "      <td>from u / orangejews4u here 's my \" must read \"...</td>\n",
       "      <td>not worth ) ] ( https://www.reddit.com/r/howto...</td>\n",
       "      <td>[0, 1, 0, 1]</td>\n",
       "      <td>[0.23651113155692202, 0.336435558104879, 0.165...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train-TLDR_RS_2015-04-39164.json</td>\n",
       "      <td>hello / r / relationships , i did n't really k...</td>\n",
       "      <td>wife wants a break , has started going out par...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.044352260202115, 0.016666897769790003, 0.02...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train-TLDR_RC_2016-05-cm-45089.json</td>\n",
       "      <td>i agree with you to be honest .&lt;/s&gt;&lt;s&gt; my dad ...</td>\n",
       "      <td>i agree with you . regular dad things . some s...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.43353065193323603, 0.054993317369966, 0.047...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    id  \\\n",
       "0     train-TLDR_RS_2019-07-25907.json   \n",
       "1      train-TLDR_RS_2011-07-7721.json   \n",
       "2   train-TLDR_RC_2019-08-cm-9972.json   \n",
       "3     train-TLDR_RS_2015-04-39164.json   \n",
       "4  train-TLDR_RC_2016-05-cm-45089.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  hey y' all , i 've been a lurker in this commu...   \n",
       "1  as the title says , dreams that really scared ...   \n",
       "2  from u / orangejews4u here 's my \" must read \"...   \n",
       "3  hello / r / relationships , i did n't really k...   \n",
       "4  i agree with you to be honest .</s><s> my dad ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  i 'm publishing betas of some stuff i 've been...   \n",
       "1  dreamed i was a cop that got shot in the face ...   \n",
       "2  not worth ) ] ( https://www.reddit.com/r/howto...   \n",
       "3  wife wants a break , has started going out par...   \n",
       "4  i agree with you . regular dad things . some s...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "1                                    [1, 1, 0, 0, 1]   \n",
       "2                                       [0, 1, 0, 1]   \n",
       "3  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "4               [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0]   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.040456223279431006, 0.022386650144119002, 0...  \n",
       "1  [0.17657869627445003, 0.382656749215711, 0.157...  \n",
       "2  [0.23651113155692202, 0.336435558104879, 0.165...  \n",
       "3  [0.044352260202115, 0.016666897769790003, 0.02...  \n",
       "4  [0.43353065193323603, 0.054993317369966, 0.047...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus['train'].head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i support a draft simply because a society should feel it if the nation is at war .', ' it should be more of an issue , and if you have a draft i think it helps to put it in your mind .', \" if regular fellas get drafted there is a better chance one feels sympathetic towards them , it 's not just some bunch of professional soldiers .\", ' and there is the chance that you , your kids or somebody you know get drafted .', ' i think just the danger of getting drafted changes the conception , you at least have to think of ways to avoid getting drafted .', ' i think it puts the society back into the army , and the army back into the minds of the people of the society they are supposedly fighting for .', ' i know it would suck for the individual and may be a hassle ( a least to think of ways to not get drafted ) , but it would be for the greater good of the society and how it deals with wars .', ' because it should be unpleasant if your country fights a war .']\n"
     ]
    }
   ],
   "source": [
    "print(corpus['train'].sample(1)['document'].iloc[0])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check shape and dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key:  train\n",
      "Shape:  (1590132, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n",
      "Key:  val\n",
      "Shape:  (40481, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n",
      "Key:  test\n",
      "Shape:  (40486, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "count = []\n",
    "for key, value in corpus.items():\n",
    "    print(\"Key: \", key)\n",
    "    print(\"Shape: \", value.shape)\n",
    "    print(\"Dtypes: \", value.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key:  train\n",
      "Shape:  (1590132, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n",
      "Key:  val\n",
      "Shape:  (40481, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n",
      "Key:  test\n",
      "Shape:  (40486, 5)\n",
      "Dtypes:  id            object\n",
      "document      object\n",
      "summary       object\n",
      "ext_labels    object\n",
      "rg_labels     object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# iterate on corpus dict of dataframe and print key value and dataframe shape and dtypes \n",
    "for key, value in corpus.items():\n",
    "    print(\"Key: \", key)\n",
    "    print(\"Shape: \", value.shape)\n",
    "    print(\"Dtypes: \", value.dtypes)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Memory usage dei dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total memory usage: 3.8 GB\n",
      "train:None\n",
      "Total memory usage: 196.9 MB\n",
      "val:None\n",
      "Total memory usage: 99.0 MB\n",
      "test:None\n"
     ]
    }
   ],
   "source": [
    "import humanize\n",
    "\n",
    "def print_memory_usage(df):\n",
    "  print(f\"Total memory usage: {humanize.naturalsize(df.memory_usage(deep=True).sum())}\")\n",
    "  # print(f\"Memory usage by column:\")\n",
    "  # print(df.memory_usage(deep=True))\n",
    "\n",
    "for key, value in corpus.items():\n",
    "  print(f\"{key}:{print_memory_usage(value)}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Temporanea rimozione del ds più oneroso"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#del train"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 Check ext_labels nulle"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "alcuni post hanno come ext_labels una lista di 0, in pratica nessuna frase è stata scelta per il sommario. Associate per lo più a post \"corti\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(train['ext_labels'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train- number of rows with only 0 values in ext_labels:466/1590132\n",
      "val- number of rows with only 0 values in ext_labels:23/80967\n",
      "test- number of rows with only 0 values in ext_labels:6/40486\n"
     ]
    }
   ],
   "source": [
    "def sum_is_zero(row):\n",
    "  return sum(row['ext_labels']) == 0\n",
    "\n",
    "for key, value in corpus.items():\n",
    "    num_rows = value[value.apply(sum_is_zero, axis=1)].shape[0]\n",
    "    print(f\"{key}- number of rows with only 0 values in ext_labels:{num_rows}/{value.shape[0]}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 Post/Summaries Statistics and Viz"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lunghezza post"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Caratteri"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verifico presenza di post molto brevi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of short documents:\n",
      "24\n",
      "620646    i really love her .\n",
      "Name: document, dtype: object\n",
      "val - number of short documents:\n",
      "2\n",
      "9299    you 're and idiot .\n",
      "Name: document, dtype: object\n",
      "test - number of short documents:\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "n = 20\n",
    "\n",
    "for key, value in corpus.items():\n",
    "    print(f\"{key} - number of short documents:\")\n",
    "    num_short_documents = len(value[value['document'].apply(len) < 20])\n",
    "    print(num_short_documents)\n",
    "    if num_short_documents != 0:\n",
    "        short_documents = value[value['document'].apply(len) < 20]['document']\n",
    "        # start_index = random.randint(0, len(short_documents) - 5)\n",
    "        #short_documents.iloc[start_index:start_index+5]\n",
    "        print(short_documents.sample())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Osserviamo la **distribuzione** delle lunghezze di caratteri dei post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAIACAYAAACmbZRAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA/10lEQVR4nO3de3wU5d3///cmYZOgJAQwCalBoijng6DE2IJSUgLm1qK0VUREjSI2oUIsIIoYtb+GQhFREW5vldiKgvSrqEjBEE4iASUQTkrqAUQrG7yFZAUhh93r9wfN3KwBhBzY2c3r+XjMA3auT2auGfLYN3O4ZhzGGCMAAGA7If7uAAAAODlCGgAAmyKkAQCwKUIaAACbIqQBALApQhoAAJsipAEAsClCGgAAmyKkAQCwKUIagF+0b99ed9xxh7+7AdgaIQ3glDZs2KCcnByVlZX5uytAk+Tg2d0ATuWvf/2rJkyYoD179qh9+/YNuuyKigqFhISoWbNmDbpcIJhwJA2g3rxer44dO3ZWPxMeHk5AAz+BkAZwUjk5OZowYYIkKSkpSQ6HQw6HQ3v37pXD4VBWVpYWLFigrl27Kjw8XMuXL5d0/Oj76quvVuvWrRUZGak+ffroH//4R63l//iadF5enhwOhz744ANlZ2frggsu0Hnnnacbb7xR33777TnZZsBuwvzdAQD2dNNNN+lf//qXXnvtNc2aNUtt2rSRJF1wwQWSpFWrVun1119XVlaW2rRpY50Onz17tm644QaNGDFClZWVWrhwoX77299q6dKlSk9P/8n1jh07VjExMXr00Ue1d+9ePfXUU8rKytKiRYsabVsBuyKkAZxUjx491Lt3b7322msaOnRorWvSJSUl2rFjh7p06eIz/1//+pciIyOtz1lZWerdu7eefPLJMwrp1q1b67333pPD4ZB0/FT6008/rfLyckVHR9d/w4AAwuluAHVyzTXX1ApoST4BfejQIZWXl6tfv37asmXLGS139OjRVkBLUr9+/eTxePTll1/Wv9NAgOFIGkCdJCUlnXT+0qVL9ac//UnFxcWqqKiw5p8YvKfTrl07n88xMTGSjgc+0NRwJA2gTk48Yq7x/vvv64YbblBERISee+45LVu2TPn5+br11lt1pqM9Q0NDTzqf0aJoijiSBnBKZ3r0W+P//b//p4iICK1YsULh4eHW/Pnz5zd014AmgSNpAKd03nnnSdIZP3EsNDRUDodDHo/Hmrd3714tWbKkEXoHBD9CGsAp9enTR5L08MMP6+9//7sWLlyoI0eOnLI+PT1dP/zwgwYPHqx58+bp8ccfV3Jysjp06HCuugwEFU53AzilK6+8Uk888YTmzZun5cuXy+v1as+ePaes/+Uvf6kXX3xR06ZN07hx45SUlKS//OUv2rt3r7Zv334Oew4EB57dDQCATXG6GwAAmyKkAQCwKUIaAACbIqQBALApQhoAAJsipAEAsClCuoEYY+R2u3m+MACgwRDSDeT7779XdHS0vv/+e393BQAQJAhpAABsipAGAMCmCGkAAGyKkAYAwKYIaQAAbIqQBgDApghpAABsipAGAMCmCGkAAGyKkAYAwKYIaQAAbIqQBgDApghpAABsipAGAMCmCGkAAGyKkAYAwKYIaQAAbIqQBgDApghpAABsipAGAMCmCOkA4fF45PF4/N0NAMA5REgHCEIaAJoeQhoAAJsipAOAMUZHjx5VRUWFv7sCADiHCOkAUFVVpduff19er9ffXQEAnEOEdIBwhPBPBQBNDd/8AcDj8UjG+LsbAIBzjJAGAMCmCGkAAGyKkAYAwKYI6QBw/CEmXJMGgKaGkA4AHo9HxhieOgYATQwhDQCATRHSAADYFCENAIBNEdIBouaatOGhJgDQZBDSgcJ4dc/ft/L8bgBoQgjpAMLzuwGgaeFbHwAAmyKkAQCwKb+GdG5urq688kq1aNFCsbGxGjp0qEpKSnxqjh07pszMTLVu3Vrnn3++hg0bptLSUp+affv2KT09Xc2bN1dsbKwmTJig6upqn5o1a9aod+/eCg8PV4cOHZSXl1erP3PmzFH79u0VERGh5ORkffjhhw2+zQAAnCm/hvTatWuVmZmpjRs3Kj8/X1VVVRo0aJCOHDli1YwfP17vvPOOFi9erLVr1+qbb77RTTfdZLV7PB6lp6ersrJSGzZs0Msvv6y8vDxNnTrVqtmzZ4/S09M1YMAAFRcXa9y4cbr77ru1YsUKq2bRokXKzs7Wo48+qi1btqhnz55KS0vTgQMHzs3OAADgx4yNHDhwwEgya9euNcYYU1ZWZpo1a2YWL15s1XzyySdGkiksLDTGGLNs2TITEhJiXC6XVTN37lwTFRVlKioqjDHGTJw40XTt2tVnXTfffLNJS0uzPvft29dkZmZanz0ej0lISDC5ubln1Pfy8nIjyZSXl5/lVv+0srIyc/20N83QWfnWNgEAgp+trkmXl5dLklq1aiVJKioqUlVVlVJTU62aTp06qV27diosLJQkFRYWqnv37oqLi7Nq0tLS5Ha7tWvXLqvmxGXU1NQso7KyUkVFRT41ISEhSk1NtWp+rKKiQm6322dqDOY/46MlyXh5djcANCW2CWmv16tx48bp5z//ubp16yZJcrlccjqdatmypU9tXFycXC6XVXNiQNe017Sdrsbtduvo0aP63//9X3k8npPW1Czjx3JzcxUdHW1NiYmJddvwn+D1enXnC++LZ5gAQNNjm5DOzMzUzp07tXDhQn935YxMnjxZ5eXl1vTVV1812rpqxkdzJA0ATUuYvzsgSVlZWVq6dKnWrVunCy+80JofHx+vyspKlZWV+RxNl5aWKj4+3qr58V3YNXd/n1jz4zvCS0tLFRUVpcjISIWGhio0NPSkNTXL+LHw8HCFh4fXbYMBADgDfj2SNsYoKytLb775platWqWkpCSf9j59+qhZs2YqKCiw5pWUlGjfvn1KSUmRJKWkpGjHjh0+d2Hn5+crKipKXbp0sWpOXEZNTc0ynE6n+vTp41Pj9XpVUFBg1fjL8ed1+7ULAAA/8euRdGZmpl599VW99dZbatGihXX9Nzo6WpGRkYqOjlZGRoays7PVqlUrRUVFaezYsUpJSdFVV10lSRo0aJC6dOmikSNHavr06XK5XJoyZYoyMzOtI90xY8bo2Wef1cSJE3XXXXdp1apVev311/Xuu+9afcnOztaoUaN0xRVXqG/fvnrqqad05MgR3Xnnned+x5yE8XpEWgNAE+PPW8slnXSaP3++VXP06FHz+9//3sTExJjmzZubG2+80ezfv99nOXv37jVDhgwxkZGRpk2bNuaBBx4wVVVVPjWrV682vXr1Mk6n01x88cU+66jxzDPPmHbt2hmn02n69u1rNm7ceMbb0lhDsCoqKswN09826f/fP8wN05ea77//vkGXDwCwL4cxHJ41BLfbrejoaJWXlysqKqrBlltZWanfzl4hT1WlQptFaMF91+j8889vsOUDAOzLNnd346cZ3ikNAE0KIR1IjFd35m3mndIA0EQQ0gHEeD2Sg38yAGgq+MYHAMCmCGkAAGyKkAYAwKYIaQAAbIqQBgDApghpAABsipAGAMCmCGkAAGyKkA4wxuuRx+PxdzcAAOcAIQ0AgE0R0gAA2BQhDQCATRHSAADYFCENAIBNEdIBhru7AaDpIKQBALApQhoAAJsipAEAsClC2uY8Ho9kjL+7AQDwA0IaAACbIqRt7vid3BxJA0BTREgHGIZgAUDTQUgDAGBThDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgHGIZgAUDTQUgDAGBThDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgDAGBTfg3pdevW6frrr1dCQoIcDoeWLFni0+5wOE46zZgxw6pp3759rfZp06b5LGf79u3q16+fIiIilJiYqOnTp9fqy+LFi9WpUydFRESoe/fuWrZsWaNsMwAAZ8qvIX3kyBH17NlTc+bMOWn7/v37faaXXnpJDodDw4YN86l7/PHHferGjh1rtbndbg0aNEgXXXSRioqKNGPGDOXk5Oj555+3ajZs2KDhw4crIyNDW7du1dChQzV06FDt3LmzcTYcAIAzEObPlQ8ZMkRDhgw5ZXt8fLzP57feeksDBgzQxRdf7DO/RYsWtWprLFiwQJWVlXrppZfkdDrVtWtXFRcX68knn9To0aMlSbNnz9bgwYM1YcIESdITTzyh/Px8Pfvss5o3b159NrHB8cQxAGg6AuaadGlpqd59911lZGTUaps2bZpat26tyy+/XDNmzFB1dbXVVlhYqP79+8vpdFrz0tLSVFJSokOHDlk1qampPstMS0tTYWHhKftTUVEht9vtM50Lxhh5PB4ZY87J+gAA/hMwIf3yyy+rRYsWuummm3zm/+EPf9DChQu1evVq3Xvvvfrzn/+siRMnWu0ul0txcXE+P1Pz2eVynbampv1kcnNzFR0dbU2JiYn12r4zZrwa/co2eb3ec7M+AIDf+PV099l46aWXNGLECEVERPjMz87Otv7eo0cPOZ1O3XvvvcrNzVV4eHij9Wfy5Mk+63a73Y0S1Cc7anaEBMz/rQAA9RAQIf3++++rpKREixYt+sna5ORkVVdXa+/everYsaPi4+NVWlrqU1PzueY69qlqTnWdW5LCw8Mb9T8BAAAExCHZiy++qD59+qhnz54/WVtcXKyQkBDFxsZKklJSUrRu3TpVVVVZNfn5+erYsaNiYmKsmoKCAp/l5OfnKyUlpQG3AgCAs+PXkD58+LCKi4tVXFwsSdqzZ4+Ki4u1b98+q8btdmvx4sW6++67a/18YWGhnnrqKW3btk1ffPGFFixYoPHjx+u2226zAvjWW2+V0+lURkaGdu3apUWLFmn27Nk+p6rvv/9+LV++XDNnztTu3buVk5OjzZs3Kysrq3F3AAAAp+HX092bN2/WgAEDrM81wTlq1Cjl5eVJkhYuXChjjIYPH17r58PDw7Vw4ULl5OSooqJCSUlJGj9+vE8AR0dH67333lNmZqb69OmjNm3aaOrUqdbwK0m6+uqr9eqrr2rKlCl66KGHdOmll2rJkiXq1q1bI205AAA/zWEYy9Mg3G63oqOjVV5erqioqAZbbnl5uUbOWy2vxyOHI0TGeOU8L0aLs65RaGhog60HAGA/AXFNGgCApoiQBgDApgjpAMSjQQGgaSCkAQCwKUI6AHEkDQBNAyENAIBNEdIAANgUIW1jNa+lBAA0TYS0jXm9Xt35wvvicTMA0DQR0jZ3stdScuMYADQNhDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgDAGBThDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgDAGBThDQAADZFSAcg3oIFAE0DIQ0AgE0R0gHIGKPKykpVV1f7uysAgEZESAci49U9f98qr9fr754AABoRIR2gHCH80wFAsOObHgAAmyKkAQCwKUIaAACbIqQBALApQhoAAJsipAEAsClC2sY8Ho+M8XcvAAD+QkgDAGBThHSA4iUbABD8CGkAAGyKkAYAwKb8GtLr1q3T9ddfr4SEBDkcDi1ZssSn/Y477pDD4fCZBg8e7FNz8OBBjRgxQlFRUWrZsqUyMjJ0+PBhn5rt27erX79+ioiIUGJioqZPn16rL4sXL1anTp0UERGh7t27a9myZQ2+vQAAnA2/hvSRI0fUs2dPzZkz55Q1gwcP1v79+63ptdde82kfMWKEdu3apfz8fC1dulTr1q3T6NGjrXa3261BgwbpoosuUlFRkWbMmKGcnBw9//zzVs2GDRs0fPhwZWRkaOvWrRo6dKiGDh2qnTt3NvxGAwBwhhzG2GOQj8Ph0JtvvqmhQ4da8+644w6VlZXVOsKu8cknn6hLly766KOPdMUVV0iSli9fruuuu05ff/21EhISNHfuXD388MNyuVxyOp2SpAcffFBLlizR7t27JUk333yzjhw5oqVLl1rLvuqqq9SrVy/NmzfvjPrvdrsVHR2t8vJyRUVF1WEP1FZZWanfzl4hr8cj4/XI4QiRMV45HCEKDW+u1+7rr8jIyAZZFwDAfmx/TXrNmjWKjY1Vx44ddd999+m7776z2goLC9WyZUsroCUpNTVVISEh2rRpk1XTv39/K6AlKS0tTSUlJTp06JBVk5qa6rPetLQ0FRYWNuamAQBwWmH+7sDpDB48WDfddJOSkpL0+eef66GHHtKQIUNUWFio0NBQuVwuxcbG+vxMWFiYWrVqJZfLJUlyuVxKSkryqYmLi7PaYmJi5HK5rHkn1tQs42QqKipUUVFhfXa73fXaVgAAfszWIX3LLbdYf+/evbt69OihSy65RGvWrNHAgQP92DMpNzdXjz32mF/7AAAIbrY/3X2iiy++WG3atNFnn30mSYqPj9eBAwd8aqqrq3Xw4EHFx8dbNaWlpT41NZ9/qqam/WQmT56s8vJya/rqq6/qt3EAAPxIQIX0119/re+++05t27aVJKWkpKisrExFRUVWzapVq+T1epWcnGzVrFu3TlVVVVZNfn6+OnbsqJiYGKumoKDAZ135+flKSUk5ZV/Cw8MVFRXlMwEA0JD8GtKHDx9WcXGxiouLJUl79uxRcXGx9u3bp8OHD2vChAnauHGj9u7dq4KCAv36179Whw4dlJaWJknq3LmzBg8erHvuuUcffvihPvjgA2VlZemWW25RQkKCJOnWW2+V0+lURkaGdu3apUWLFmn27NnKzs62+nH//fdr+fLlmjlzpnbv3q2cnBxt3rxZWVlZ53yfAABQw68hvXnzZl1++eW6/PLLJUnZ2dm6/PLLNXXqVIWGhmr79u264YYbdNlllykjI0N9+vTR+++/r/DwcGsZCxYsUKdOnTRw4EBdd911+sUvfuEzBjo6Olrvvfee9uzZoz59+uiBBx7Q1KlTfcZSX3311Xr11Vf1/PPPq2fPnvrHP/6hJUuWqFu3buduZwAA8CO2GScd6BgnDQBoaAF1TRr/h7dgAUDwI6RtzOPxSJzoAIAmi5AGAMCmCGkAAGyKkA5QXJMGgOBHSAMAYFOEdIDiSBoAgh8hHaCMMfJ4PGKYOwAEL0I6UBmvRr+yTV6v1989AQA0EkI6gDlC+OcDgGDGtzwAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgDAGBThHQA46ljABDcCGkAAGyKkAYAwKYIaQAAbIqQBgDApghpAABsipAOYNzdDQDBjZAGAMCmCGkbO36UbPzdDQCAnxDSAADYFCENAIBNEdIBjBvHACC4EdIAANgUIQ0AgE0R0gAA2BQhDQCATRHSAYwbxwAguBHSAADYFCENAIBNEdIAANgUIQ0AgE0R0gHMGCOPxyNjeAkHAAQjQjqQGa8y/rZVXq/X3z0BADQCQjrAOUL4JwSAYOXXb/h169bp+uuvV0JCghwOh5YsWWK1VVVVadKkSerevbvOO+88JSQk6Pbbb9c333zjs4z27dvL4XD4TNOmTfOp2b59u/r166eIiAglJiZq+vTptfqyePFiderUSREREerevbuWLVvWKNsMAMCZ8mtIHzlyRD179tScOXNqtf3www/asmWLHnnkEW3ZskVvvPGGSkpKdMMNN9Sqffzxx7V//35rGjt2rNXmdrs1aNAgXXTRRSoqKtKMGTOUk5Oj559/3qrZsGGDhg8froyMDG3dulVDhw7V0KFDtXPnzsbZcAAAzkCYP1c+ZMgQDRky5KRt0dHRys/P95n37LPPqm/fvtq3b5/atWtnzW/RooXi4+NPupwFCxaosrJSL730kpxOp7p27ari4mI9+eSTGj16tCRp9uzZGjx4sCZMmCBJeuKJJ5Sfn69nn31W8+bNa4hNbTQ1Tx0LDQ31d1cAAA0soC5olpeXy+FwqGXLlj7zp02bptatW+vyyy/XjBkzVF1dbbUVFhaqf//+cjqd1ry0tDSVlJTo0KFDVk1qaqrPMtPS0lRYWHjKvlRUVMjtdvtMAAA0JL8eSZ+NY8eOadKkSRo+fLiioqKs+X/4wx/Uu3dvtWrVShs2bNDkyZO1f/9+Pfnkk5Ikl8ulpKQkn2XFxcVZbTExMXK5XNa8E2tcLtcp+5Obm6vHHnusoTYPAIBaAiKkq6qq9Lvf/U7GGM2dO9enLTs72/p7jx495HQ6de+99yo3N1fh4eGN1qfJkyf7rNvtdisxMbHR1gcAaHpsH9I1Af3ll19q1apVPkfRJ5OcnKzq6mrt3btXHTt2VHx8vEpLS31qaj7XXMc+Vc2prnNLUnh4eKP+JwAAAFtfk64J6E8//VQrV65U69atf/JniouLFRISotjYWElSSkqK1q1bp6qqKqsmPz9fHTt2VExMjFVTUFDgs5z8/HylpKQ04NYAAHB2/HokffjwYX322WfW5z179qi4uFitWrVS27Zt9Zvf/EZbtmzR0qVL5fF4rGvErVq1ktPpVGFhoTZt2qQBAwaoRYsWKiws1Pjx43XbbbdZAXzrrbfqscceU0ZGhiZNmqSdO3dq9uzZmjVrlrXe+++/X9dcc41mzpyp9PR0LVy4UJs3b/YZpgUAwLnmMH588POaNWs0YMCAWvNHjRqlnJycWjd81Vi9erWuvfZabdmyRb///e+1e/duVVRUKCkpSSNHjlR2drbPqejt27crMzNTH330kdq0aaOxY8dq0qRJPstcvHixpkyZor179+rSSy/V9OnTdd11153xtrjdbkVHR6u8vPwnT8mfqfLyco2ct1rGHB9q5XCEyBivz5+h4c312n39FRkZ2SDrBADYh19DOpgQ0gCAhmbra9IAADRlhDQAADZVp5C++OKL9d1339WaX1ZWposvvrjenQIAAHUM6b1798rj8dSaX1FRoX//+9/17hQAADjLIVhvv/229fcVK1YoOjra+uzxeFRQUKD27ds3WOcAAGjKziqkhw4dKklyOBwaNWqUT1uzZs3Uvn17zZw5s8E6BwBAU3ZWIe31eiVJSUlJ1phjAADQOOr0xLE9e/Y0dD8AAMCP1PmxoAUFBSooKNCBAwesI+waL730Ur07BgBAU1enkH7sscf0+OOP64orrlDbtm3lcDgaul8AADR5dQrpefPmKS8vTyNHjmzo/gAAgP+o0zjpyspKXX311Q3dF9SBMUYej0c8gh0Agk+dQvruu+/Wq6++2tB9QV0Yr0a99GGt+wIAAIGvTqe7jx07pueff14rV65Ujx491KxZM5/2J598skE6h59mvB6JewIAICjVKaS3b9+uXr16SZJ27tzp08ZNZAAANIw6hfTq1asbuh+oB+P1yOPxKDQ01N9dAQA0IF5VGeCM1yPuGQOA4FSnI+kBAwac9rT2qlWr6twhAABwXJ1CuuZ6dI2qqioVFxdr586dtV68AQAA6qZOIT1r1qyTzs/JydHhw4fr1SEAAHBcg16Tvu2223huNwAADaRBQ7qwsFARERENuUgAAJqsOp3uvummm3w+G2O0f/9+bd68WY888kiDdAwAgKauTiEdHR3t8zkkJEQdO3bU448/rkGDBjVIx6D/PJPb370AAPhLnUJ6/vz5Dd0PAADwI3UK6RpFRUX65JNPJEldu3bV5Zdf3iCdwtmpeeIYACC41CmkDxw4oFtuuUVr1qxRy5YtJUllZWUaMGCAFi5cqAsuuKAh+wgAQJNUp7u7x44dq++//167du3SwYMHdfDgQe3cuVNut1t/+MMfGrqPAAA0SXU6kl6+fLlWrlypzp07W/O6dOmiOXPmcOOYH3C6GwCCU52OpL1eb613SEtSs2bN5PV6690pAABQx5D+5S9/qfvvv1/ffPONNe/f//63xo8fr4EDBzZY5wAAaMrqFNLPPvus3G632rdvr0suuUSXXHKJkpKS5Ha79cwzzzR0HwEAaJLqdE06MTFRW7Zs0cqVK7V7925JUufOnZWamtqgnQMAoCk7qyPpVatWqUuXLnK73XI4HPrVr36lsWPHauzYsbryyivVtWtXvf/++43VVwAAmpSzCumnnnpK99xzj6Kiomq1RUdH695779WTTz7ZYJ3DmeHubgAITmcV0tu2bdPgwYNP2T5o0CAVFRXVu1MAAOAsQ7q0tPSkQ69qhIWF6dtvv613p3B2OJIGgOB0ViH9s5/9TDt37jxl+/bt29W2bdt6dwpnxxjznzdm8cosAAgmZxXS1113nR555BEdO3asVtvRo0f16KOP6r/+678arHM4Q8ar0a9s40EyABBkziqkp0yZooMHD+qyyy7T9OnT9dZbb+mtt97SX/7yF3Xs2FEHDx7Uww8/fMbLW7duna6//nolJCTI4XBoyZIlPu3GGE2dOlVt27ZVZGSkUlNT9emnn/rUHDx4UCNGjFBUVJRatmypjIwMHT582Kdm+/bt6tevnyIiIpSYmKjp06fX6svixYvVqVMnRUREqHv37lq2bNmZ7xgbcITUacg7AMDGzuqbPS4uThs2bFC3bt00efJk3Xjjjbrxxhv10EMPqVu3blq/fr3i4uLOeHlHjhxRz549NWfOnJO2T58+XU8//bTmzZunTZs26bzzzlNaWprPkfyIESO0a9cu5efna+nSpVq3bp1Gjx5ttbvdbg0aNEgXXXSRioqKNGPGDOXk5Oj555+3ajZs2KDhw4crIyNDW7du1dChQzV06NDTntoHAKCxOUwdL2QeOnRIn332mYwxuvTSSxUTE1O/jjgcevPNNzV06FBJx4+iExIS9MADD+iPf/yjJKm8vFxxcXHKy8vTLbfcok8++URdunTRRx99pCuuuELS8Zd/XHfddfr666+VkJCguXPn6uGHH5bL5ZLT6ZQkPfjgg1qyZIn1IJabb75ZR44c0dKlS63+XHXVVerVq5fmzZt3Rv13u92Kjo5WeXn5SYeo1cXBgwc16vl1x/eH1yOHI0TGeH3+lMMh4/XIeV6MFmddo9DQ0AZZNwDA/+p8jjQmJkZXXnml+vbtW++APpk9e/bI5XL5PMUsOjpaycnJKiwslCQVFhaqZcuWVkBLUmpqqkJCQrRp0yarpn///lZAS1JaWppKSkp06NAhq+bHT0tLS0uz1nMyFRUVcrvdPhMAAA3JthcyXS6XJNU6fR4XF2e1uVwuxcbG+rSHhYWpVatWPjUnW8aJ6zhVTU37yeTm5io6OtqaEhMTz3YTAQA4LduGtN1NnjxZ5eXl1vTVV1/5u0sAgCBj25COj4+XdPwBKicqLS212uLj43XgwAGf9urqah08eNCn5mTLOHEdp6qpaT+Z8PBwRUVF+UwAADQk24Z0UlKS4uPjVVBQYM1zu93atGmTUlJSJEkpKSkqKyvzeRTpqlWr5PV6lZycbNWsW7dOVVVVVk1+fr46duxoXUtPSUnxWU9NTc16AADwB7+G9OHDh1VcXKzi4mJJx28WKy4u1r59++RwODRu3Dj96U9/0ttvv60dO3bo9ttvV0JCgnUHeOfOnTV48GDdc889+vDDD/XBBx8oKytLt9xyixISEiRJt956q5xOpzIyMrRr1y4tWrRIs2fPVnZ2ttWP+++/X8uXL9fMmTO1e/du5eTkaPPmzcrKyjrXu6TOeDQoAASfOr1PuqFs3rxZAwYMsD7XBOeoUaOUl5eniRMn6siRIxo9erTKysr0i1/8QsuXL1dERIT1MwsWLFBWVpYGDhyokJAQDRs2TE8//bTVHh0drffee0+ZmZnq06eP2rRpo6lTp/qMpb766qv16quvasqUKXrooYd06aWXasmSJerWrds52AsAAJxcncdJw5e/x0k3i4zSosxrfIaaAQACm22vSQMA0NQR0gAA2BQhbVM1r5+UzuxqBDeOAUDwIaRtyuv1KuPF9880owEAQYiQtjFePwkATRspAACATRHSAADYFCENAIBNEdJBgru7ASD4ENIAANgUIQ0AgE0R0kGi5uEnPIodAIIHIR0sjFd35m2W1+v1d08AAA2EkA4iPPwEAIIL3+oAANgUIQ0AgE0R0kHCeD3yehgrDQDBhJAGAMCmCGkAAGyKkA4iPBoUAIILIR1ECGkACC6ENAAANkVIAwBgU4Q0AAA2RUgHEa5JA0BwIaQBALApQhoAAJsipAEAsClCGgAAmyKkAQCwKULapjwej4zxdy8AAP5ESAcRhmABQHAhpAEAsClCOogYY/5zmpzz5AAQDAjpYGK8Gv3KNnm9Xn/3BADQAAjpIOMI4Z8UAIIF3+hBhpvHACB4ENIAANgUIQ0AgE3ZPqTbt28vh8NRa8rMzJQkXXvttbXaxowZ47OMffv2KT09Xc2bN1dsbKwmTJig6upqn5o1a9aod+/eCg8PV4cOHZSXl3euNhEAgJMK83cHfspHH33kc411586d+tWvfqXf/va31rx77rlHjz/+uPW5efPm1t89Ho/S09MVHx+vDRs2aP/+/br99tvVrFkz/fnPf5Yk7dmzR+np6RozZowWLFiggoIC3X333Wrbtq3S0tLOwVYCAFCb7UP6ggsu8Pk8bdo0XXLJJbrmmmusec2bN1d8fPxJf/69997Txx9/rJUrVyouLk69evXSE088oUmTJiknJ0dOp1Pz5s1TUlKSZs6cKUnq3Lmz1q9fr1mzZhHSAAC/sf3p7hNVVlbqlVde0V133SWHw2HNX7Bggdq0aaNu3bpp8uTJ+uGHH6y2wsJCde/eXXFxcda8tLQ0ud1u7dq1y6pJTU31WVdaWpoKCwtP2ZeKigq53W6fCQCAhmT7I+kTLVmyRGVlZbrjjjusebfeeqsuuugiJSQkaPv27Zo0aZJKSkr0xhtvSJJcLpdPQEuyPrtcrtPWuN1uHT16VJGRkbX6kpubq8cee6whNw8AAB8BFdIvvviihgwZooSEBGve6NGjrb93795dbdu21cCBA/X555/rkksuabS+TJ48WdnZ2dZnt9utxMTERlvfmWKcNAAEj4AJ6S+//FIrV660jpBPJTk5WZL02Wef6ZJLLlF8fLw+/PBDn5rS0lJJsq5jx8fHW/NOrImKijrpUbQkhYeHKzw8vE7bAgDAmQiYa9Lz589XbGys0tPTT1tXXFwsSWrbtq0kKSUlRTt27NCBAwesmvz8fEVFRalLly5WTUFBgc9y8vPzlZKS0oBbAADA2QmIkPZ6vZo/f75GjRqlsLD/O/j//PPP9cQTT6ioqEh79+7V22+/rdtvv139+/dXjx49JEmDBg1Sly5dNHLkSG3btk0rVqzQlClTlJmZaR0JjxkzRl988YUmTpyo3bt367nnntPrr7+u8ePH+2V764PT3QAQPAIipFeuXKl9+/bprrvu8pnvdDq1cuVKDRo0SJ06ddIDDzygYcOG6Z133rFqQkNDtXTpUoWGhiolJUW33Xabbr/9dp9x1UlJSXr33XeVn5+vnj17aubMmXrhhRcCcvgVIQ0AwcNhePlwg3C73YqOjlZ5ebmioqLqvbzKykr9ZtYyGa+RHA4Zr0cOR4iM8fr8WastJEwvj+6nmJgYn2FqAIDAExBH0jgLxquMv23lndIAEAQI6SDEO6UBIDjwbQ4AgE0R0gAA2BQhHYS4wxsAggMhHYQIaQAIDoQ0AAA2RUgHIY6kASA4ENIAANgUIQ0AgE0R0jbl8XgkntgKAE0aIQ0AgE0R0kGIG8cAIDgQ0kGIkAaA4EBIAwBgU4R0EDLGyOPxiFeFA0BgI6SDkfFq9CvbeKc0AAQ4QjpI8U5pAAh8fJMDAGBThDQAADZFSAcphmEBQOAjpIMUIQ0AgY+QBgDApghpAABsipC2qeOnquv+MBJOdwNA4COkgxQhDQCBj5AGAMCmCOkgxfO7ASDwEdLByniV8betPL8bAAIYIR3EeH43AAQ2vsUBALApQjqIcYc3AAQ2QhoAAJsipIMYR9IAENgIaQAAbIqQBgDApghpAABsipAOYlyTBoDARkgHMUIaAAKbrUM6JydHDofDZ+rUqZPVfuzYMWVmZqp169Y6//zzNWzYMJWWlvosY9++fUpPT1fz5s0VGxurCRMmqLq62qdmzZo16t27t8LDw9WhQwfl5eWdi80DAOC0bB3SktS1a1ft37/fmtavX2+1jR8/Xu+8844WL16stWvX6ptvvtFNN91ktXs8HqWnp6uyslIbNmzQyy+/rLy8PE2dOtWq2bNnj9LT0zVgwAAVFxdr3Lhxuvvuu7VixYpzup0AAPxYmL878FPCwsIUHx9fa355eblefPFFvfrqq/rlL38pSZo/f746d+6sjRs36qqrrtJ7772njz/+WCtXrlRcXJx69eqlJ554QpMmTVJOTo6cTqfmzZunpKQkzZw5U5LUuXNnrV+/XrNmzVJaWto53daGZrweVVZWyuPxKDQ01N/dAQCcJdsfSX/66adKSEjQxRdfrBEjRmjfvn2SpKKiIlVVVSk1NdWq7dSpk9q1a6fCwkJJUmFhobp37664uDirJi0tTW63W7t27bJqTlxGTU3NMk6loqJCbrfbZ7IbXlcJAIHN1iGdnJysvLw8LV++XHPnztWePXvUr18/ff/993K5XHI6nWrZsqXPz8TFxcnlckmSXC6XT0DXtNe0na7G7Xbr6NGjp+xbbm6uoqOjrSkxMbG+m9vwjFdjFmzjdZUAEKBsfbp7yJAh1t979Oih5ORkXXTRRXr99dcVGRnpx55JkydPVnZ2tvXZ7XbbMqh5XSUABK6A+gZv2bKlLrvsMn322WeKj49XZWWlysrKfGpKS0uta9jx8fG17vau+fxTNVFRUaf9j0B4eLiioqJ8JgAAGlJAhfThw4f1+eefq23bturTp4+aNWumgoICq72kpET79u1TSkqKJCklJUU7duzQgQMHrJr8/HxFRUWpS5cuVs2Jy6ipqVmGvzTUtWTGSgNA4LJ1SP/xj3/U2rVrtXfvXm3YsEE33nijQkNDNXz4cEVHRysjI0PZ2dlavXq1ioqKdOeddyolJUVXXXWVJGnQoEHq0qWLRo4cqW3btmnFihWaMmWKMjMzFR4eLkkaM2aMvvjiC02cOFG7d+/Wc889p9dff13jx4/356YDAGDva9Jff/21hg8fru+++04XXHCBfvGLX2jjxo264IILJEmzZs1SSEiIhg0bpoqKCqWlpem5556zfj40NFRLly7Vfffdp5SUFJ133nkaNWqUHn/8casmKSlJ7777rsaPH6/Zs2frwgsv1AsvvBDww68AAIHPYRif0yDcbreio6NVXl7eINeny8vLddtzKyWFSA6HjNcjhyNExnh9/jxdmzFehTqba8GYfjrvvPPkcDjqv6EAgHPG1qe7UX/eqmMa+cImhmEBQAAipJsAhmEBQGDi27sJ8FZXqrKy0t/dAACcJUIaAACbIqQBALApQroJ4IEmABCYCGkAAGyKkG4iPB6OpgEg0BDSTQCnuwEgMBHSAADYFCHdBBhjGuytWgCAc4eQbgqMV/f8fSuPBgWAAENINxE8GhQAAg/f3AAA2BQh3URwhzcABB5CGgAAmyKkAQCwKUK6ieB1lQAQeAhpG6oZ19yQvJ5qVVZWMlYaAAIIIW1DXq9Xd77wvho0T41X9/xti6qqqhpwoQCAxkRI21RjjGtmrDQABBa+tZsQhmEBQGAhpJswXl8JAPZGSDchHEkDQGAhpJsQ7vAGgMBCSDclxquMv/E2LAAIFIR0E8Md3gAQOPjGbmK4Lg0AgYOQbmIIaQAIHIR0E0NIA0DgIKSbGO7wBoDAQUg3McZTpYyXt3CHNwAEAEK6SWr4t2wBABoeIQ0AgE0R0k0QN48BQGAgpJsgb3WlKisr/d0NAMBPIKSbIGOOX5PmDm8AsDdCuin6zzO8a15VSVgDgD0R0k2UIyREXq9Xw/97A8OxAMCmbB3Subm5uvLKK9WiRQvFxsZq6NChKikp8am59tpr5XA4fKYxY8b41Ozbt0/p6elq3ry5YmNjNWHCBFVXV/vUrFmzRr1791Z4eLg6dOigvLy8xt68Uzp+dNu466i5eYwXbgCAfdn6G3rt2rXKzMzUxo0blZ+fr6qqKg0aNEhHjhzxqbvnnnu0f/9+a5o+fbrV5vF4lJ6ersrKSm3YsEEvv/yy8vLyNHXqVKtmz549Sk9P14ABA1RcXKxx48bp7rvv1ooVK87ZtgIA8GNh/u7A6Sxfvtznc15enmJjY1VUVKT+/ftb85s3b674+PiTLuO9997Txx9/rJUrVyouLk69evXSE088oUmTJiknJ0dOp1Pz5s1TUlKSZs6cKUnq3Lmz1q9fr1mzZiktLa3xNtCPGIYFAPZn6yPpHysvL5cktWrVymf+ggUL1KZNG3Xr1k2TJ0/WDz/8YLUVFhaqe/fuiouLs+alpaXJ7XZr165dVk1qaqrPMtPS0lRYWHjKvlRUVMjtdvtMAAA0JFsfSZ/I6/Vq3Lhx+vnPf65u3bpZ82+99VZddNFFSkhI0Pbt2zVp0iSVlJTojTfekCS5XC6fgJZkfXa5XKetcbvdOnr0qCIjI2v1Jzc3V4899liDbuO5xFhpALC/gAnpzMxM7dy5U+vXr/eZP3r0aOvv3bt3V9u2bTVw4EB9/vnnuuSSSxqtP5MnT1Z2drb12e12KzExsdHW19C8nmodPXpUjX6HGgCgzgLidHdWVpaWLl2q1atX68ILLzxtbXJysiTps88+kyTFx8ertLTUp6bmc8117FPVREVFnfQoWpLCw8MVFRXlMwUU41Xmgi3yeqq4Ng0ANmXrkDbGKCsrS2+++aZWrVqlpKSkn/yZ4uJiSVLbtm0lSSkpKdqxY4cOHDhg1eTn5ysqKkpdunSxagoKCnyWk5+fr5SUlAbaEnti+BUA2Jutv6UzMzP1yiuv6NVXX1WLFi3kcrnkcrmOn6aV9Pnnn+uJJ55QUVGR9u7dq7ffflu33367+vfvrx49ekiSBg0apC5dumjkyJHatm2bVqxYoSlTpigzM1Ph4eGSpDFjxuiLL77QxIkTtXv3bj333HN6/fXXNX78eL9tOwAAtg7puXPnqry8XNdee63atm1rTYsWLZIkOZ1OrVy5UoMGDVKnTp30wAMPaNiwYXrnnXesZYSGhmrp0qUKDQ1VSkqKbrvtNt1+++16/PHHrZqkpCS9++67ys/PV8+ePTVz5ky98MILQTv8qobxNv5DUwAAdecwPLi5QbjdbkVHR6u8vLze16crKyv129kr5KmqlMMRIjkcMl6PHI4QGeP1+fN0bWdSExbZQot+319Op7OB9gQAoKHY+kgajY+HmgCAfRHSTRzjpQHAvghpG/J4PIxfBgAQ0k2d11OtyspK3ikNADZESDd1xqu78z7SsWPH/N0TAMCPENKQZLh5DABsiJAGp7wBwKYIaUjGq4y/bZXX6/V3TwAAJyCkIUnyVldYj1sFANgDIQ1Jx19mUllZqerqan93BQDwH4Q0JEnGU6W78jZzyhsAbISQhoVT3gBgL4S0DR0fDnXu77TmLm8AsBdCGv/HeDX6lW2c8gYAmyCk8SM82AQA7IKQhg9P1fHr0pzyBgD/I6Thw3iqdPv/bOBZ3gBgA4S0DXk8Hr8eyRrj5QYyALABQhq1eKuOadQLG1VVVeXvrgBAk0ZI46S8niquTQOAnxHSOCnjqdId83kCGQD4EyGNU+IJZADgX4Q0TslTVaHDhw/z0g0A8BNC2oaO393t715IMl7d+0qxqqqquDYNAH5ASNuMMcZvz+4+GeOp1K3z3udObwDwA0LaZrxerzJefN8uGS3j9ai64gcdPnzY310BgCaHkLYhR4i9/lmMMaqsrOTaNACcY/ZKA9iT8erulwr17bff6ocffuD6NACcI4Q0zojDIY1++UPd9j+FjJ0GgHOEkMZZMPJU8yQyADhXCGmcFU/FEd363Fq53W6uUQNAIyOkcdaMp1K3PrtShw4d4ogaABoRIW0ztnmQyU9wOKS7XizUoUOHOKIGgEZCSKPujEcj565WaWmpKioq/N0bAAg6hDTqxeGQ7n35Q5WXl6uiooLT3wDQgML83QEEPodDuuuF9ZIjRC+P7qfo6GiFhobK4XD4u2sAENAIaTQIR0iIjNejUfPWSCGheinjakVFRSksLIzABoA6IqTRoBwhITLGo7v+533J4VBIswgtGNNPERERcjgcBDYAnAVCGo3ieFh7JW+1hj+9XGERUXKEhupvGclyOp1yOp2ENQD8BG4c+5E5c+aoffv2ioiIUHJysj788EN/dymgGa9HcoTIU/mDqo8d1i2zl+s3s1bo4MGDOnLkiI4dO6aqqipVVVWpurqaG88A4AQcSZ9g0aJFys7O1rx585ScnKynnnpKaWlpKikpUWxsrL+7FxQcISGS8ej2uavkcBw/2g4JbSbJK0dYhObflayIiAg5nU7fn/vRqXJjjLxer0JCQjgiBxC0HIZDF0tycrKuvPJKPfvss5KOv9s5MTFRY8eO1YMPPnjan3W73YqOjlZ5ebmioqLq3IfKykr9ZtYyGa+RMV45HCGSwyHj9VihduKfp2s7lzUNtQ5jTK0aORxyhDbTi3cm67zzzlNoaKi8Xq9u+58NeuWeq9WsWbMz2rdcEwcQaDiS/o/KykoVFRVp8uTJ1ryQkBClpqaqsLCwVn1FRYXPAzzKy8slHQ/r+vaj8ofDTTakT1cz4ql35QhxSHLIeL0KCQ3Tr/+0SI4Qh4zX+LTVzAsJC5PxVCukWbhCw5pp3qirFBkZqdDQ0Hr9OwFouhry+6NFixanPXAgpP/jf//3f+XxeBQXF+czPy4uTrt3765Vn5ubq8cee6zW/MTExEbrI+qv3SP+7gEA/J+fOvtKSNfR5MmTlZ2dbX32er06ePCgWrduXa/TqW63W4mJifrqq6/qddq8qWL/1R/7sH7Yf/XT1PZfixYtTttOSP9HmzZtFBoaqtLSUp/5paWlio+Pr1UfHh6u8PBwn3ktW7ZssP5ERUU1iV/QxsL+qz/2Yf2w/+qH/XccQ7D+w+l0qk+fPiooKLDmeb1eFRQUKCUlxY89AwA0VRxJnyA7O1ujRo3SFVdcob59++qpp57SkSNHdOedd/q7awCAJoiQPsHNN9+sb7/9VlOnTpXL5VKvXr20fPnyWjeTNabw8HA9+uijtU6l48yw/+qPfVg/7L/6Yf/5Ypw0AAA2xTVpAABsipAGAMCmCGkAAGyKkAYAwKYIaZtpiq/KXLduna6//nolJCTI4XBoyZIlPu3GGE2dOlVt27ZVZGSkUlNT9emnn/rUHDx4UCNGjFBUVJRatmypjIwMHT582Kdm+/bt6tevnyIiIpSYmKjp06fX6svixYvVqVMnRUREqHv37lq2bFmDb29Dy83N1ZVXXqkWLVooNjZWQ4cOVUlJiU/NsWPHlJmZqdatW+v888/XsGHDaj24Z9++fUpPT1fz5s0VGxurCRMmqLq62qdmzZo16t27t8LDw9WhQwfl5eXV6k+g/Q7PnTtXPXr0sB6ekZKSon/+859WO/vu7EybNk0Oh0Pjxo2z5rEP68HANhYuXGicTqd56aWXzK5du8w999xjWrZsaUpLS/3dtUa1bNky8/DDD5s33njDSDJvvvmmT/u0adNMdHS0WbJkidm2bZu54YYbTFJSkjl69KhVM3jwYNOzZ0+zceNG8/7775sOHTqY4cOHW+3l5eUmLi7OjBgxwuzcudO89tprJjIy0vz3f/+3VfPBBx+Y0NBQM336dPPxxx+bKVOmmGbNmpkdO3Y0+j6oj7S0NDN//nyzc+dOU1xcbK677jrTrl07c/jwYatmzJgxJjEx0RQUFJjNmzebq666ylx99dVWe3V1tenWrZtJTU01W7duNcuWLTNt2rQxkydPtmq++OIL07x5c5OdnW0+/vhj88wzz5jQ0FCzfPlyqyYQf4fffvtt8+6775p//etfpqSkxDz00EOmWbNmZufOncYY9t3Z+PDDD0379u1Njx49zP3332/NZx/WHSFtI3379jWZmZnWZ4/HYxISEkxubq4fe3Vu/TikvV6viY+PNzNmzLDmlZWVmfDwcPPaa68ZY4z5+OOPjSTz0UcfWTX//Oc/jcPhMP/+97+NMcY899xzJiYmxlRUVFg1kyZNMh07drQ+/+53vzPp6ek+/UlOTjb33ntvg25jYztw4ICRZNauXWuMOb6/mjVrZhYvXmzVfPLJJ0aSKSwsNMYc/49SSEiIcblcVs3cuXNNVFSUtc8mTpxounbt6rOum2++2aSlpVmfg+V3OCYmxrzwwgvsu7Pw/fffm0svvdTk5+eba665xgpp9mH9cLrbJmpelZmammrNO92rMpuKPXv2yOVy+eyX6OhoJScnW/ulsLBQLVu21BVXXGHVpKamKiQkRJs2bbJq+vfvL6fTadWkpaWppKREhw4dsmpOXE9NTaDt/5rXprZq1UqSVFRUpKqqKp9t69Spk9q1a+ezD7t37+7z4J60tDS53W7t2rXLqjnd/gmG32GPx6OFCxfqyJEjSklJYd+dhczMTKWnp9faTvZh/fDEMZs421dlNhUul0uSTrpfatpcLpdiY2N92sPCwtSqVSufmqSkpFrLqGmLiYmRy+U67XoCgdfr1bhx4/Tzn/9c3bp1k3R8+5xOZ60XwPx4H55s22vaTlfjdrt19OhRHTp0KGB/h3fs2KGUlBQdO3ZM559/vt5880116dJFxcXF7LszsHDhQm3ZskUfffRRrTZ+/+qHkAaCSGZmpnbu3Kn169f7uysBpWPHjiouLlZ5ebn+8Y9/aNSoUVq7dq2/uxUQvvrqK91///3Kz89XRESEv7sTdDjdbRNn+6rMpqJm20+3X+Lj43XgwAGf9urqah08eNCn5mTLOHEdp6oJlP2flZWlpUuXavXq1brwwgut+fHx8aqsrFRZWZlP/Y/3YV33T1RUlCIjIwP6d9jpdKpDhw7q06ePcnNz1bNnT82ePZt9dwaKiop04MAB9e7dW2FhYQoLC9PatWv19NNPKywsTHFxcezDeiCkbYJXZZ5cUlKS4uPjffaL2+3Wpk2brP2SkpKisrIyFRUVWTWrVq2S1+tVcnKyVbNu3TpVVVVZNfn5+erYsaNiYmKsmhPXU1Nj9/1vjFFWVpbefPNNrVq1qtZp/T59+qhZs2Y+21ZSUqJ9+/b57MMdO3b4/GcnPz9fUVFR6tKli1Vzuv0TTL/DXq9XFRUV7LszMHDgQO3YsUPFxcXWdMUVV2jEiBHW39mH9eDvO9fwfxYuXGjCw8NNXl6e+fjjj83o0aNNy5Ytfe54DEbff/+92bp1q9m6dauRZJ588kmzdetW8+WXXxpjjg/BatmypXnrrbfM9u3bza9//euTDsG6/PLLzaZNm8z69evNpZde6jMEq6yszMTFxZmRI0eanTt3moULF5rmzZvXGoIVFhZm/vrXv5pPPvnEPProowExBOu+++4z0dHRZs2aNWb//v3W9MMPP1g1Y8aMMe3atTOrVq0ymzdvNikpKSYlJcVqrxkCM2jQIFNcXGyWL19uLrjggpMOgZkwYYL55JNPzJw5c046BCbQfocffPBBs3btWrNnzx6zfft28+CDDxqHw2Hee+89Ywz7ri5OvLvbGPZhfRDSNvPMM8+Ydu3aGafTafr27Ws2btzo7y41utWrVxtJtaZRo0YZY44Pw3rkkUdMXFycCQ8PNwMHDjQlJSU+y/juu+/M8OHDzfnnn2+ioqLMnXfeab7//nufmm3btplf/OIXJjw83PzsZz8z06ZNq9WX119/3Vx22WXG6XSarl27mnfffbfRtruhnGzfSTLz58+3ao4ePWp+//vfm5iYGNO8eXNz4403mv379/ssZ+/evWbIkCEmMjLStGnTxjzwwAOmqqrKp2b16tWmV69exul0mosvvthnHTUC7Xf4rrvuMhdddJFxOp3mggsuMAMHDrQC2hj2XV38OKTZh3XHqyoBALAprkkDAGBThDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA2RUgDAGBThDQAADZFSAMAYFOENAAANkVIAwBgU4Q0AAA29f8Du5SeWZCUUpoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAekAAAIACAYAAACmbZRAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0u0lEQVR4nO3de3RU5b3/8c/kMkMozASEJFABQ6lgEEFRIVWplhwCRiuFLgERqOAFDRwRf2hpKV561sFFvSNFrcXYpRTUJdaCIiHcVMLF1AgBzFGLB6tOQsXMBEpmkpnn9wfNPgzBCyFkniTv11p7LWY/3+z57m3aT/bdZYwxAgAA1kmIdwMAAODECGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAM2qoKBALpdLn3zySbxbAaxHSAMAYClCGgAASxHSAABYipAG8I1efvlluVwubdq0qcHYU089JZfLpbKyMu3cuVO/+MUv1Lt3b7Vr104ZGRmaOnWqvvzyyzh0DbQOhDSAb5SXl6cOHTroxRdfbDC2YsUK9e/fX+eee64KCwv197//XTfccIMWLVqk8ePHa/ny5bryyivFG3GBxkmKdwMA7JaSkqKrr75aL7/8sh5//HElJiZKkvx+vzZt2qR7771XknTbbbfpzjvvjPnZoUOHasKECXr77bd12WWXNXfrQIvHnjSAbzVu3DhVVlZq48aNzryXX35Z0WhU48aNk3Q0zOvV1NTon//8p4YOHSpJ+tvf/tas/QKtBSEN4FuNHDlSPp9PK1ascOatWLFCgwYN0tlnny1JOnjwoG6//Xalp6crJSVFXbt2VWZmpiQpEAjEpW+gpSOkAXwrj8ej0aNHa+XKlaqrq9Nnn32md955x9mLlqRrr71Wf/jDHzR9+nS98sorWrt2rdasWSNJikaj8WodaNE4Jw3gOxk3bpyee+45FRUVae/evTLGOCH91VdfqaioSPfdd5/mz5/v/MyHH34Yr3aBVoGQBvCd5OTkqHPnzlqxYoX27t2riy++2DmcXX8x2fFXcT/66KPN3SbQqhDSAL6T5ORkjRkzRsuXL9fhw4f14IMPOmNer1fDhg3TwoULVVtbq+9///tau3at9u3bF8eOgZaPc9IAvrNx48bp0KFDko6egz7WsmXLlJubq8WLF2vu3LlKTk7WG2+8EY82gVbDZXjKAAAAVmJPGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQjp78AYo2AwyDtxAQDNipD+Dqqrq+Xz+VRdXR3vVgAAbQghDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWimtIL1myROedd568Xq+8Xq+ys7NjXhJfU1Oj/Px8nXHGGerQoYPGjh2rioqKmGXs379feXl5at++vdLS0jRnzhzV1dXF1GzcuFEXXHCBPB6P+vTpo4KCguZYPQAATklcQ/rMM8/UAw88oJKSEr377rv6yU9+omuuuUa7d++WJN1xxx3661//qpdeekmbNm3S559/rjFjxjg/H4lElJeXp3A4rC1btui5555TQUGB5s+f79Ts27dPeXl5uuKKK1RaWqpZs2bpxhtv1Jtvvtns6wsAwEkxlunUqZN55plnTFVVlUlOTjYvvfSSM7Z3714jyRQXFxtjjHn99ddNQkKC8fv9Ts2SJUuM1+s1oVDIGGPMXXfdZfr37x/zHePGjTO5ubnfuadAIGAkmUAgcCqrBgDASbHmnHQkEtHy5ct1+PBhZWdnq6SkRLW1tcrJyXFq+vXrp549e6q4uFiSVFxcrAEDBig9Pd2pyc3NVTAYdPbGi4uLY5ZRX1O/jBMJhUIKBoMxEwAAzS3uIb1r1y516NBBHo9H06dP18qVK5WVlSW/3y+3263U1NSY+vT0dPn9fkmS3++PCej68fqxb6oJBoM6cuTICXtasGCBfD6fM/Xo0aMpVhUAgJMS95Du27evSktLtW3bNt16662aMmWK9uzZE9ee5s6dq0Ag4EyffvppXPsBALRNSfFuwO12q0+fPpKkwYMHa8eOHXrsscc0btw4hcNhVVVVxexNV1RUKCMjQ5KUkZGh7du3xyyv/urvY2uOvyK8oqJCXq9XKSkpJ+zJ4/HI4/E0yfoBANBYcd+TPl40GlUoFNLgwYOVnJysoqIiZ6y8vFz79+9Xdna2JCk7O1u7du1SZWWlU1NYWCiv16usrCyn5thl1NfULwMAAFvFdU967ty5GjVqlHr27Knq6motW7ZMGzdu1Jtvvimfz6dp06Zp9uzZ6ty5s7xer2bOnKns7GwNHTpUkjRixAhlZWVp0qRJWrhwofx+v+bNm6f8/HxnT3j69Ol64okndNddd2nq1Klav369XnzxRa1evTqeqw4AwLeL56XlU6dONb169TJut9t07drVDB8+3Kxdu9YZP3LkiLnttttMp06dTPv27c3PfvYz88UXX8Qs45NPPjGjRo0yKSkppkuXLubOO+80tbW1MTUbNmwwgwYNMm632/Tu3ds8++yzJ9Unt2ABAOLBZYwx8f5DwXbBYFA+n0+BQEBerzfe7QAA2gjrzkkDAICjCGkLGGMUCoXEQQ0AwLEIaQuEw2H9/NE1qq6ujncrAACLENKWSExKjncLAADLENIAAFiKkAYAwFKENAAAliKk46z+ym4AAI5HSMdZOBzWxCUbFI1y+xUAIBYhbQGu7AYAnAghDQCApeL+PmlIkbpa8fcSAOB4JIMleDQoAOB4hLQlopE6TVu6VeFwON6tAAAsQUhbJCHJHe8WAAAWIaQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSFskUhdWKBSKdxsAAEsQ0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIWMcYoFArJGBPvVgAAFiCkLRKN1Gna0q0Kh8PxbgUAYAFC2jIJSe54twAAsAQhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdJxFgqFFI3yakoAQEOENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsFRcQ3rBggW66KKL1LFjR6WlpWn06NEqLy+Pqbn88svlcrlipunTp8fU7N+/X3l5eWrfvr3S0tI0Z84c1dXVxdRs3LhRF1xwgTwej/r06aOCgoLTvXoAAJySuIb0pk2blJ+fr61bt6qwsFC1tbUaMWKEDh8+HFN300036YsvvnCmhQsXOmORSER5eXkKh8PasmWLnnvuORUUFGj+/PlOzb59+5SXl6crrrhCpaWlmjVrlm688Ua9+eabzbauAACcrKR4fvmaNWtiPhcUFCgtLU0lJSUaNmyYM799+/bKyMg44TLWrl2rPXv2aN26dUpPT9egQYP029/+Vnfffbfuvfdeud1uPfnkk8rMzNRDDz0kSTrnnHP09ttv65FHHlFubu7pW8FGMMYoFArJ7XbL5XLFux0AQBxZdU46EAhIkjp37hwz/4UXXlCXLl107rnnau7cufrXv/7ljBUXF2vAgAFKT0935uXm5ioYDGr37t1OTU5OTswyc3NzVVxcfMI+QqGQgsFgzNRcopFaTXrqLYXD4Wb7TgCAneK6J32saDSqWbNm6ZJLLtG5557rzL/uuuvUq1cvde/eXTt37tTdd9+t8vJyvfLKK5Ikv98fE9CSnM9+v/8ba4LBoI4cOaKUlJSYsQULFui+++5r8nX8rhKS3HH7bgCAPawJ6fz8fJWVlentt9+OmX/zzTc7/x4wYIC6deum4cOH6+OPP9YPfvCD09LL3LlzNXv2bOdzMBhUjx49Tst3AQDwdaw43D1jxgytWrVKGzZs0JlnnvmNtUOGDJEkffTRR5KkjIwMVVRUxNTUf64/j/11NV6vt8FetCR5PB55vd6YCQCA5hbXkDbGaMaMGVq5cqXWr1+vzMzMb/2Z0tJSSVK3bt0kSdnZ2dq1a5cqKyudmsLCQnm9XmVlZTk1RUVFMcspLCxUdnZ2E60JAABNL64hnZ+fr+eff17Lli1Tx44d5ff75ff7deTIEUnSxx9/rN/+9rcqKSnRJ598otdee02TJ0/WsGHDdN5550mSRowYoaysLE2aNEnvv/++3nzzTc2bN0/5+fnyeDySpOnTp+vvf/+77rrrLn3wwQf6/e9/rxdffFF33HFH3NYdAIBvE9eQXrJkiQKBgC6//HJ169bNmVasWCFJcrvdWrdunUaMGKF+/frpzjvv1NixY/XXv/7VWUZiYqJWrVqlxMREZWdn6/rrr9fkyZN1//33OzWZmZlavXq1CgsLNXDgQD300EN65plnrLv9CgCAY7mMMSbeTdguGAzK5/MpEAg0+fnpYDCoCYvXy0SjkqTEpGQludtp+W3DnCMBAIC2yYoLxwAAQEOENAAAliKkAQCwFCFtofrnd3O5AAC0bYS0hXh+NwBAIqStxfO7AQCENAAAliKkAQCwFCENAIClCGkAACxFSFsoUlfrPCYUANB2EdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQtpSkbqwQqFQvNsAAMQRIQ0AgKUI6Tgyxvx7b9nEuxUAgIWS4t1AWxYOhzXl6U3ibyUAwImQDnGWmJQc7xYAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUnEN6QULFuiiiy5Sx44dlZaWptGjR6u8vDympqamRvn5+TrjjDPUoUMHjR07VhUVFTE1+/fvV15entq3b6+0tDTNmTNHdXV1MTUbN27UBRdcII/Hoz59+qigoOB0r94pMcYoFArJGBPvVgAAcRLXkN60aZPy8/O1detWFRYWqra2ViNGjNDhw4edmjvuuEN//etf9dJLL2nTpk36/PPPNWbMGGc8EokoLy9P4XBYW7Zs0XPPPaeCggLNnz/fqdm3b5/y8vJ0xRVXqLS0VLNmzdKNN96oN998s1nX92REI3WatnSrwuFwvFsBAMSJy1i0q3bgwAGlpaVp06ZNGjZsmAKBgLp27aply5bp5z//uSTpgw8+0DnnnKPi4mINHTpUb7zxhq666ip9/vnnSk9PlyQ9+eSTuvvuu3XgwAG53W7dfffdWr16tcrKypzvGj9+vKqqqrRmzZpv7SsYDMrn8ykQCMjr9TbZ+oZCIf380TWK1EWceYlJyYpGIzLRqNztvqcVMy6Xx+Npsu8EALQcVp2TDgQCkqTOnTtLkkpKSlRbW6ucnBynpl+/furZs6eKi4slScXFxRowYIAT0JKUm5urYDCo3bt3OzXHLqO+pn4ZxwuFQgoGgzETAADNzZqQjkajmjVrli655BKde+65kiS/3y+3263U1NSY2vT0dPn9fqfm2ICuH68f+6aaYDCoI0eONOhlwYIF8vl8ztSjR48mWUcAAE6GNSGdn5+vsrIyLV++PN6taO7cuQoEAs706aefxrslAEAblBTvBiRpxowZWrVqlTZv3qwzzzzTmZ+RkaFwOKyqqqqYvemKigplZGQ4Ndu3b49ZXv3V38fWHH9FeEVFhbxer1JSUhr04/F4OA8MAIi7uO5JG2M0Y8YMrVy5UuvXr1dmZmbM+ODBg5WcnKyioiJnXnl5ufbv36/s7GxJUnZ2tnbt2qXKykqnprCwUF6vV1lZWU7Nscuor6lfBgAANorrnnR+fr6WLVumv/zlL+rYsaNzDtnn8yklJUU+n0/Tpk3T7Nmz1blzZ3m9Xs2cOVPZ2dkaOnSoJGnEiBHKysrSpEmTtHDhQvn9fs2bN0/5+fnO3vD06dP1xBNP6K677tLUqVO1fv16vfjii1q9enXc1h0AgG8T1z3pJUuWKBAI6PLLL1e3bt2cacWKFU7NI488oquuukpjx47VsGHDlJGRoVdeecUZT0xM1KpVq5SYmKjs7Gxdf/31mjx5su6//36nJjMzU6tXr1ZhYaEGDhyohx56SM8884xyc3ObdX0BADgZVt0nbSvukwYAxIM1V3ejobrao/dr83cUALRNhLTFeDQoALRthLTlEpLc8W4BABAnhDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQBALAUIQ0AgKUI6TgKhUKKRnnDFQDgxAhpAAAsRUgDAGApQhoAAEsR0gAAWIqQBgDAUoQ0AACWIqQtF6kLKxQKxbsNAEAcENIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABYipC2nDFGoVBIxph4twIAaGaEtOWikTpNW7pV4XA43q0AAJoZId0CJCS5490CACAOCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsFSjQrp379768ssvG8yvqqpS7969T7kpAADQyJD+5JNPFIlEGswPhUL67LPPTrkpAAAgJZ1M8Wuvveb8+80335TP53M+RyIRFRUV6ayzzmqy5gAAaMtOKqRHjx4tSXK5XJoyZUrMWHJyss466yw99NBDTdYcAABt2UmFdDQalSRlZmZqx44d6tKly2lpCgAAnGRI19u3b19T9wEAAI7TqJCWpKKiIhUVFamystLZw663dOnSU24MAIC2rlEhfd999+n+++/XhRdeqG7dusnlcjV1XwAAtHmNCuknn3xSBQUFmjRpUlP3AwAA/q1R90mHw2H96Ec/aupeAADAMRoV0jfeeKOWLVvW1L3ga0TqwgqFQvFuAwDQzBp1uLumpkZPP/201q1bp/POO0/Jyckx4w8//HCTNAcAQFvWqJDeuXOnBg0aJEkqKyuLGeMiMgAAmkajQnrDhg1N3QcAADgOr6oEAMBSjdqTvuKKK77xsPb69esb3RAAADiqUSFdfz66Xm1trUpLS1VWVtbgxRsAAKBxGhXSjzzyyAnn33vvvTp06NApNQQAAI5q0nPS119/Pc/tBgCgiTRpSBcXF6tdu3ZNuUgAANqsRoX0mDFjYqaf/exnGjp0qG644Qbdcsst33k5mzdv1tVXX63u3bvL5XLp1VdfjRn/xS9+IZfLFTONHDkypubgwYOaOHGivF6vUlNTNW3atAaH3Hfu3KnLLrtM7dq1U48ePbRw4cLGrDYAAM2qUeekfT5fzOeEhAT17dtX999/v0aMGPGdl3P48GENHDhQU6dO1ZgxY05YM3LkSD377LPOZ4/HEzM+ceJEffHFFyosLFRtba1uuOEG3Xzzzc5jS4PBoEaMGKGcnBw9+eST2rVrl6ZOnarU1FTdfPPN37lXAACaW6NC+tjQPBWjRo3SqFGjvrHG4/EoIyPjhGN79+7VmjVrtGPHDl144YWSpEWLFunKK6/Ugw8+qO7du+uFF15QOBzW0qVL5Xa71b9/f5WWlurhhx8mpAEAVjulc9IlJSV6/vnn9fzzz+u9995rqp5ibNy4UWlpaerbt69uvfVWffnll85YcXGxUlNTnYCWpJycHCUkJGjbtm1OzbBhw+R2u52a3NxclZeX66uvvjrhd4ZCIQWDwZgJAIDm1qg96crKSo0fP14bN25UamqqJKmqqkpXXHGFli9frq5duzZJcyNHjtSYMWOUmZmpjz/+WL/61a80atQoFRcXKzExUX6/X2lpaTE/k5SUpM6dO8vv90uS/H6/MjMzY2rS09OdsU6dOjX43gULFui+++5rknUAAKCxGrUnPXPmTFVXV2v37t06ePCgDh48qLKyMgWDQf3nf/5nkzU3fvx4/fSnP9WAAQM0evRorVq1Sjt27NDGjRub7DtOZO7cuQoEAs706aefntbvAwDgRBq1J71mzRqtW7dO55xzjjMvKytLixcvPqkLx05W79691aVLF3300UcaPny4MjIyVFlZGVNTV1engwcPOuexMzIyVFFREVNT//nrznV7PJ4GF6gBANDcGrUnHY1GG7xDWpKSk5MVjUZPuamv849//ENffvmlunXrJknKzs5WVVWVSkpKnJr169crGo1qyJAhTs3mzZtVW1vr1BQWFqpv374nPNRtI2OMQqGQjDHxbgUA0IwaFdI/+clPdPvtt+vzzz935n322We64447NHz48O+8nEOHDqm0tFSlpaWSpH379qm0tFT79+/XoUOHNGfOHG3dulWffPKJioqKdM0116hPnz7Kzc2VJJ1zzjkaOXKkbrrpJm3fvl3vvPOOZsyYofHjx6t79+6SpOuuu05ut1vTpk3T7t27tWLFCj322GOaPXt2Y1Y9LqKROk1bulXhcDjerQAAmlGjQvqJJ55QMBjUWWedpR/84Af6wQ9+oMzMTAWDQS1atOg7L+fdd9/V+eefr/PPP1+SNHv2bJ1//vmaP3++EhMTtXPnTv30pz/V2WefrWnTpmnw4MF66623Yg5Fv/DCC+rXr5+GDx+uK6+8UpdeeqmefvppZ9zn82nt2rXat2+fBg8erDvvvFPz589vcbdfJSS5v70IANCquEwjj6EaY7Ru3Tp98MEHko7u1ebk5DRpc7YIBoPy+XwKBALyer1NutwJi9fLHHOKIDEpWdFoJGaeJLnbfU8rZlzOuXIAaENOak96/fr1ysrKUjAYlMvl0n/8x39o5syZmjlzpi666CL1799fb7311unqtU3jvDQAtD0nFdKPPvqobrrpphPuTfp8Pt1yyy16+OGHm6w5/J9opFaTnnqL89IA0IacVEi///77DV5wcawRI0bEXGmNpsV5aQBoW04qpCsqKk5461W9pKQkHThw4JSbAgAAJxnS3//+91VWVva14zt37nTuYQYAAKfmpEL6yiuv1G9+8xvV1NQ0GDty5IjuueceXXXVVU3WHAAAbdlJPRZ03rx5euWVV3T22WdrxowZ6tu3ryTpgw8+0OLFixWJRPTrX//6tDQKAEBbc1IhnZ6eri1btujWW2/V3LlznduBXC6XcnNztXjxYucNUwAA4NSc9As2evXqpddff11fffWVPvroIxlj9MMf/rDFPAcbAICWolFvwZKkTp066aKLLmrKXgAAwDEa9exuAABw+hHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQjpFiRSF1YoFIp3GwCAZkJIAwBgKUIaAABLEdIAAFiKkI4TY8y/zy+beLcCALBUo5/djVMTDoc15elN4u8kAMDXISHiKDEpOd4tAAAsRki3IPWHyOvf4w0AaN0I6RYkGqnTtKVbFQ6H490KAKAZENItTEKSO94tAACaCSENAIClCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUi3MLxkAwDaDkK6hYlGajXpqbd4yQYAtAGEdAsTqauVKyEp3m0AAJoBIQ0AgKUIaQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABYipAGAMBShDQAAJYipAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0i1QpC6sUCgU7zYAAKcZIQ0AgKXiGtKbN2/W1Vdfre7du8vlcunVV1+NGTfGaP78+erWrZtSUlKUk5OjDz/8MKbm4MGDmjhxorxer1JTUzVt2jQdOnQopmbnzp267LLL1K5dO/Xo0UMLFy483asGAMApi2tIHz58WAMHDtTixYtPOL5w4UI9/vjjevLJJ7Vt2zZ973vfU25urmpqapyaiRMnavfu3SosLNSqVau0efNm3Xzzzc54MBjUiBEj1KtXL5WUlOh3v/ud7r33Xj399NOnff0AADgVSfH88lGjRmnUqFEnHDPG6NFHH9W8efN0zTXXSJL+9Kc/KT09Xa+++qrGjx+vvXv3as2aNdqxY4cuvPBCSdKiRYt05ZVX6sEHH1T37t31wgsvKBwOa+nSpXK73erfv79KS0v18MMPx4Q5AAC2sfac9L59++T3+5WTk+PM8/l8GjJkiIqLiyVJxcXFSk1NdQJaknJycpSQkKBt27Y5NcOGDZPb7XZqcnNzVV5erq+++qqZ1gYAgJMX1z3pb+L3+yVJ6enpMfPT09OdMb/fr7S0tJjxpKQkde7cOaYmMzOzwTLqxzp16tTgu0OhUMzV08Fg8BTXBgCAk2ftnnQ8LViwQD6fz5l69OgR75YAAG2QtSGdkZEhSaqoqIiZX1FR4YxlZGSosrIyZryurk4HDx6MqTnRMo79juPNnTtXgUDAmT799NNTX6EmZIxRMBiMuYAOAND6WBvSmZmZysjIUFFRkTMvGAxq27Ztys7OliRlZ2erqqpKJSUlTs369esVjUY1ZMgQp2bz5s2qra11agoLC9W3b98THuqWJI/HI6/XGzPZJBqp0y3PbVc4HI53KwCA0yiuIX3o0CGVlpaqtLRU0tGLxUpLS7V//365XC7NmjVL//Vf/6XXXntNu3bt0uTJk9W9e3eNHj1aknTOOedo5MiRuummm7R9+3a98847mjFjhsaPH6/u3btLkq677jq53W5NmzZNu3fv1ooVK/TYY49p9uzZcVrrppGQlBzvFgAAp1lcLxx79913dcUVVzif64NzypQpKigo0F133aXDhw/r5ptvVlVVlS699FKtWbNG7dq1c37mhRde0IwZMzR8+HAlJCRo7Nixevzxx51xn8+ntWvXKj8/X4MHD1aXLl00f/58br8CAFjPZYwx8W7CdsFgUD6fT4FAoMkOfYdCIf380TWK1EVi5icmJSsajchEo984PzEpWS/c+mPrDsUDAJqOteekAQBo6whpAAAsRUgDAGApQhoAAEsR0gAAWIqQjgNjTMyzwQEAOBFCOg7C4bAmLtmgaJS73wAAX4+QjpNEnhgGAPgWhDQAAJYipAEAsBQhDQCApQjpFqr+CnEevQ4ArRch3UJFI3WatnQr75QGgFaMkG7BEpLc8W4BAHAaEdIAAFiKkAYAwFKENAAAliKkAQCwFCHdgtXVhhQMBrkNCwBaKUK6BeM2LABo3QjpFo7bsACg9SKkAQCwFCENAIClCGkAACxFSAMAYClCuoWL1IUVCoXi3QYA4DQgpAEAsBQhDQCApQhpAAAsRUgDAGApQhoAAEsR0i2cMUahUIiXbABAK0RIt3C8ZAMAWi9CuhXgJRsA0DoR0gAAWIqQBgDAUoQ0AACWIqQBALAUId0KcBsWALROhHQrEI3UatJTb3EbFgC0MoR0K8FtWADQ+hDSrQTvlQaA1oeQBgDAUoQ0AACWIqQBALAUIQ0AgKUIaQAALEVItyI81AQAWhdCupUwxqi6ulrjnijioSYA0EoQ0q1ENFKnW57bLldCcrxbAQA0EUK6FTGSTDQa7zYAAE2EkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCupXhvdIA0HoQ0gAAWIqQBgDAUoQ0AACWIqQBALCU1SF97733yuVyxUz9+vVzxmtqapSfn68zzjhDHTp00NixY1VRURGzjP379ysvL0/t27dXWlqa5syZo7q6uuZeFQAATlpSvBv4Nv3799e6deucz0lJ/9fyHXfcodWrV+ull16Sz+fTjBkzNGbMGL3zzjuSpEgkory8PGVkZGjLli364osvNHnyZCUnJ+u///u/m31dAAA4GdaHdFJSkjIyMhrMDwQC+uMf/6hly5bpJz/5iSTp2Wef1TnnnKOtW7dq6NChWrt2rfbs2aN169YpPT1dgwYN0m9/+1vdfffduvfee+V2u5t7dU47Y4xCoZCMMXK5XPFuBwBwCqw+3C1JH374obp3767evXtr4sSJ2r9/vySppKREtbW1ysnJcWr79eunnj17qri4WJJUXFysAQMGKD093anJzc1VMBjU7t27v/Y7Q6GQgsFgzNRSRCN1mrZ0q8LhcLxbAQCcIqtDesiQISooKNCaNWu0ZMkS7du3T5dddpmqq6vl9/vldruVmpoa8zPp6eny+/2SJL/fHxPQ9eP1Y19nwYIF8vl8ztSjR4+mXbHTzEg80AQAWgGrD3ePGjXK+fd5552nIUOGqFevXnrxxReVkpJy2r537ty5mj17tvM5GAy2uKAGALR8Vu9JHy81NVVnn322PvroI2VkZCgcDquqqiqmpqKiwjmHnZGR0eBq7/rPJzrPXc/j8cjr9cZMTSkUCikaNU26TABA69OiQvrQoUP6+OOP1a1bNw0ePFjJyckqKipyxsvLy7V//35lZ2dLkrKzs7Vr1y5VVlY6NYWFhfJ6vcrKymr2/gEAOBlWH+7+f//v/+nqq69Wr1699Pnnn+uee+5RYmKiJkyYIJ/Pp2nTpmn27Nnq3LmzvF6vZs6cqezsbA0dOlSSNGLECGVlZWnSpElauHCh/H6/5s2bp/z8fHk8njiv3enDFd4A0DpYvSf9j3/8QxMmTFDfvn117bXX6owzztDWrVvVtWtXSdIjjzyiq666SmPHjtWwYcOUkZGhV155xfn5xMRErVq1SomJicrOztb111+vyZMn6/7774/XKjULrvAGgNbBZYzh5Oi3CAaD8vl8CgQCTXJ+OhgM6vqn3lKkrlYmGo0ZS0xKVjQa+c7zv27M3e57WjHj8lZ9xAAAWjur96QBAGjLCGkAACxFSLdSkbqw80CTUCjEw00AoAUipAEAsBQhDQCApQhpAAAsRUgDAGApQrqVOvapYwCAlomQbqV46hgAtHyEdCvmSkxmbxoAWjBCuhWLRmo16am32JsGgBaKkG7lEpLc8W4BANBIhHQrxwVkANByEdKtXG3oX/rF0xzyBoCWiJBuAxKSkuPdAgCgEQjpNoBD3gDQMhHSbQD3TANAy0RItxFc5Q0ALQ8hDQCApQhpAAAsRUi3EZG6sEKhULzbAACcBEIaAABLEdJtBLdhAUDLQ0i3EdyGBQAtDyHdhkSNUTAYZG8aAFoIQroNYW8aAFoWQrqN4aEmANByENIAAFiKkAYAwFKENAAAliKkAQCwVFK8G0DzitSFVVNTI0lyu91yuVxx7ggA8HXYk25m9U/+kuJzr7IxRtXV1fr5o2tUXV0dlx4AAN8NId3MwuGwpjy9SSYan5CORup0y3Pb5UrgIAoA2I6QjoPEpOS4fn/Cv78/FArxZiwAsBghDQCApQjpNo69aQCwFyHdRvHqSgCwHyHdRtVfQMbLNgDAXoR0G5ZwzAVs7FkDgH0I6TbMGKOamhrV1NQoFApp3BNF7FkDgEUI6TYsGqnTTUu3aNKTmxUOh5XIaywBwCqEdBuXkJSsqDEKBoPxbgUAcBxCGopG6nTjH99RpC4S71YAAMcgpCEp9iIyAIAdCGkAACxFSKMBbscCADsQ0nDU1YYUDAYVCoV4lSUAWICQhiMaqdPUPxaruro67m/qAgAQ0s0uFAopGqd3SX8nLpdueOYdRaOGl28AQJwR0mjASDLRaLzbAIA2LyneDcBO9Y8MTU5OljFGHo9HLpcr3m0BQJtCSOOE6h8ZKknJnhS9cNvl6tixI0ENAM2Iw934WglJyUcfcuJyadJTb/HyDQBoZoQ0vhNXYrICgYACgYCi0Sj3UQNAMyCk8a0idbWK1IY05alNmrBonT7//HNdu2idqqurCWoAOI0IaXxn9Ye+b/zjO4pGos4h8GOfUMZtWwDQdAhpnLT6l3FEjVEgEFAwGNS4J4qcgGbvGgCaBld3o9GikTrd8Ie35UpIUHK77ykcDmvikg16fvrlcrlccrvdXA0OAKeAPWk0SqSuViYaVUJSsoyk2tARBYNBJSYlKxwO69pF6xQMBhUIBFRTUxPvdgGgRWJPGk0iGqnTjX98R0meFIVCIblcLk38/QYZY/TsjZcoFArJ4/GoXbt28W4VAFqMNrUnvXjxYp111llq166dhgwZou3bt8e7pVYlISlZ0Uidbnluu2rDYbkSEiWXS5OXbNCEY64GN8YoGAyqpqYm5qIzXpEJALHaTEivWLFCs2fP1j333KO//e1vGjhwoHJzc1VZWRnv1lqd45/9XX9V+JSnNuvAgQMKBoO67vfrdeDAAR04cEDXLloX8+/q6modOXJEgUBAR44cccL82wKdkAfQ2rhMG/l/tCFDhuiiiy7SE088IUmKRqPq0aOHZs6cqV/+8pff+LPBYFA+n0+BQEBer/eU+ggGg5qweL0SEhIVjUYavMgiMSn5pOY35mfiuaxoXe3R12AmJCgSPnqrVkJScsx8E4lo0YRBmrGsRCZqlOxJ0fO3/liSNHHJBi2d+iN5PB5N/sPb+tNNl8rj8cjtduvLL7/UtGe36vnpP1aHDh2cJ6Sd6LnjxhiFw2Hn4rbjPwOADdpESIfDYbVv314vv/yyRo8e7cyfMmWKqqqq9Je//CWm/vh7fQOBgHr27KlPP/20SUL6hiVFciUkypwg3BKSkk9qfmN+piUsKxqpU0Ji7CUTxkTkciUqGqk7Wv/vcVdCgh6//mLN/NMWSQlKSErWkzf8SDf/YYOkBC2aPFQdO3aUx+NxlhUKhXTTH9/SH6ZdJo/H0+Dzyaj/XTnZnwPQMjXl/9a/9Z0Ipg347LPPjCSzZcuWmPlz5swxF198cYP6e+65x+jfR22ZmJiYmJhO1xQIBL4xv7i6+wTmzp2r2bNnO5+j0agOHjyoM84445QPhQaDQfXo0aNJ9srxf9iuTY9tenqwXU+PlrpdO3bs+I3jbSKku3TposTERFVUVMTMr6ioUEZGRoN6j8fT4HBGampqk/bk9Xpb1C9SS8F2bXps09OD7Xp6tLbt2iau7na73Ro8eLCKioqcedFoVEVFRcrOzo5jZwAAfL02sSctSbNnz9aUKVN04YUX6uKLL9ajjz6qw4cP64Ybboh3awAAnFCbCelx48bpwIEDmj9/vvx+vwYNGqQ1a9YoPT29WfvweDy65557uBK4ibFdmx7b9PRgu54erXW7tolbsAAAaInaxDlpAABaIkIaAABLEdIAAFiKkAYAwFKEdDPiVZn/Z/Pmzbr66qvVvXt3uVwuvfrqqzHjxhjNnz9f3bp1U0pKinJycvThhx/G1Bw8eFATJ06U1+tVamqqpk2bpkOHDsXU7Ny5U5dddpnatWunHj16aOHChQ16eemll9SvXz+1a9dOAwYM0Ouvv97k69tcFixYoIsuukgdO3ZUWlqaRo8erfLy8piampoa5efn64wzzlCHDh00duzYBg/62b9/v/Ly8tS+fXulpaVpzpw5qquri6nZuHGjLrjgAnk8HvXp00cFBQUN+mkNv/NLlizReeed5zwkIzs7W2+88YYzzvZsGg888IBcLpdmzZrlzGPbSm3i2d02WL58uXG73Wbp0qVm9+7d5qabbjKpqammoqIi3q3Fxeuvv25+/etfm1deecVIMitXrowZf+CBB4zP5zOvvvqqef/9981Pf/pTk5mZaY4cOeLUjBw50gwcONBs3brVvPXWW6ZPnz5mwoQJznggEDDp6elm4sSJpqyszPz5z382KSkp5qmnnnJq3nnnHZOYmGgWLlxo9uzZY+bNm2eSk5PNrl27Tvs2OB1yc3PNs88+a8rKykxpaam58sorTc+ePc2hQ4ecmunTp5sePXqYoqIi8+6775qhQ4eaH/3oR854XV2dOffcc01OTo557733zOuvv266dOli5s6d69T8/e9/N+3btzezZ882e/bsMYsWLTKJiYlmzZo1Tk1r+Z1/7bXXzOrVq83//M//mPLycvOrX/3KJCcnm7KyMmMM27MpbN++3Zx11lnmvPPOM7fffrszn2179L28aAYXX3yxyc/Pdz5HIhHTvXt3s2DBgjh2ZYfjQzoajZqMjAzzu9/9zplXVVVlPB6P+fOf/2yMMWbPnj1GktmxY4dT88YbbxiXy2U+++wzY4wxv//9702nTp1MKBRyau6++27Tt29f5/O1115r8vLyYvoZMmSIueWWW5p0HeOlsrLSSDKbNm0yxhzdjsnJyeall15yavbu3WskmeLiYmPM0T+gEhISjN/vd2qWLFlivF6vsy3vuusu079//5jvGjdunMnNzXU+t+bf+U6dOplnnnmG7dkEqqurzQ9/+ENTWFhofvzjHzshzbY9isPdzSAcDqukpEQ5OTnOvISEBOXk5Ki4uDiOndlp37598vv9MdvL5/NpyJAhzvYqLi5WamqqLrzwQqcmJydHCQkJ2rZtm1MzbNgwud1upyY3N1fl5eX66quvnJpjv6e+prX8dwkEApKkzp07S5JKSkpUW1sbs879+vVTz549Y7btgAEDYh70k5ubq2AwqN27dzs137TdWuvvfCQS0fLly3X48GFlZ2ezPZtAfn6+8vLyGqw/2/aoNvPEsXj65z//qUgk0uDpZunp6frggw/i1JW9/H6/JJ1we9WP+f1+paWlxYwnJSWpc+fOMTWZmZkNllE/1qlTJ/n9/m/8npYsGo1q1qxZuuSSS3TuuedKOrrebre7wQtjjt+2J9om9WPfVBMMBnXkyBF99dVXrep3fteuXcrOzlZNTY06dOiglStXKisrS6WlpWzPU7B8+XL97W9/044dOxqM8bt6FCENtFL5+fkqKyvT22+/He9WWry+ffuqtLRUgUBAL7/8sqZMmaJNmzbFu60W7dNPP9Xtt9+uwsJCtWvXLt7tWIvD3c3gZF+V2dbVb5Nv2l4ZGRmqrKyMGa+rq9PBgwdjak60jGO/4+tqWvp/lxkzZmjVqlXasGGDzjzzTGd+RkaGwuGwqqqqYuqP37aN3W5er1cpKSmt7nfe7XarT58+Gjx4sBYsWKCBAwfqscceY3uegpKSElVWVuqCCy5QUlKSkpKStGnTJj3++ONKSkpSeno621aEdLPgVZknJzMzUxkZGTHbKxgMatu2bc72ys7OVlVVlUpKSpya9evXKxqNasiQIU7N5s2bVVtb69QUFhaqb9++6tSpk1Nz7PfU17TU/y7GGM2YMUMrV67U+vXrGxzuHzx4sJKTk2PWuby8XPv374/Ztrt27Yr5I6iwsFBer1dZWVlOzTdtt9b+Ox+NRhUKhdiep2D48OHatWuXSktLnenCCy/UxIkTnX+zbcUtWM1l+fLlxuPxmIKCArNnzx5z8803m9TU1JirEtuS6upq895775n33nvPSDIPP/ywee+998z//u//GmOO3oKVmppq/vKXv5idO3eaa6655oS3YJ1//vlm27Zt5u233zY//OEPY27BqqqqMunp6WbSpEmmrKzMLF++3LRv377BLVhJSUnmwQcfNHv37jX33HNPi74F69ZbbzU+n89s3LjRfPHFF870r3/9y6mZPn266dmzp1m/fr159913TXZ2tsnOznbG629rGTFihCktLTVr1qwxXbt2PeFtLXPmzDF79+41ixcvPuFtLa3hd/6Xv/yl2bRpk9m3b5/ZuXOn+eUvf2lcLpdZu3atMYbt2ZSOvbrbGLatMdyC1awWLVpkevbsadxut7n44ovN1q1b491S3GzYsMFIajBNmTLFGHP0Nqzf/OY3Jj093Xg8HjN8+HBTXl4es4wvv/zSTJgwwXTo0MF4vV5zww03mOrq6pia999/31x66aXG4/GY73//++aBBx5o0MuLL75ozj77bON2u03//v3N6tWrT9t6n24n2qaSzLPPPuvUHDlyxNx2222mU6dOpn379uZnP/uZ+eKLL2KW88knn5hRo0aZlJQU06VLF3PnnXea2tramJoNGzaYQYMGGbfbbXr37h3zHfVaw+/81KlTTa9evYzb7TZdu3Y1w4cPdwLaGLZnUzo+pNm2xvCqSgAALMU5aQAALEVIAwBgKUIaAABLEdIAAFiKkAYAwFKENAAAliKkAQCwFCENAIClCGkAACxFSAMAYClCGgAASxHSAABY6v8Dtpf2jA6pNcUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAIACAYAAABNWi9DAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2eklEQVR4nO3deXRU9f3/8ddkmQlRkrAlQ2qEiIIF2YyaphWqhRKQL9ZKqyhKVCTqN6BCS2kqsmkNBYtbqZZfRWylYv0epBYtJSwSlYgQiWw2VQvFYhIqS4ZlJtt8fn/Q3DomiISE+SR5Ps65h7n385l73/c6nlfu7jLGGAEAACtFhLsAAABwcgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoATRo48aNmjVrlg4fPtxsy3jkkUe0YsWKZps/0BoQ1AAatHHjRs2ePZugBsKMoAYAwGIENYB6Zs2apalTp0qSUlNT5XK55HK5tGfPHknSCy+8oLS0NLVr104dO3bUmDFj9Mknn4TM48MPP9To0aPl9XoVExOj8847T2PGjFFFRYUkyeVy6dixY3r++eed+d92221nczWBFiEq3AUAsM/111+vv//973rxxRf12GOPqXPnzpKkLl266Oc//7kefPBB3XDDDbrzzjv173//W0899ZQGDx6srVu3KiEhQVVVVcrMzFRlZaUmTZokr9erffv2aeXKlTp8+LDi4+P1+9//XnfeeaeuuOIKZWdnS5J69OgRztUG7GQAoAHz5883kszu3budaXv27DGRkZHm5z//eUjf7du3m6ioKGf61q1bjSTz8ssvf+kyzjnnHJOVldXUpQOtCoe+AXxly5cvVzAY1A033KDPPvvMGbxery666CKtX79ekhQfHy9J+utf/6rjx4+Hs2SgxePQN4Cv7MMPP5QxRhdddFGD7dHR0ZJOnNeeMmWKFixYoKVLl2rQoEG69tprdcsttzghDuCrIagBfGXBYFAul0t/+ctfFBkZWa/93HPPdT7/8pe/1G233aY//elPWr16te69917l5eXpnXfe0XnnnXc2ywZaNIIaQINcLle9aT169JAxRqmpqerZs+cp59G3b1/17dtX06dP18aNG/Wtb31LzzzzjB5++OGTLgNAKM5RA2jQOeecI0khDzy5/vrrFRkZqdmzZ8sYE9LfGKMDBw5Iknw+n2pqakLa+/btq4iICFVWVoYsozkfqAK0BuxRA2hQWlqaJOmBBx7QmDFjFB0drVGjRunhhx9Wbm6u9uzZo+uuu07t27fX7t279corryg7O1s//vGPtW7dOk2cOFE//OEP1bNnT9XU1Oj3v/+9IiMjNXr06JBlrFmzRgsWLFBycrJSU1OVnp4erlUGrOQyX/yzGAD+4+GHH9Yzzzyj0tJSBYNB7d69W927d9fy5cv12GOPaevWrZKklJQUDRkyRPfee6969uyp3bt36+GHH9aGDRu0b98+xcbGqn///nrggQc0ZMgQZ/4lJSXKzs7W5s2b5ff7lZWVpSVLloRpbQE7EdQAAFiMc9QAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUH8Fxhj5fL56T2ICAKC5EdRfwZEjRxQfH68jR46EuxQAQBtDUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoLaAMUZ+v1/GmHCXAgCwDEFtgUAgoBufzFcgEAh3KQAAyxDUloh0e8JdAgDAQgS1Bfx+v0wwGO4yAAAWIqgBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoLcHzvgEADSGoLRGsqVLWogKe9w0ACEFQWyQymud9AwBCEdQAAFiMoAYAwGIENQAAFiOoAQCwGEEdZnW3ZYm7sgAADSCowywQCGj8ojcUDAbDXQoAwEIEtQUio93hLgEAYCmCGgAAixHUAABYjKAGAMBiYQ3qgoICjRo1SsnJyXK5XFqxYkVIu8vlanCYP3++06d79+712ufOnRsyn23btmnQoEGKiYlRSkqK5s2bdzZWDwCAMxbWoD527Jj69++vhQsXNtheWloaMixevFgul0ujR48O6TdnzpyQfpMmTXLafD6fhg0bpm7duqmoqEjz58/XrFmztGjRomZdNwAAmkJUOBc+YsQIjRgx4qTtXq83ZPxPf/qTrr76al1wwQUh09u3b1+vb52lS5eqqqpKixcvltvtVp8+fVRcXKwFCxYoOzv7zFcCAIBm1GLOUZeXl+u1117T+PHj67XNnTtXnTp10sCBAzV//nzV1NQ4bYWFhRo8eLDc7v/eApWZmamSkhIdOnSowWVVVlbK5/OFDAAAhENY96hPx/PPP6/27dvr+uuvD5l+77336tJLL1XHjh21ceNG5ebmqrS0VAsWLJAklZWVKTU1NeQ7SUlJTluHDh3qLSsvL0+zZ89upjUBAOCrazFBvXjxYo0dO1YxMTEh06dMmeJ87tevn9xut+666y7l5eXJ42nc+51zc3ND5uvz+ZSSktK4wgEAOAMtIqjffPNNlZSU6KWXXjpl3/T0dNXU1GjPnj3q1auXvF6vysvLQ/rUjZ/svLbH42l0yAMA0JRaxDnqZ599Vmlpaerfv/8p+xYXFysiIkKJiYmSpIyMDBUUFKi6utrpk5+fr169ejV42DucaqsrT7ygAwCA/whrUB89elTFxcUqLi6WJO3evVvFxcXau3ev08fn8+nll1/WnXfeWe/7hYWFevzxx/X+++/rH//4h5YuXarJkyfrlltucUL45ptvltvt1vjx47Vz50699NJLeuKJJ0IObQMAYKuwHvresmWLrr76ame8LjyzsrK0ZMkSSdKyZctkjNFNN91U7/sej0fLli3TrFmzVFlZqdTUVE2ePDkkhOPj47V69Wrl5OQoLS1NnTt31owZM7g1CwDQIriMMbwJ+RR8Pp/i4+NVUVGhuLi4Jp233+/XD365UnJFKiIyQs9nD1bHjh2bdBkAgJarRZyjBgCgrSKoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlBbxBgjv98v3pMCAKhDUFskWFOl7CWbFAgEwl0KAMASBLVlIqM94S4BAGARghoAAIsR1AAAWIygDqO6i8cAADgZgjqMAoGAbv31WhkTDHcpAABLEdRhxsVjAIAvQ1ADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoLZM3Ru1jDHhLgUAYAGC2jLBmiplLSpQIBAIdykAAAsQ1BbijVoAgDoENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsFtagLigo0KhRo5ScnCyXy6UVK1aEtN92221yuVwhw/Dhw0P6HDx4UGPHjlVcXJwSEhI0fvx4HT16NKTPtm3bNGjQIMXExCglJUXz5s1r7lUDAKBJhDWojx07pv79+2vhwoUn7TN8+HCVlpY6w4svvhjSPnbsWO3cuVP5+flauXKlCgoKlJ2d7bT7fD4NGzZM3bp1U1FRkebPn69Zs2Zp0aJFzbZeAAA0lahwLnzEiBEaMWLEl/bxeDzyer0Ntn3wwQdatWqVNm/erMsuu0yS9NRTT+maa67Ro48+quTkZC1dulRVVVVavHix3G63+vTpo+LiYi1YsCAk0AEAsJH156jfeOMNJSYmqlevXrrnnnt04MABp62wsFAJCQlOSEvS0KFDFRERoU2bNjl9Bg8eLLfb7fTJzMxUSUmJDh061OAyKysr5fP5QgYAAMLB6qAePny4fve732nt2rX6xS9+oQ0bNmjEiBGqra2VJJWVlSkxMTHkO1FRUerYsaPKysqcPklJSSF96sbr+nxRXl6e4uPjnSElJaWpV+1L1VZXyu/3n9VlAgDsFNZD36cyZswY53Pfvn3Vr18/9ejRQ2+88YaGDBnSbMvNzc3VlClTnHGfz3fWwxoAAMnyPeovuuCCC9S5c2d99NFHkiSv16v9+/eH9KmpqdHBgwed89per1fl5eUhferGT3bu2+PxKC4uLmQAACAcWlRQ/+tf/9KBAwfUtWtXSVJGRoYOHz6soqIip8+6desUDAaVnp7u9CkoKFB1dbXTJz8/X7169VKHDh3O7goAAHCawhrUR48eVXFxsYqLiyVJu3fvVnFxsfbu3aujR49q6tSpeuedd7Rnzx6tXbtW3/ve93ThhRcqMzNTkvT1r39dw4cP14QJE/Tuu+/q7bff1sSJEzVmzBglJydLkm6++Wa53W6NHz9eO3fu1EsvvaQnnngi5NA2AAC2CmtQb9myRQMHDtTAgQMlSVOmTNHAgQM1Y8YMRUZGatu2bbr22mvVs2dPjR8/XmlpaXrzzTfl8XiceSxdulQXX3yxhgwZomuuuUZXXnllyD3S8fHxWr16tXbv3q20tDT96Ec/0owZM7g1CwDQIriMMSbcRdjO5/MpPj5eFRUVTXq+2u/368Yn81VT5ZcrIlomWC1XRLQiIiP0fPZgdezYscmWBQBomVrUOWoAANoaghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gtZIyR3+8XF+QDAAhqCwVrqpS9ZJMCgUC4SwEAhBlBbanIaM+pOwEAWj2CGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALBYWIO6oKBAo0aNUnJyslwul1asWOG0VVdXa9q0aerbt6/OOeccJScna9y4cfr0009D5tG9e3e5XK6QYe7cuSF9tm3bpkGDBikmJkYpKSmaN2/e2Vg9AADOWFiD+tixY+rfv78WLlxYr+348eN677339OCDD+q9997T8uXLVVJSomuvvbZe3zlz5qi0tNQZJk2a5LT5fD4NGzZM3bp1U1FRkebPn69Zs2Zp0aJFzbpuAAA0hahwLnzEiBEaMWJEg23x8fHKz88PmfarX/1KV1xxhfbu3avzzz/fmd6+fXt5vd4G57N06VJVVVVp8eLFcrvd6tOnj4qLi7VgwQJlZ2c33coAANAMWtQ56oqKCrlcLiUkJIRMnzt3rjp16qSBAwdq/vz5qqmpcdoKCws1ePBgud1uZ1pmZqZKSkp06NChBpdTWVkpn88XMgAAEA5h3aM+HYFAQNOmTdNNN92kuLg4Z/q9996rSy+9VB07dtTGjRuVm5ur0tJSLViwQJJUVlam1NTUkHklJSU5bR06dKi3rLy8PM2ePbsZ1wYAgK+mRQR1dXW1brjhBhlj9PTTT4e0TZkyxfncr18/ud1u3XXXXcrLy5PH42nU8nJzc0Pm6/P5lJKS0rjiAQA4A9YHdV1I//Of/9S6detC9qYbkp6erpqaGu3Zs0e9evWS1+tVeXl5SJ+68ZOd1/Z4PI0OeQAAmpLV56jrQvrDDz/UmjVr1KlTp1N+p7i4WBEREUpMTJQkZWRkqKCgQNXV1U6f/Px89erVq8HD3gAA2CSse9RHjx7VRx995Izv3r1bxcXF6tixo7p27aof/OAHeu+997Ry5UrV1taqrKxMktSxY0e53W4VFhZq06ZNuvrqq9W+fXsVFhZq8uTJuuWWW5wQvvnmmzV79myNHz9e06ZN044dO/TEE0/oscceC8s6AwBwOsIa1Fu2bNHVV1/tjNedF87KytKsWbP06quvSpIGDBgQ8r3169frqquuksfj0bJlyzRr1ixVVlYqNTVVkydPDjm/HB8fr9WrVysnJ0dpaWnq3LmzZsyYwa1ZAIAWwWWMMeEuwnY+n0/x8fGqqKg45Tny0+H3+3Xjk/mqqfLLFREtE6x2/o3ynKuXJn1H7dq1a7LlAQBaHqvPUbdlxhj5/X7xdxQAtG0EtaWCNVXKWlSgQCAQ7lIAAGFEUFssMppbxACgrSOoAQCwGEENAIDFCGqLcUEZAICgthgXlAEACGrLcUEZALRtBHUYnTisHQx3GQAAixHUlqutrpTf7w93GQCAMCGoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgS15Ywx8vv9MsaEuxQAQBgQ1JYL1lQpe8kmBQKBcJcCAAgDgroFiIz2hLsEAECYENQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAizUqqC+44AIdOHCg3vTDhw/rggsuOOOiAADACY0K6j179qi2trbe9MrKSu3bt++MiwIAACdEnU7nV1991fn817/+VfHx8c54bW2t1q5dq+7duzdZcQAAtHWnFdTXXXedJMnlcikrKyukLTo6Wt27d9cvf/nLJisOAIC27rSCOhgMSpJSU1O1efNmde7cuVmKAgAAJ5xWUNfZvXt3U9cBAAAa0KiglqS1a9dq7dq12r9/v7OnXWfx4sVnXBgAAGhkUM+ePVtz5szRZZddpq5du8rlcjV1XQAAQI0M6meeeUZLlizRrbfe2tT1AACAz2nUfdRVVVX65je/2dS1AACAL2hUUN955536wx/+cMYLLygo0KhRo5ScnCyXy6UVK1aEtBtjNGPGDHXt2lXt2rXT0KFD9eGHH4b0OXjwoMaOHau4uDglJCRo/PjxOnr0aEifbdu2adCgQYqJiVFKSormzZt3xrUDAHA2NOrQdyAQ0KJFi7RmzRr169dP0dHRIe0LFiz4SvM5duyY+vfvrzvuuEPXX399vfZ58+bpySef1PPPP6/U1FQ9+OCDyszM1K5duxQTEyNJGjt2rEpLS5Wfn6/q6mrdfvvtys7Odv6Q8Pl8GjZsmIYOHapnnnlG27dv1x133KGEhARlZ2c3ZvUBADhrGhXU27Zt04ABAyRJO3bsCGk7nQvLRowYoREjRjTYZozR448/runTp+t73/ueJOl3v/udkpKStGLFCo0ZM0YffPCBVq1apc2bN+uyyy6TJD311FO65ppr9Oijjyo5OVlLly5VVVWVFi9eLLfbrT59+qi4uFgLFiwgqAEA1mtUUK9fv76p66hn9+7dKisr09ChQ51p8fHxSk9PV2FhocaMGaPCwkIlJCQ4IS1JQ4cOVUREhDZt2qTvf//7Kiws1ODBg+V2u50+mZmZ+sUvfqFDhw6pQ4cO9ZZdWVmpyspKZ9zn8zXTWgIA8OWsfc1lWVmZJCkpKSlkelJSktNWVlamxMTEkPaoqCh17NgxpE9D8/j8Mr4oLy9P8fHxzpCSknLmKwQAQCM0ao/66quv/tJD3OvWrWt0QTbIzc3VlClTnHGfz0dYAwDColFBXXd+uk51dbWKi4u1Y8eOei/raCyv1ytJKi8vV9euXZ3p5eXlzvK9Xq/2798f8r2amhodPHjQ+b7X61V5eXlIn7rxuj5f5PF45PF4mmQ9AAA4E40K6scee6zB6bNmzap3a1Rjpaamyuv1au3atU4w+3w+bdq0Sffcc48kKSMjQ4cPH1ZRUZHS0tIkndibDwaDSk9Pd/o88MADqq6udq5Oz8/PV69evRo8Pw0AgE2a9Bz1LbfcclrP+T569KiKi4tVXFws6cQFZMXFxdq7d69cLpfuv/9+Pfzww3r11Ve1fft2jRs3TsnJyc7rNr/+9a9r+PDhmjBhgt599129/fbbmjhxosaMGaPk5GRJ0s033yy3263x48dr586deumll/TEE0+EHNoGAMBWjX4pR0MKCwud+5u/ii1btujqq692xuvCMysrS0uWLNFPfvITHTt2TNnZ2Tp8+LCuvPJKrVq1KmQZS5cu1cSJEzVkyBBFRERo9OjRevLJJ532+Ph4rV69Wjk5OUpLS1Pnzp01Y8YMbs0CALQILmOMOd0vffHhJMYYlZaWasuWLXrwwQc1c+bMJivQBj6fT/Hx8aqoqFBcXFyTzffgwYPKWlSg2upKuSKiZYLVIf9GREaotrpSke5z9PyEb6pDhw68AAUA2phG7VHHx8eHjEdERKhXr16aM2eOhg0b1iSF4b+CNVXKWlSgP96XqXbt2oW7HADAWdSooH7uueeaug6cQmQ0V6EDQFt0Rueoi4qK9MEHH0iS+vTpo4EDBzZJUQAA4IRGBfX+/fs1ZswYvfHGG0pISJAkHT58WFdffbWWLVumLl26NGWNAAC0WY26PWvSpEk6cuSIdu7cqYMHD+rgwYPasWOHfD6f7r333qauEQCANqtRe9SrVq3SmjVr9PWvf92Z1rt3by1cuJCLyQAAaEKN2qMOBoP13kEtSdHR0QoGg2dcFAAAOKFRQf2d73xH9913nz799FNn2r59+zR58mQNGTKkyYoDAKCta1RQ/+pXv5LP51P37t3Vo0cP9ejRQ6mpqfL5fHrqqaeaukYAANqsRp2jTklJ0Xvvvac1a9bob3/7m6QTz90eOnRokxYHAEBbd1p71OvWrVPv3r3l8/nkcrn03e9+V5MmTdKkSZN0+eWXq0+fPnrzzTebq1YAANqc0wrqxx9/XBMmTGjwedfx8fG66667tGDBgiYrDgCAtu60gvr999/X8OHDT9o+bNgwFRUVnXFRAADghNMK6vLy8gZvy6oTFRWlf//732dcFAAAOOG0gvprX/uaduzYcdL2bdu2qWvXrmdcFAAAOOG0gvqaa67Rgw8+qEAgUK/N7/dr5syZ+p//+Z8mKw4AgLbutG7Pmj59upYvX66ePXtq4sSJ6tWrlyTpb3/7mxYuXKja2lo98MADzVJoW1dbXSm/38/7qAGgjTmtoE5KStLGjRt1zz33KDc3V8YYSZLL5VJmZqYWLlyopKSkZikUAIC26LQfeNKtWze9/vrrOnTokD766CMZY3TRRRepQ4cOzVEfAABtWqOeTCZJHTp00OWXX96UtbQpxhj5/X7JhLsSAIDNGvWsb5y5QCCg8Yve4G1jAIAvRVCHUWS0O9wlAAAsR1ADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6hbiLr7ruueBgcAaBsI6hYiWFOl7CWbGnwhCgCg9SKoW5DIaE+4SwAAnGUENQAAFiOoAQCwGEHdgnBBGQC0PQR1CxKsqVLWogIuKAOANoSgbmG4oAwA2haCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQjqFqa2ulJ+vz/cZQAAzhKCGgAAixHUAABYjKAGAMBiBDUAABazPqi7d+8ul8tVb8jJyZEkXXXVVfXa7r777pB57N27VyNHjlRsbKwSExM1depU1dTUhGN1AAA4LVHhLuBUNm/erNraWmd8x44d+u53v6sf/vCHzrQJEyZozpw5znhsbKzzuba2ViNHjpTX69XGjRtVWlqqcePGKTo6Wo888sjZWQkAABrJ+qDu0qVLyPjcuXPVo0cPffvb33amxcbGyuv1Nvj91atXa9euXVqzZo2SkpI0YMAAPfTQQ5o2bZpmzZolt9vdrPUDAHAmrD/0/XlVVVV64YUXdMcdd8jlcjnTly5dqs6dO+uSSy5Rbm6ujh8/7rQVFhaqb9++SkpKcqZlZmbK5/Np586dDS6nsrJSPp8vZAAAIBys36P+vBUrVujw4cO67bbbnGk333yzunXrpuTkZG3btk3Tpk1TSUmJli9fLkkqKysLCWlJznhZWVmDy8nLy9Ps2bObZyUAADgNLSqon332WY0YMULJycnOtOzsbOdz37591bVrVw0ZMkQff/yxevTo0ajl5ObmasqUKc64z+dTSkpK4wsHAKCRWsyh73/+859as2aN7rzzzi/tl56eLkn66KOPJEler1fl5eUhferGT3Ze2+PxKC4uLmSwhTFGfr9fxphwlwIAOAtaTFA/99xzSkxM1MiRI7+0X3FxsSSpa9eukqSMjAxt375d+/fvd/rk5+crLi5OvXv3brZ6m0uwpkrZSzYpEAiEuxQAwFnQIg59B4NBPffcc8rKylJU1H9L/vjjj/WHP/xB11xzjTp16qRt27Zp8uTJGjx4sPr16ydJGjZsmHr37q1bb71V8+bNU1lZmaZPn66cnBx5PJ5wrdIZiYxumXUDAE5fiwjqNWvWaO/evbrjjjtCprvdbq1Zs0aPP/64jh07ppSUFI0ePVrTp093+kRGRmrlypW65557lJGRoXPOOUdZWVkh910DAGCrFhHUw4YNa/CcbEpKijZs2HDK73fr1k2vv/56c5QGAECzajHnqAEAaIsIagAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygboGMMfL7/TLGhLsUAEAzI6hboGBNlbIWFSgQCIS7FABAMyOoW6jIaE+4SwAAnAUENQAAFiOoAQCwGEENAIDFCGoAACxGULdQtdWV8vv94S4DANDMCGoAACxGUAMAYDGCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1C2UMUZ+v1/GmHCXAgBoRgR1CxWsqVL2kk0KBALhLgUA0IwI6hYsMtoT7hIAAM2MoAYAwGIENQAAFiOow6DuQjAAAE7F6qCeNWuWXC5XyHDxxRc77YFAQDk5OerUqZPOPfdcjR49WuXl5SHz2Lt3r0aOHKnY2FglJiZq6tSpqqmpOdurEiIQCOjWX6+VMcGw1gEAsF9UuAs4lT59+mjNmjXOeFTUf0uePHmyXnvtNb388suKj4/XxIkTdf311+vtt9+WJNXW1mrkyJHyer3auHGjSktLNW7cOEVHR+uRRx456+vyeZHRHtVUndledd2eeUxMjFwuVxNVBgCwidV71NKJYPZ6vc7QuXNnSVJFRYWeffZZLViwQN/5zneUlpam5557Ths3btQ777wjSVq9erV27dqlF154QQMGDNCIESP00EMPaeHChaqqqgrnajWJYE2VshYVcIsWALRi1gf1hx9+qOTkZF1wwQUaO3as9u7dK0kqKipSdXW1hg4d6vS9+OKLdf7556uwsFCSVFhYqL59+yopKcnpk5mZKZ/Pp507d550mZWVlfL5fCGDrSKi3Dz4BABaMauDOj09XUuWLNGqVav09NNPa/fu3Ro0aJCOHDmisrIyud1uJSQkhHwnKSlJZWVlkqSysrKQkK5rr2s7mby8PMXHxztDSkpK065YE2KvGgBaN6vPUY8YMcL53K9fP6Wnp6tbt2764x//qHbt2jXbcnNzczVlyhRn3OfzWR3WPPgEAFovq/eovyghIUE9e/bURx99JK/Xq6qqKh0+fDikT3l5ubxeryTJ6/XWuwq8bryuT0M8Ho/i4uJCBgAAwqFFBfXRo0f18ccfq2vXrkpLS1N0dLTWrl3rtJeUlGjv3r3KyMiQJGVkZGj79u3av3+/0yc/P19xcXHq3bv3Wa8fAIDTZfWh7x//+McaNWqUunXrpk8//VQzZ85UZGSkbrrpJsXHx2v8+PGaMmWKOnbsqLi4OE2aNEkZGRn6xje+IUkaNmyYevfurVtvvVXz5s1TWVmZpk+frpycHHk8HC4GANjP6qD+17/+pZtuukkHDhxQly5ddOWVV+qdd95Rly5dJEmPPfaYIiIiNHr0aFVWViozM1O//vWvne9HRkZq5cqVuueee5SRkaFzzjlHWVlZmjNnTrhWCQCA0+Iy3NdzSj6fT/Hx8aqoqGiS89V+v183Ppmvmiq/XBHRMsHqBv+NiIxQbXXlKftERnv04v9e1awX2AEAwqNFnaMGAKCtIagBALAYQQ0AgMUIagAALEZQtwK11ZW83xoAWimCGgAAixHUAABYjKAGAMBiBDUAABYjqAEAsBhBDQCAxQhqAAAsRlC3AsYY+f1+8X4VAGh9COpWIFhTpewlmxQIBMJdCgCgiRHUrURElJu9agBohQjqViJYU6WsRQXsVQNAK0NQtyLsVQNA60NQtyLsVQNA60NQtyK11ZVyRUSHuwwAQBMiqAEAsBhBDQCAxQhqAAAsRlADAGAxghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYIaAACLEdStkN/vl9/vD3cZAIAmEBXuAtC0jDE6fvy4jDGKiYmRy+UKd0kAgDPAHnUrE6yp0vhF6zXumQ0KBALhLgcAcIYI6lYoMtqtyGhPuMsAADQBghoAAIsR1AAAWIygBgDAYgQ1AAAWI6gBALAYQQ0AgMUIagAALEZQAwBgMYK6lTLGyO/3yxgT7lIAAGfA6qDOy8vT5Zdfrvbt2ysxMVHXXXedSkpKQvpcddVVcrlcIcPdd98d0mfv3r0aOXKkYmNjlZiYqKlTp6qmpuZsrspZF6ypUtaiAgUCAV7SAQAtmNUv5diwYYNycnJ0+eWXq6amRj/72c80bNgw7dq1S+ecc47Tb8KECZozZ44zHhsb63yura3VyJEj5fV6tXHjRpWWlmrcuHGKjo7WI488clbX52zjMaIA0PJZHdSrVq0KGV+yZIkSExNVVFSkwYMHO9NjY2Pl9XobnMfq1au1a9curVmzRklJSRowYIAeeughTZs2TbNmzZLb7W7WdQAA4ExYfej7iyoqKiRJHTt2DJm+dOlSde7cWZdccolyc3N1/Phxp62wsFB9+/ZVUlKSMy0zM1M+n087d+48O4WHCeepAaDls3qP+vOCwaDuv/9+fetb39Ill1ziTL/55pvVrVs3JScna9u2bZo2bZpKSkq0fPlySVJZWVlISEtyxsvKyhpcVmVlpSorK51xn8/X1KtzVtSdp34+e7DatWsX7nIAAI3QYoI6JydHO3bs0FtvvRUyPTs72/nct29fde3aVUOGDNHHH3+sHj16NGpZeXl5mj179hnVa4Pa6kpFec4NdxkAgDPQIg59T5w4UStXrtT69et13nnnfWnf9PR0SdJHH30kSfJ6vSovLw/pUzd+svPaubm5qqiocIZPPvnkTFcBAIBGsTqojTGaOHGiXnnlFa1bt06pqamn/E5xcbEkqWvXrpKkjIwMbd++Xfv373f65OfnKy4uTr17925wHh6PR3FxcSFDS8V5agBo2awO6pycHL3wwgv6wx/+oPbt26usrExlZWXOPcEff/yxHnroIRUVFWnPnj169dVXNW7cOA0ePFj9+vWTJA0bNky9e/fWrbfeqvfff19//etfNX36dOXk5Mjjaf23LwVrqjRh8VsKBALhLgUA0AhWB/XTTz+tiooKXXXVVerataszvPTSS5Ikt9utNWvWaNiwYbr44ov1ox/9SKNHj9af//xnZx6RkZFauXKlIiMjlZGRoVtuuUXjxo0Lue+6teN+agBouay+mOxUh2tTUlK0YcOGU86nW7duev3115uqLAAAzhqr96gBAGjrCGoAACxGUAMAYDGCOgxO3C4VDHcZAIAWgKAGAMBiBHUbwENPAKDlIqjbgGBNlbKXbOKhJwDQAhHUbURElJu9agBogQjqNqLulZfsVQNAy0JQtyE8ShQAWh6Cug3hojIAaHkI6jaEw98A0PIQ1G0Mh78BoGUhqNsYDn8DQMtCULcxHP4GgJaFoG6DOPwNAC0HQd3G1FZXKljLC0EAoKUgqNsgzlMDQMtBULdBwZoqjf3Van366aeENQBYjqBuo1wuFy/qAIAWgKBuw7ioDADsR1ADAGAxghoAAIsR1AAAWIygbsOMMTp48KCOHz8e7lIAACdBULdhwZoqTVj8lvx+P/dVA4ClCOo2LiLKrUOHDmn0o3/mvmoAsBBBfZbVPRVMluRhsKZKE1/YLMlwXzUAWIigPssCgYDGL3pDwaA9z9uuu5+a+6oBwD4EdRhERrvDXQIAoIUgqBGCF3YAgF0IaoQIBAK64YnVOnToEGENABYgqOGo25t2uVzKWlTAhWUAYAGCGo66+6qNCXJhGQBYgqBGiM8HNOerASD8CGqcVCAQ0I1P5nMIHADCKCrcBcA+tdWVkitKfr9fkW4OgQNAOLFHjQYFa6p0x2/Wqba6JtylAECbxh41Tioy2i1jjI4fP65gMCiXy6V27drJ5XKFuzQAaDMIanypYE2Vbv31GkknXuDxu7u+rZiYGCe0AQDNi0Pf+EpcEdFyuVwa95sNzjusuRocAJofe9T4ymqrK+WKiNb4ResV6Y7V7++5Su3atQs5HO73+yWJvW0AaCIENU5bZLRbLpdLt/56jSKjPXrp3mGSTtx3HQgECGkAaEIENRotMtojY4w+/fRT3bdsq4w58erOF3K+q9jYWCe4685pAwBOH+eocUaCNVWa+MJmGROUKyLaCW+/3y+/368bnlitgwcP6tixY5zXBoBGYI8aZywy2iMTrHbGA4GAbvt/b+mZWy9zDpHX9fvjfZkcGgeA09Cm9qgXLlyo7t27KyYmRunp6Xr33XfDXVKrU7c3Hayp0vj/V6CaKr+zp133HPG6e7MPHDigY8eOnegfDOr48eMhe948axwA2lBQv/TSS5oyZYpmzpyp9957T/3791dmZqb2798f7tJalc8fCv/iG7hqqgI6cOCA9u3bpx88+mfdsjBfpaWlGv3on/WPf/xDP1zwmq7/xXL9YMFf9Omnn+rgwYP6wS9X6tChQ5JCXxJCiANoK9pMUC9YsEATJkzQ7bffrt69e+uZZ55RbGysFi9eHO7SWp2TvSIzWFOlO3/7psYvWi9X5In7sie+sFmS0cQXNisiKtq5ovzOZ9/UoUOHFBHldgK57iUhfr9fhw4dcj7X7Yl/9tln+uyzz3Ts2DFnOBvnxfmjAUBzahPnqKuqqlRUVKTc3FxnWkREhIYOHarCwsJ6/SsrK1VZWemMV1RUSJJ8Pt8Z1+L3+1V59LDkipAJ1soVEXnSf10REQrWVJ9xn3DO72R9a6ur6vWprQ6E9JnwzFpFRLqU9au/6ldZ35QkBY4cUklJiX7yxy1SRJRKSkr0o6WFMoqUCdbIFRHl/BsReeLv0IW3D1KHDh2c7S81fJ93Y+8B9/v9uuvZDfrN+G9/pe9yrznQsjXl/7vt27c/9V0xpg3Yt2+fkWQ2btwYMn3q1KnmiiuuqNd/5syZRhIDAwMDA0OzDhUVFafMsDaxR326cnNzNWXKFGc8GAzq4MGD6tSp0xnfD+zz+ZSSkqJPPvlEcXFxZ1oq/oPt2vTYps2D7do8Wup2bd++/Sn7tImg7ty5syIjI1VeXh4yvby8XF6vt15/j8cjjyf0PGtCQkKT1hQXF9eifkwtBdu16bFNmwfbtXm0xu3aJi4mc7vdSktL09q1a51pwWBQa9euVUZGRhgrAwDgy7WJPWpJmjJlirKysnTZZZfpiiuu0OOPP65jx47p9ttvD3dpAACcVJsJ6htvvFH//ve/NWPGDJWVlWnAgAFatWqVkpKSzmodHo9HM2fOrHdoHWeG7dr02KbNg+3aPFrzdnUZw82fAADYqk2cowYAoKUiqAEAsBhBDQCAxQhqAAAsRlCfRbxm878KCgo0atQoJScny+VyacWKFSHtxhjNmDFDXbt2Vbt27TR06FB9+OGHIX0OHjyosWPHKi4uTgkJCRo/fryOHj0a0mfbtm0aNGiQYmJilJKSonnz5tWr5eWXX9bFF1+smJgY9e3bV6+//nqTr+/ZkpeXp8svv1zt27dXYmKirrvuOpWUlIT0CQQCysnJUadOnXTuuedq9OjR9R4GtHfvXo0cOVKxsbFKTEzU1KlTVVNTE9LnjTfe0KWXXiqPx6MLL7xQS5YsqVdPa/jNP/300+rXr5/zII2MjAz95S9/cdrZnk1j7ty5crlcuv/++51pbNv/aJqnaeNUli1bZtxut1m8eLHZuXOnmTBhgklISDDl5eXhLi0sXn/9dfPAAw+Y5cuXG0nmlVdeCWmfO3euiY+PNytWrDDvv/++ufbaa01qaqrx+/1On+HDh5v+/fubd955x7z55pvmwgsvNDfddJPTXlFRYZKSkszYsWPNjh07zIsvvmjatWtnfvOb3zh93n77bRMZGWnmzZtndu3aZaZPn26io6PN9u3bm30bNIfMzEzz3HPPmR07dpji4mJzzTXXmPPPP98cPXrU6XP33XeblJQUs3btWrNlyxbzjW98w3zzm9902mtqaswll1xihg4darZu3Wpef/1107lzZ5Obm+v0+cc//mFiY2PNlClTzK5du8xTTz1lIiMjzapVq5w+reU3/+qrr5rXXnvN/P3vfzclJSXmZz/7mYmOjjY7duwwxrA9m8K7775runfvbvr162fuu+8+Zzrb9gSC+iy54oorTE5OjjNeW1trkpOTTV5eXhirssMXgzoYDBqv12vmz5/vTDt8+LDxeDzmxRdfNMYYs2vXLiPJbN682enzl7/8xbhcLrNv3z5jjDG//vWvTYcOHUxlZaXTZ9q0aaZXr17O+A033GBGjhwZUk96erq56667mnQdw2X//v1GktmwYYMx5sR2jI6ONi+//LLT54MPPjCSTGFhoTHmxB9RERERpqyszOnz9NNPm7i4OGdb/uQnPzF9+vQJWdaNN95oMjMznfHW/Jvv0KGD+e1vf8v2bAJHjhwxF110kcnPzzff/va3naBm2/4Xh77PgrrXbA4dOtSZ9mWv2Wzrdu/erbKyspDtFR8fr/T0dGd7FRYWKiEhQZdddpnTZ+jQoYqIiNCmTZucPoMHD5bb7Xb6ZGZmqqSkRIcOHXL6fH45dX1ay3+Xule0duzYUZJUVFSk6urqkHW++OKLdf7554ds2759+4Y8DCgzM1M+n087d+50+nzZdmutv/na2lotW7ZMx44dU0ZGBtuzCeTk5GjkyJH11p9t+19t5slk4fTZZ5+ptra23lPQkpKS9Le//S1MVdmrrKxMkhrcXnVtZWVlSkxMDGmPiopSx44dQ/qkpqbWm0ddW4cOHVRWVvaly2nJgsGg7r//fn3rW9/SJZdcIunEervd7novmfnitm1om9S1fVkfn88nv9+vQ4cOtarf/Pbt25WRkaFAIKBzzz1Xr7zyinr37q3i4mK25xlYtmyZ3nvvPW3evLleG7/V/yKogVYqJydHO3bs0FtvvRXuUlq8Xr16qbi4WBUVFfq///s/ZWVlacOGDeEuq0X75JNPdN999yk/P18xMTHhLsdqHPo+C073NZttXd02+bLt5fV6tX///pD2mpoaHTx4MKRPQ/P4/DJO1qel/3eZOHGiVq5cqfXr1+u8885zpnu9XlVVVenw4cMh/b+4bRu73eLi4tSuXbtW95t3u9268MILlZaWpry8PPXv319PPPEE2/MMFBUVaf/+/br00ksVFRWlqKgobdiwQU8++aSioqKUlJTEtv0Pgvos4DWbpyc1NVVerzdke/l8Pm3atMnZXhkZGTp8+LCKioqcPuvWrVMwGFR6errTp6CgQNXV1U6f/Px89erVSx06dHD6fH45dX1a6n8XY4wmTpyoV155RevWrat36D8tLU3R0dEh61xSUqK9e/eGbNvt27eH/CGUn5+vuLg49e7d2+nzZduttf/mg8GgKisr2Z5nYMiQIdq+fbuKi4ud4bLLLtPYsWOdz2zb/wj31WxtxbJly4zH4zFLliwxu3btMtnZ2SYhISHkasW25MiRI2br1q1m69atRpJZsGCB2bp1q/nnP/9pjDlxe1ZCQoL505/+ZLZt22a+973vNXh71sCBA82mTZvMW2+9ZS666KKQ27MOHz5skpKSzK233mp27Nhhli1bZmJjY+vdnhUVFWUeffRR88EHH5iZM2e26Nuz7rnnHhMfH2/eeOMNU1pa6gzHjx93+tx9993m/PPPN+vWrTNbtmwxGRkZJiMjw2mvu+Vl2LBhpri42Kxatcp06dKlwVtepk6daj744AOzcOHCBm95aQ2/+Z/+9Kdmw4YNZvfu3Wbbtm3mpz/9qXG5XGb16tXGGLZnU/r8Vd/GsG3rENRn0VNPPWXOP/9843a7zRVXXGHeeeedcJcUNuvXrzeS6g1ZWVnGmBO3aD344IMmKSnJeDweM2TIEFNSUhIyjwMHDpibbrrJnHvuuSYuLs7cfvvt5siRIyF93n//fXPllVcaj8djvva1r5m5c+fWq+WPf/yj6dmzp3G73aZPnz7mtddea7b1bm4NbVNJ5rnnnnP6+P1+87//+7+mQ4cOJjY21nz/+983paWlIfPZs2ePGTFihGnXrp3p3Lmz+dGPfmSqq6tD+qxfv94MGDDAuN1uc8EFF4Qso05r+M3fcccdplu3bsbtdpsuXbqYIUOGOCFtDNuzKX0xqNm2J/CaSwAALMY5agAALEZQAwBgMYIaAACLEdQAAFiMoAYAwGIENQAAFiOoAQCwGEENAIDFCGoAACxGUAMAYDGCGgAAixHUAABY7P8Dr96n7VLFKjwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 500x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    string_lengths = [len(string) for string in value['document']]\n",
    "    sns.displot(string_lengths)\n",
    "    plt.title(key)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of emojis"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - sentenctes per document:\n",
      "count    1555030\n",
      "mean          16\n",
      "std           15\n",
      "min            1\n",
      "25%            7\n",
      "50%           11\n",
      "75%           19\n",
      "max          976\n",
      "Name: ext_labels, dtype: object\n",
      "val - sentenctes per document:\n",
      "count    80747\n",
      "mean        16\n",
      "std         15\n",
      "min          1\n",
      "25%          7\n",
      "50%         12\n",
      "75%         20\n",
      "max        493\n",
      "Name: ext_labels, dtype: object\n",
      "test - sentenctes per document:\n",
      "count    40413\n",
      "mean        16\n",
      "std         16\n",
      "min          1\n",
      "25%          7\n",
      "50%         12\n",
      "75%         20\n",
      "max        493\n",
      "Name: ext_labels, dtype: object\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    print(f\"{key} - sentenctes per document:\")\n",
    "    sentences = value['ext_labels'].str.len()\n",
    "    print(sentences.describe().apply(\"{0:.0f}\".format))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Number of words"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Le principali statistiche riguardo il numero di parole per post"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for key, value in corpus.items():\n",
    "#     if (key != \"train\"):\n",
    "#         print(f\"{key.upper()} - words stats\")\n",
    "#         # Calcola il numero di parole inglesi in ogni stringa contenuta in 'document'\n",
    "#         english_word_counts = []\n",
    "#         for string in value['document']:\n",
    "#             # Divide la stringa in parole\n",
    "#             words = nltk.word_tokenize(string)\n",
    "#             # Conta solo le parole che non sono stopwords\n",
    "#             english_word_counts.append(len([word for word in words if word.lower() not in stop_words_en]))\n",
    "\n",
    "#         # Utilizza displot di seaborn per plottare la distribuzione del numero di parole inglesi\n",
    "#         # sns.displot(english_word_counts)\n",
    "        \n",
    "#         stats = statistics.describe(english_word_counts)\n",
    "\n",
    "#         # Stampa le statistiche\n",
    "#         print(\"Number of elements:\", stats.n)\n",
    "#         print(\"Min:\", stats.minmax[0])\n",
    "#         print(\"Max:\", stats.minmax[1])\n",
    "#         print(\"Mean:\", stats.mean)\n",
    "#         print(\"Median:\", stats.median)\n",
    "#         print(\"Standard deviation:\", stats.stdev)\n",
    "#         print(\"First quartile:\", stats.quartiles[0])\n",
    "#         print(\"Third quartile:\", stats.quartiles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing tkn_test.json...\n",
      "Processing tkn_train_1.json...\n",
      "Processing tkn_train_10.json...\n",
      "Processing tkn_train_2.json...\n",
      "Processing tkn_train_3.json...\n",
      "Processing tkn_train_4.json...\n",
      "Processing tkn_train_5.json...\n",
      "Processing tkn_train_6.json...\n",
      "Processing tkn_train_7.json...\n",
      "Processing tkn_train_8.json...\n",
      "Processing tkn_train_9.json...\n",
      "Processing tkn_val.json...\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from itertools import chain\n",
    "\n",
    "# Initialize empty lists to store the results\n",
    "unique_words_doc = []\n",
    "num_sentences_doc = []\n",
    "total_words_doc = []\n",
    "avg_words_per_sentence_doc = []\n",
    "\n",
    "unique_words_sum = []\n",
    "num_sentences_sum = []\n",
    "total_words_sum = []\n",
    "avg_words_per_sentence_sum = []\n",
    "\n",
    "def func(x):\n",
    "    return list(set(chain.from_iterable(x)))\n",
    "\n",
    "# Iterate through all files in the directory\n",
    "for file in os.listdir('./ProcessedData/'):\n",
    "    # Check if the file name contains \"tkn\"\n",
    "    if \"tkn\" in file:\n",
    "        # Read the JSON file into a dataframe\n",
    "        df = pd.read_json(f'./ProcessedData/{file}', orient='records', lines=True)\n",
    "        print(f\"Processing {file}...\")\n",
    "        \n",
    "        # Get a list of unique words from the \"document_normalized\" column\n",
    "        unique_words_doc += func(df['document_normalized'].map(func).tolist())\n",
    "        \n",
    "        # Get a list of number of sentences per row from the \"document_normalized\" column\n",
    "        num_sentences_doc += df['document_normalized'].map(lambda x: len(x)).tolist()\n",
    "        \n",
    "        # Get a list of total number of words per row from the \"document_normalized\" column\n",
    "        total_words_doc += df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)).tolist()\n",
    "        \n",
    "        # Get a list of average number of words per sentence per row from the \"document_normalized\" column\n",
    "        avg_words_per_sentence_doc += df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)/len(x)).tolist()\n",
    "        \n",
    "        \n",
    "        # Get a list of unique words from the \"summary_normalized\" column\n",
    "        unique_words_sum += func(df['summary_normalized'].map(func).tolist())\n",
    "        \n",
    "        # Get a list of number of sentences per row from the \"summary_normalized\" column\n",
    "        num_sentences_sum += df['summary_normalized'].map(lambda x: len(x)).tolist()\n",
    "        \n",
    "        # Get a list of total number of words per row from the \"summary_normalized\" column\n",
    "        total_words_sum += df['summary_normalized'].map(lambda x: sum(len(sent) for sent in x)).tolist()\n",
    "        \n",
    "        # Get a list of average number of words per sentence per row from the \"summary_normalized\" column\n",
    "        avg_words_per_sentence_sum += df['summary_normalized'].map(lambda x: sum(len(sent) for sent in x)/len(x)).tolist()\n",
    "\n",
    "# Round all elements in the list to 2 decimal places\n",
    "avg_words_per_sentence_sum = [round(num, 2) for num in avg_words_per_sentence_sum]\n",
    "avg_words_per_sentence_doc = [round(num, 2) for num in avg_words_per_sentence_doc]\n",
    "# Get unique words from the concatenated list of unique words\n",
    "unique_words_doc = list(set(unique_words_doc))\n",
    "unique_words_sum = list(set(unique_words_sum))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of total words in document_normalized:  468805750\n",
      "Number of total words in summary_normalized:  38140312\n",
      "Number of average words per row in document_normalized:  291.8191726828734\n",
      "Number of average words per row in summary_normalized:  23.741334857148548\n",
      "Number of total sentences in document_normalized:  24922429\n",
      "Number of total sentences in summary_normalized:  3752801\n",
      "Number of average sentences per row in document_normalized:  15.513552493815725\n",
      "Number of average sentences per row in summary_normalized:  2.3360193066391783\n",
      "Number of average words per sentence in document_normalized:  20.089549802240537\n",
      "Number of average words per sentence in summary_normalized:  11.276562993699482\n",
      "Number of total unique words in document_normalized:  737813\n",
      "Number of total unique words in summary_normalized:  253807\n"
     ]
    }
   ],
   "source": [
    "# Print the results\n",
    "print(\"Number of total words in document_normalized: \", sum(total_words_doc))\n",
    "print(\"Number of total words in summary_normalized: \", sum(total_words_sum))\n",
    "print(\"Number of average words per row in document_normalized: \", sum(total_words_doc)/len(total_words_doc))\n",
    "print(\"Number of average words per row in summary_normalized: \", sum(total_words_sum)/len(total_words_sum))\n",
    "print(\"Number of total sentences in document_normalized: \", sum(num_sentences_doc))\n",
    "print(\"Number of total sentences in summary_normalized: \", sum(num_sentences_sum))\n",
    "print(\"Number of average sentences per row in document_normalized: \", sum(num_sentences_doc)/len(num_sentences_doc))\n",
    "print(\"Number of average sentences per row in summary_normalized: \", sum(num_sentences_sum)/len(num_sentences_sum))\n",
    "print(\"Number of average words per sentence in document_normalized: \", sum(avg_words_per_sentence_doc)/len(avg_words_per_sentence_doc))\n",
    "print(\"Number of average words per sentence in summary_normalized: \", sum(avg_words_per_sentence_sum)/len(avg_words_per_sentence_sum))\n",
    "print(\"Number of total unique words in document_normalized: \", len(unique_words_doc))\n",
    "print(\"Number of total unique words in summary_normalized: \", len(unique_words_sum))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lemmatized:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing test.json...\n",
      "Processing train_1.json...\n",
      "Processing train_10.json...\n",
      "Processing train_2.json...\n",
      "Processing train_3.json...\n",
      "Processing train_4.json...\n",
      "Processing train_5.json...\n",
      "Processing train_6.json...\n",
      "Processing train_7.json...\n",
      "Processing train_8.json...\n",
      "Processing train_9.json...\n",
      "Processing val.json...\n"
     ]
    }
   ],
   "source": [
    "# Initialize empty lists to store the results\n",
    "unique_words_doc = []\n",
    "num_sentences_doc = []\n",
    "total_words_doc = []\n",
    "avg_words_per_sentence_doc = []\n",
    "\n",
    "def func(x):\n",
    "    return list(set(chain.from_iterable(x)))\n",
    "\n",
    "# Iterate through all files in the directory\n",
    "for file in os.listdir('./ProcessedData/'):\n",
    "    # Check if the file name contains \"tkn\"\n",
    "    if \"tkn\" not in file:\n",
    "        # Read the JSON file into a dataframe\n",
    "        df = pd.read_json(f'./ProcessedData/{file}', orient='records', lines=True)[['document_normalized']]\n",
    "        print(f\"Processing {file}...\")\n",
    "        \n",
    "        # Get a list of unique words from the \"document_normalized\" column\n",
    "        unique_words_doc += func(df['document_normalized'].map(func).tolist())\n",
    "        \n",
    "        # Get a list of number of sentences per row from the \"document_normalized\" column\n",
    "        num_sentences_doc += df['document_normalized'].map(lambda x: len(x)).tolist()\n",
    "        \n",
    "        # Get a list of total number of words per row from the \"document_normalized\" column\n",
    "        total_words_doc += df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)).tolist()\n",
    "        \n",
    "        # Get a list of average number of words per sentence per row from the \"document_normalized\" column\n",
    "        avg_words_per_sentence_doc += df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)/len(x)).tolist()\n",
    "\n",
    "# Round all elements in the list to 2 decimal places\n",
    "avg_words_per_sentence_doc = [round(num, 2) for num in avg_words_per_sentence_doc]\n",
    "# Get unique words from the concatenated list of unique words\n",
    "unique_words_doc = list(set(unique_words_doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of total words in document_lemmatized:  213959019\n",
      "Number of average words per row in document_lemmatized:  133.18382701709436\n",
      "Number of total sentences in document_lemmatized:  24922429\n",
      "Number of average sentences per row in document_lemmatized:  15.513552493815725\n",
      "Number of average words per sentence in document_lemmatized:  9.304792809683672\n",
      "Number of total unique words in document_lemmatized:  714892\n"
     ]
    }
   ],
   "source": [
    "# Print the results\n",
    "print(\"Number of total words in document_lemmatized: \", sum(total_words_doc))\n",
    "print(\"Number of average words per row in document_lemmatized: \", sum(total_words_doc)/len(total_words_doc))\n",
    "print(\"Number of total sentences in document_lemmatized: \", sum(num_sentences_doc))\n",
    "print(\"Number of average sentences per row in document_lemmatized: \", sum(num_sentences_doc)/len(num_sentences_doc))\n",
    "print(\"Number of average words per sentence in document_lemmatized: \", sum(avg_words_per_sentence_doc)/len(avg_words_per_sentence_doc))\n",
    "print(\"Number of total unique words in document_lemmatized: \", len(unique_words_doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ciai', 'ciao', 'gi', 'ciau']\n",
      "['ciao', 'ciai', 'ciau', 'gi']\n",
      "[3, 2]\n",
      "[14, 10]\n",
      "[4.666666666666667, 5.0]\n",
      "[4.7, 5.0]\n"
     ]
    }
   ],
   "source": [
    "# from itertools import chain\n",
    "\n",
    "# def func(x):\n",
    "#     return list(set(chain.from_iterable(x)))\n",
    "\n",
    "# data = {'document_normalized': [[['ciao', 'ciao', 'gi', 'ciao', 'ciao', 'ciao'], ['ciao', 'ciao', 'ciao', 'ciau'], ['ciao', 'ciao', 'ciao', 'ciau']], [['ciao', 'ciao', 'ciao', 'ciao', 'ciai', 'ciao'], ['ciao', 'ciao', 'ciao', 'ciau']]]}\n",
    "# df = pd.DataFrame(data)\n",
    "# print(func(df['document_normalized'].map(func).tolist()))\n",
    "# print(list(set(func(df['document_normalized'].map(func).tolist()))))\n",
    "# print(df['document_normalized'].map(lambda x: len(x)).tolist())\n",
    "# print(df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)).tolist())\n",
    "# print(df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)/len(x)).tolist())\n",
    "# print([round(num, 1) for num in df['document_normalized'].map(lambda x: sum(len(sent) for sent in x)/len(x)).tolist()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ciau', 'ciao']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = [[\"ciao\", \"ciao\", \"ciao\", \"ciao\", \"ciao\", \"ciao\"], [\"ciao\", \"ciao\", \"ciao\", \"ciau\"]]\n",
    "list(set(chain.from_iterable(test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "1 columns passed, passed data had 2 columns",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\internals\\construction.py:969\u001b[0m, in \u001b[0;36m_finalize_columns_and_data\u001b[1;34m(content, columns, dtype)\u001b[0m\n\u001b[0;32m    968\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 969\u001b[0m     columns \u001b[39m=\u001b[39m _validate_or_indexify_columns(contents, columns)\n\u001b[0;32m    970\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mAssertionError\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m    971\u001b[0m     \u001b[39m# GH#26429 do not raise user-facing AssertionError\u001b[39;00m\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\internals\\construction.py:1017\u001b[0m, in \u001b[0;36m_validate_or_indexify_columns\u001b[1;34m(content, columns)\u001b[0m\n\u001b[0;32m   1015\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m is_mi_list \u001b[39mand\u001b[39;00m \u001b[39mlen\u001b[39m(columns) \u001b[39m!=\u001b[39m \u001b[39mlen\u001b[39m(content):  \u001b[39m# pragma: no cover\u001b[39;00m\n\u001b[0;32m   1016\u001b[0m     \u001b[39m# caller's responsibility to check for this...\u001b[39;00m\n\u001b[1;32m-> 1017\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mAssertionError\u001b[39;00m(\n\u001b[0;32m   1018\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mlen\u001b[39m(columns)\u001b[39m}\u001b[39;00m\u001b[39m columns passed, passed data had \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m   1019\u001b[0m         \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mlen\u001b[39m(content)\u001b[39m}\u001b[39;00m\u001b[39m columns\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m   1020\u001b[0m     )\n\u001b[0;32m   1021\u001b[0m \u001b[39melif\u001b[39;00m is_mi_list:\n\u001b[0;32m   1022\u001b[0m \n\u001b[0;32m   1023\u001b[0m     \u001b[39m# check if nested list column, length of each sub-list should be equal\u001b[39;00m\n",
      "\u001b[1;31mAssertionError\u001b[0m: 1 columns passed, passed data had 2 columns",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 5\u001b[0m\n\u001b[0;32m      1\u001b[0m data \u001b[39m=\u001b[39m [[[[\u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m], [\u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mciau\u001b[39m\u001b[39m\"\u001b[39m]],\n\u001b[0;32m      2\u001b[0m         [[\u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciai\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m], [\u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciao\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mciau\u001b[39m\u001b[39m'\u001b[39m]]]]\n\u001b[0;32m      4\u001b[0m \u001b[39m# Crea il dataframe\u001b[39;00m\n\u001b[1;32m----> 5\u001b[0m df \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mDataFrame(data, columns\u001b[39m=\u001b[39;49m[\u001b[39m\"\u001b[39;49m\u001b[39mdocument_normalized\u001b[39;49m\u001b[39m\"\u001b[39;49m])\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\frame.py:745\u001b[0m, in \u001b[0;36mDataFrame.__init__\u001b[1;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[0;32m    743\u001b[0m     \u001b[39mif\u001b[39;00m columns \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    744\u001b[0m         columns \u001b[39m=\u001b[39m ensure_index(columns)\n\u001b[1;32m--> 745\u001b[0m     arrays, columns, index \u001b[39m=\u001b[39m nested_data_to_arrays(\n\u001b[0;32m    746\u001b[0m         \u001b[39m# error: Argument 3 to \"nested_data_to_arrays\" has incompatible\u001b[39;49;00m\n\u001b[0;32m    747\u001b[0m         \u001b[39m# type \"Optional[Collection[Any]]\"; expected \"Optional[Index]\"\u001b[39;49;00m\n\u001b[0;32m    748\u001b[0m         data,\n\u001b[0;32m    749\u001b[0m         columns,\n\u001b[0;32m    750\u001b[0m         index,  \u001b[39m# type: ignore[arg-type]\u001b[39;49;00m\n\u001b[0;32m    751\u001b[0m         dtype,\n\u001b[0;32m    752\u001b[0m     )\n\u001b[0;32m    753\u001b[0m     mgr \u001b[39m=\u001b[39m arrays_to_mgr(\n\u001b[0;32m    754\u001b[0m         arrays,\n\u001b[0;32m    755\u001b[0m         columns,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    758\u001b[0m         typ\u001b[39m=\u001b[39mmanager,\n\u001b[0;32m    759\u001b[0m     )\n\u001b[0;32m    760\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\internals\\construction.py:510\u001b[0m, in \u001b[0;36mnested_data_to_arrays\u001b[1;34m(data, columns, index, dtype)\u001b[0m\n\u001b[0;32m    507\u001b[0m \u001b[39mif\u001b[39;00m is_named_tuple(data[\u001b[39m0\u001b[39m]) \u001b[39mand\u001b[39;00m columns \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    508\u001b[0m     columns \u001b[39m=\u001b[39m ensure_index(data[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39m_fields)\n\u001b[1;32m--> 510\u001b[0m arrays, columns \u001b[39m=\u001b[39m to_arrays(data, columns, dtype\u001b[39m=\u001b[39;49mdtype)\n\u001b[0;32m    511\u001b[0m columns \u001b[39m=\u001b[39m ensure_index(columns)\n\u001b[0;32m    513\u001b[0m \u001b[39mif\u001b[39;00m index \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\internals\\construction.py:875\u001b[0m, in \u001b[0;36mto_arrays\u001b[1;34m(data, columns, dtype)\u001b[0m\n\u001b[0;32m    872\u001b[0m     data \u001b[39m=\u001b[39m [\u001b[39mtuple\u001b[39m(x) \u001b[39mfor\u001b[39;00m x \u001b[39min\u001b[39;00m data]\n\u001b[0;32m    873\u001b[0m     arr \u001b[39m=\u001b[39m _list_to_arrays(data)\n\u001b[1;32m--> 875\u001b[0m content, columns \u001b[39m=\u001b[39m _finalize_columns_and_data(arr, columns, dtype)\n\u001b[0;32m    876\u001b[0m \u001b[39mreturn\u001b[39;00m content, columns\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\internals\\construction.py:972\u001b[0m, in \u001b[0;36m_finalize_columns_and_data\u001b[1;34m(content, columns, dtype)\u001b[0m\n\u001b[0;32m    969\u001b[0m     columns \u001b[39m=\u001b[39m _validate_or_indexify_columns(contents, columns)\n\u001b[0;32m    970\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mAssertionError\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m    971\u001b[0m     \u001b[39m# GH#26429 do not raise user-facing AssertionError\u001b[39;00m\n\u001b[1;32m--> 972\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(err) \u001b[39mfrom\u001b[39;00m \u001b[39merr\u001b[39;00m\n\u001b[0;32m    974\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(contents) \u001b[39mand\u001b[39;00m contents[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mdtype \u001b[39m==\u001b[39m np\u001b[39m.\u001b[39mobject_:\n\u001b[0;32m    975\u001b[0m     contents \u001b[39m=\u001b[39m _convert_object_array(contents, dtype\u001b[39m=\u001b[39mdtype)\n",
      "\u001b[1;31mValueError\u001b[0m: 1 columns passed, passed data had 2 columns"
     ]
    }
   ],
   "source": [
    "data = [[[[\"ciao\", \"ciao\", \"ciao\", \"ciao\", \"ciao\", \"ciao\"], [\"ciao\", \"ciao\", \"ciao\", \"ciau\"]],\n",
    "        [['ciao', 'ciao', 'ciao', 'ciao', 'ciai', 'ciao'], ['ciao', 'ciao', 'ciao', 'ciau']]]]\n",
    "\n",
    "# Crea il dataframe\n",
    "df = pd.DataFrame(data, columns=[\"document_normalized\"])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Data Cleaning"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.0 Analisi degli split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = [os.path.join(dirpath,f) for (dirpath, dirnames, filenames) in os.walk(\"Dataset_TLDRHQ\\\\\") for f in filenames]\n",
    "#files = files[0:100] # subset composed of train, val, test\n",
    "\n",
    "dataset = pd.DataFrame()\n",
    "\n",
    "for file in files:\n",
    "    temp = pd.read_json(file, lines=True)\n",
    "    #temp.set_index(\"id\", inplace=True)\n",
    "    dataset = pd.concat([dataset, temp])\n",
    "    \n",
    "dataset.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>summary</th>\n",
       "      <th>ext_labels</th>\n",
       "      <th>rg_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>train-TLDR_RS_2019-02-45730.json</td>\n",
       "      <td>so i just had the thought of what if people co...</td>\n",
       "      <td>name a new eve every 2 - 5 generations so that...</td>\n",
       "      <td>[0, 0, 1, 0, 0]</td>\n",
       "      <td>[0.145231701293733, 0.082184056849748, 0.47712...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>train-TLDR_RC_2011-05-cm-12238.json</td>\n",
       "      <td>this is kind of an interesting topic for me , ...</td>\n",
       "      <td>my mom is crazy , hates me , wishes she would ...</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "      <td>[0.053637180715166, 0.025363965159833003, 0.02...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>train-TLDR_RC_2020-06-04-cm-1657.json</td>\n",
       "      <td>i was running a campaign for my friends and fa...</td>\n",
       "      <td>the alchemist gave the warlock cocaine to keep...</td>\n",
       "      <td>[0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.0, 0.096297655087828, 0.17063644555607402, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>train-TLDR_RC_2019-12-cm-45553.json</td>\n",
       "      <td>i ve tried and failed at a lot of things .&lt;/s&gt;...</td>\n",
       "      <td>is this dumb fuckin juul is the only thing tha...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.027738905810231, 0.066785729204345, 0.03698...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>train-TLDR_RS_2013-04-21129.json</td>\n",
       "      <td>( sorry for the wall of text , i 'm upset and ...</td>\n",
       "      <td>my bf believes that we are eventually going to...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 1, 0]</td>\n",
       "      <td>[0.294148746317319, 0.070204034502615, 0.02593...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index                                     id  \\\n",
       "0      0       train-TLDR_RS_2019-02-45730.json   \n",
       "1      1    train-TLDR_RC_2011-05-cm-12238.json   \n",
       "2      2  train-TLDR_RC_2020-06-04-cm-1657.json   \n",
       "3      3    train-TLDR_RC_2019-12-cm-45553.json   \n",
       "4      4       train-TLDR_RS_2013-04-21129.json   \n",
       "\n",
       "                                            document  \\\n",
       "0  so i just had the thought of what if people co...   \n",
       "1  this is kind of an interesting topic for me , ...   \n",
       "2  i was running a campaign for my friends and fa...   \n",
       "3  i ve tried and failed at a lot of things .</s>...   \n",
       "4  ( sorry for the wall of text , i 'm upset and ...   \n",
       "\n",
       "                                             summary  \\\n",
       "0  name a new eve every 2 - 5 generations so that...   \n",
       "1  my mom is crazy , hates me , wishes she would ...   \n",
       "2  the alchemist gave the warlock cocaine to keep...   \n",
       "3  is this dumb fuckin juul is the only thing tha...   \n",
       "4  my bf believes that we are eventually going to...   \n",
       "\n",
       "                                          ext_labels  \\\n",
       "0                                    [0, 0, 1, 0, 0]   \n",
       "1  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...   \n",
       "2                                    [0, 0, 0, 1, 0]   \n",
       "3      [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "4                     [1, 0, 0, 0, 0, 0, 0, 0, 1, 0]   \n",
       "\n",
       "                                           rg_labels  \n",
       "0  [0.145231701293733, 0.082184056849748, 0.47712...  \n",
       "1  [0.053637180715166, 0.025363965159833003, 0.02...  \n",
       "2  [0.0, 0.096297655087828, 0.17063644555607402, ...  \n",
       "3  [0.027738905810231, 0.066785729204345, 0.03698...  \n",
       "4  [0.294148746317319, 0.070204034502615, 0.02593...  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = dataset.duplicated(subset = [\"rg_labels\"]).sum()\n",
    "print(f\"number of duplicated documents:{count}/{dataset.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1671099/1671099)\n",
      "(1632980/1671099)\n"
     ]
    }
   ],
   "source": [
    "print(f\"({len(dataset['id'].unique())}/{dataset.shape[0]})\")\n",
    "print(f\"({len(dataset['document'].unique())}/{dataset.shape[0]})\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Duplicati"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Assenza duplicati in id e documenti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of unique id:\n",
      "(1590132/1590132)\n",
      "train - number of unique documents:\n",
      "1555030\n",
      "val - number of unique id:\n",
      "(40481/40481)\n",
      "val - number of unique documents:\n",
      "40399\n",
      "test - number of unique id:\n",
      "(40486/40486)\n",
      "test - number of unique documents:\n",
      "40413\n"
     ]
    }
   ],
   "source": [
    "# exploring train, val and test documents\n",
    "# train[\"document\"][0][1:200]\n",
    "for key, value in corpus.items():\n",
    "    print(f\"{key} - number of unique id:\")\n",
    "    print(f\"({len(value['id'].unique())}/{value.shape[0]})\")\n",
    "    print(f\"{key} - number of unique documents:\")\n",
    "    print(len(value['document'].unique()))\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Possibile presenza di duplicati in `document`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of duplicated documents:35102/1590132\n",
      "val - number of duplicated documents:82/40481\n",
      "test - number of duplicated documents:73/40486\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    count = value.duplicated(subset = [\"document\"]).sum()\n",
    "    print(f\"{key} - number of duplicated documents:{count}/{value.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key\n",
      "                                                document  count\n",
      "5760   hey there , i 'm a bot and something you said ...    959\n",
      "26242  your post has been removed for the following r...    477\n",
      "26243  your post has been temporarily removed for the...    116\n",
      "70     # let 's make a plan !</s><s> this week 's kic...    113\n",
      "2784   basically it goes a little like this ... i bou...    106\n",
      "key\n",
      "                                              document  count\n",
      "37   hey there , i 'm a bot and something you said ...     43\n",
      "121  your post has been removed for the following r...     24\n",
      "20   basically it goes a little like this ... i bou...      5\n",
      "31   heads up , there are troubling reports of pack...      4\n",
      "122  your post has been temporarily removed for the...      4\n",
      "key\n",
      "                                             document  count\n",
      "14  hey there , i 'm a bot and something you said ...     22\n",
      "37  your post has been removed for the following r...      9\n",
      "3   ** hello , my name is rene i m a professional ...      2\n",
      "1   # let 's make a plan !</s><s> this week 's kic...      2\n",
      "7   basically it goes a little like this ... i bou...      2\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "for key, value in corpus.items():\n",
    "    print(\"key\")\n",
    "    # generate a random start index\n",
    "    duplicate = value[value.duplicated(subset = [\"document\"])][['document', 'id']]\n",
    "\n",
    "    # printa radomici\n",
    "    #start_index = random.randint(0, len(duplicate_test) - 5)\n",
    "    #duplicate_test.iloc[start_index:start_index+5]\n",
    "\n",
    "    df_grouped = duplicate.groupby('document').size().reset_index(name='count')\n",
    "    df_sorted = df_grouped.sort_values(by='count', ascending=False)\n",
    "    print(df_sorted.head(5))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sembra che questi duplicati siano post ricorrenti / messaggi di bot / post automatici (tipo presentazione, rimozione post):\n",
    "- Procedo alla rimozione"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of duplicated documents:0/1555030\n",
      "val - number of duplicated documents:0/40399\n",
      "test - number of duplicated documents:0/40413\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    # remove duplicates from train, val and test\n",
    "    value.drop_duplicates(subset = [\"document\"], inplace=True)\n",
    "    \n",
    "for key, value in corpus.items():\n",
    "    count = value.duplicated(subset = [\"document\"]).sum()\n",
    "    print(f\"{key} - number of duplicated documents:{count}/{value.shape[0]}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Duplicati nei `summary`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of duplicated documents:29075/1555030\n",
      "val - number of duplicated documents:129/40399\n",
      "test - number of duplicated documents:132/40413\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    count = value.duplicated(subset = [\"summary\"]).sum()\n",
    "    print(f\"{key} - number of duplicated documents:{count}/{value.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "key\n",
      "                                                 summary  count\n",
      "4284   do not request an appeal if your post contains...   1962\n",
      "19149  this bot is not affiliated with the moderators...    412\n",
      "19156  this bot is not affiliated with the moderators...    299\n",
      "18767  the phrase \" black lives matter \" carries an i...    113\n",
      "19150  this bot is not affiliated with the moderators...    100\n",
      "key\n",
      "                                              summary  count\n",
      "12  do not request an appeal if your post contains...     52\n",
      "42  this bot is not affiliated with the moderators...     13\n",
      "40  this bot is not affiliated with the moderators...      9\n",
      "7               comment a new rule suggestion below !      3\n",
      "41  this bot is not affiliated with the moderators...      3\n",
      "key\n",
      "                                              summary  count\n",
      "6   do not request an appeal if your post contains...     62\n",
      "45  this bot is not affiliated with the moderators...     14\n",
      "43  this bot is not affiliated with the moderators...      8\n",
      "2               comment a new rule suggestion below !      2\n",
      "41  the phrase \" black lives matter \" carries an i...      2\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "for key, value in corpus.items():\n",
    "    print(\"key\")\n",
    "    # generate a random start index\n",
    "    duplicate = value[value.duplicated(subset = [\"summary\"])][['id', 'summary']]\n",
    "\n",
    "    # printa radomici\n",
    "    #start_index = random.randint(0, len(duplicate_test) - 5)\n",
    "    #duplicate_test.iloc[start_index:start_index+5]\n",
    "\n",
    "    df_grouped = duplicate.groupby('summary').size().reset_index(name='count')\n",
    "    df_sorted = df_grouped.sort_values(by='count', ascending=False)\n",
    "    print(df_sorted.head(5))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Procedo anche qui alla rimozione"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of duplicated documents:0/1525955\n",
      "val - number of duplicated documents:0/40270\n",
      "test - number of duplicated documents:0/40281\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    # remove duplicates from train, val and test\n",
    "    value.drop_duplicates(subset = [\"summary\"], inplace=True)\n",
    "    \n",
    "for key, value in corpus.items():\n",
    "    count = value.duplicated(subset = [\"summary\"]).sum()\n",
    "    print(f\"{key} - number of duplicated documents:{count}/{value.shape[0]}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Missing Values"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train - number of null values in columns id and document:\n",
      "id            0\n",
      "document      0\n",
      "summary       0\n",
      "ext_labels    0\n",
      "rg_labels     0\n",
      "dtype: int64\n",
      "val - number of null values in columns id and document:\n",
      "id            0\n",
      "document      0\n",
      "summary       0\n",
      "ext_labels    0\n",
      "rg_labels     0\n",
      "dtype: int64\n",
      "test - number of null values in columns id and document:\n",
      "id            0\n",
      "document      0\n",
      "summary       0\n",
      "ext_labels    0\n",
      "rg_labels     0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for key, value in corpus.items():\n",
    "    # print number of null values in columns \"id\" and \"document\"\n",
    "    print(f\"{key} - number of null values in columns id and document:\")\n",
    "    print(value.isnull().sum())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Pre-Preprocessing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 Sentences Splitting"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ricompongo le frasi trasformando le stringhe in liste di stringhe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_df = test.copy()\n",
    "# def split_string(string):\n",
    "#     return string.split(\"</s><s>\")\n",
    "\n",
    "# test_df[\"document\"] = test_df[\"document\"].apply(split_string)\n",
    "\n",
    "# print(test_df['document'][0])\n",
    "# type(test_df['document'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_string(string):\n",
    "    return string.split(\"</s><s>\")\n",
    "\n",
    "for key, value in corpus.items():\n",
    "    value[\"document\"] = value[\"document\"].apply(split_string)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2 Normalization - Text Cleaning"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "mcpzjLi8RPOU"
   },
   "source": [
    "### HTML Tags (es `<span>`) and Entities (es. &nbsp) Removal"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tags:\n",
    "- < p\n",
    "- < a\n",
    "- < div >"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "rtaD4aLuRSC4"
   },
   "outputs": [],
   "source": [
    "def remove_html_tags(sentence):\n",
    "    pattern = re.compile(\"<.*?>\") \n",
    "    cleaned_sentence = re.sub(pattern,'',sentence).strip()\n",
    "    return cleaned_sentence\n",
    "\n",
    "def remove_html_entities(sentence):\n",
    "    pattern = re.compile(\"&[a-z0-9]+|&#[0-9]{1,6}|&#x[0-9a-f]{1,6}\")\n",
    "    cleaned_sentence = re.sub(pattern,'',sentence).strip()\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from bs4 import BeautifulSoup \n",
    "# import time\n",
    "\n",
    "# pattern = re.compile(r\"\\s\\d+\\s\")\n",
    "\n",
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if re.search(pattern, string):\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       # print(pattern.sub(r'\\1', string))\n",
    "#       time.sleep(2)\n",
    "#       #break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~~Code Removal~~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"highlight\"><pre>        \n",
      "</pre></div>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# string = \"ciao import pandas as pd\"\n",
    "\n",
    "# def remove_html_tags(string):\n",
    "#     # Cerca i tag HTML nella stringa e sostituiscili con una stringa vuota\n",
    "#     stripped_string = re.sub('<[^>]*>', '', string)\n",
    "    \n",
    "#     return stripped_string\n",
    "\n",
    "# def remove_code(string):\n",
    "#     # Evidenzia il codice utilizzando il lexer rilevato\n",
    "#     highlighted_code = highlight(string, guess_lexer(string), HtmlFormatter())\n",
    "    \n",
    "#     code_pattern = re.compile(r'<span[^>]*>([^<]*)</span>')\n",
    "\n",
    "#     # Replace the code with an empty string, while keeping the non-code text\n",
    "#     return re.sub(code_pattern, '', highlighted_code)\n",
    "\n",
    "# print(remove_code(string))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Excessive Whitespaces and \\t Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_extra_whitespaces(text):\n",
    "    cleaned_sentence = re.sub(r'^\\s*|\\s\\s*', ' ', text).strip()\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Ciao come stai 99 00- [] ;l'"
      ]
     },
     "execution_count": 410,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove_extra_whitespaces_func(\"Ciao \\t  come stai 99   00- []    ;l\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "GpI_jgBSRW1T"
   },
   "source": [
    "### URL Removal"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gli url hanno un formato particolare qui \n",
    "- [ ipcc report ] ( https://www.ipcc.ch/report/ar5/ ) -> tenere la parola a sx \n",
    "- [ https://github.com/tomnific/xpwnd ] ( https://github.com/tomnific/xpwnd ) -> eliminare tutto \n",
    "\n",
    "Semplice: rimozione url e poi simboli, la parola rimane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "Ah9iMXw1Rfqh"
   },
   "outputs": [],
   "source": [
    "def remove_urls(sentence):\n",
    "    http_pattern = re.compile(r\"http\\S+\") \n",
    "    cleaned_sentence = re.sub(http_pattern,'',sentence).strip()\n",
    "    www_pattern = re.compile(r\"www\\S+\") \n",
    "    cleaned_sentence = re.sub(www_pattern,'',cleaned_sentence)\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #pattern = re.compile(r'\\[([^\\]]*)\\]\\s*\\(([^)]*)\\)')\n",
    "\n",
    "# pattern = re.compile(r\"http\\S+\")\n",
    "\n",
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if re.search(pattern, string):\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       # print(pattern.sub(r'\\1', string))\n",
    "#       time.sleep(10)\n",
    "#       #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello this is my  website \n",
      "<re.Match object; span=(17, 54), match='[ website ]( https://www.google.com )'>\n"
     ]
    }
   ],
   "source": [
    "# import re\n",
    "\n",
    "# pattern = re.compile('\\[(.*?)\\]\\(.*?\\)')\n",
    "# #pattern = re.compile(r'\\[([^\\]]*)\\]\\s*\\(([^)]*)\\)')\n",
    "# comment = \"Hello this is my [ website ]( https://www.google.com )\"\n",
    "\n",
    "# print(pattern.sub(r'\\1', comment))\n",
    "# print(re.search(pattern, comment))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "T8_h3ihG72e1"
   },
   "source": [
    "### Emoji Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "rqCDv7iv8EOW"
   },
   "outputs": [],
   "source": [
    "def remove_emoji(phrase):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                               u\"\\U00002500-\\U00002BEF\"  # chinese char\n",
    "                               u\"\\U00002702-\\U000027B0\"\n",
    "                               u\"\\U00002702-\\U000027B0\"\n",
    "                               u\"\\U000024C2-\\U0001F251\"\n",
    "                               u\"\\U0001f926-\\U0001f937\"\n",
    "                               u\"\\U00010000-\\U0010ffff\"\n",
    "                               u\"\\u2640-\\u2642\"\n",
    "                               u\"\\u2600-\\u2B55\"\n",
    "                               u\"\\u200d\"\n",
    "                               u\"\\u23cf\"\n",
    "                               u\"\\u23e9\"\n",
    "                               u\"\\u231a\"\n",
    "                               u\"\\ufe0f\"  # dingbats\n",
    "                               u\"\\u3030\"\n",
    "                               \"]+\", flags=re.UNICODE)\n",
    "    cleaned_sentence = emoji_pattern.sub(r'', phrase)\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "cjSvhHPL8bD8"
   },
   "source": [
    "### Numbers Removal / NumbersTowords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Conversione ordinali 1st 2nd etc in words\n",
    "def replace_ordinal_numbers(text):\n",
    "    re_results = re.findall('(\\d+(st|nd|rd|th))', text)\n",
    "    for enitre_result, suffix in re_results:\n",
    "        num = int(enitre_result[:-len(suffix)])\n",
    "        text = text.replace(enitre_result, num2words(num, ordinal=True))\n",
    "    return text\n",
    "\n",
    "# Replace numbers with words (lone numbers)\n",
    "def replace_numbers(phrase):\n",
    "    cleaned_sentence = re.sub(r\"\\s\\d+\\s\", lambda x: f\" {num2words(x.group())} \", phrase)\n",
    "    return cleaned_sentence\n",
    "\n",
    "# Rimuovi i rimanenti\n",
    "def remove_numbers(phrase):\n",
    "  cleaned_sentence=re.sub(r'\\d+', '',phrase)\n",
    "  return cleaned_sentence\n",
    "\n",
    "# Funzione finale\n",
    "def process_numbers(phrase):\n",
    "    phrase = replace_ordinal_numbers(phrase)\n",
    "    try:\n",
    "        phrase = replace_numbers(phrase)\n",
    "        phrase = remove_numbers(phrase)\n",
    "    except:\n",
    "        phrase = remove_numbers(phrase)\n",
    "    return phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lista = [\"i 'm on the 8 ' th euler problem , the challenge is to find the thirteen adjacent digits in a 1000 digit number that has the highest product .\", \"i 've got some code that works for the example 4 adjacent digit product example but it 's not working for the 13 digits .\", \"here is the code i have ( forgive the poor naming i 'm not very good at it ) :\", 'long long get_product ( string ) ;', 'int convert_num ( char ) ;', '/ / to store the digits to be converted and checked', 'string start = \" \" ;', '/ / will eventuall store the largest product', 'long largest_product = 0 ;', \"/ / ca n't handle a 1000 digit number so store as string , convert as necessary\", 'string starting_number = \" 7316717653133062491922511967442657474235534919493496983520312774506326239578318016984801869478851843858615607891129494954595017379583319528532088055111254069874715852386305071569329096329522744304355766896648950445244523161731856403098711121722383113622298934233803081353362766142828064444866452387493035890729629049156044077239071381051585930796086670172427121883998797908792274921901699720888093776657273330010533678812202354218097512545405947522435258490771167055601360483958644670632441572215539753697817977846174064955149290862569321978468622482839722413756570560574902614079729686524145351004748216637048440319989000889524345065854122758866688116427171479924442928230863465674813919123162824586178664583591245665294765456828489128831426076900422421902267105562632111110937054421750694165896040807198403850962455444362981230987879927244284909188845801561660979191338754992005240636899125607176060588611646710940507754100225698315520005593572972571636269561882670428252483600823257530420752963450 \" ;', '/ / this should stop checking when there are less than 13 digits left to check', 'for ( int i = 0 ; i < starting_number .', 'size ( ) - 12 ; i + + )', '/ / add digits from the current position to the stirng', 'for ( int n = i ; n < i +13 ; n + + )', 'start + = starting_number [ n ] ;', '/ / convert the string to digits and get their product', 'long long product = get_product ( start ) ;', '/ / if the product is greater than the currently stored product replace it', 'if ( product > largest_product )', '/ / reset the start string', 'start = \" \" ;', '/ / output the largest product ( should be the answer )', 'cout < < \" length : \" < < endl < < largest_product ;', 'long long get_product ( string start )', \"/ / i do n't know why this works as opposed to having the starting number as ' 0 '\", 'long long total = convert_num ( start [ 0 ] ) ;', '/ / get the product of all the numbers in the string', 'for ( size_t i = 0 ; i < start.size ( ) - 1 ; i + + )', 'total = total * convert_num ( start [ i ] ) ;', 'int convert_num ( char value )', '/ / convert char to int', \"int num = value - ' 0 ' ;\", 'my eventual output is :', \"apparently that is wrong but i do n't know * why * given that this code works for the 4 digit version ( replacing starting_number .\", 'size ( ) - 12 with starting_number .', 'size ( ) - 3 and position +13 with position +4 ) .']\n",
    "\n",
    "# for i in lista:\n",
    "#     print(process_numbers(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " first The two second of January two thousand and twenty-one it's a good six hundred and fifty-four htttp////sx \n"
     ]
    }
   ],
   "source": [
    "# string = \"21 1st The 2 2nd of January 2021 it's a good 654 htttp//234//sx 123\"\n",
    "\n",
    "# print(process_numbers(string))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tag Channel / Userstag Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if ' u ' in string:\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       time.sleep(1)\n",
    "#       #break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Tag a subreddit e utenti**\n",
    "- Rimuovere tag a utenti e subreddit\n",
    "- Tenere la parola dopo"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Si nota utilizzo di: \n",
    "- \\\\ r \\\\ - \\ r \\ - / r / - r / per indicare i tag dei subreddit\n",
    "- / u / - \\\\ u \\\\ - u / per indicare altri utenti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_reddit_tags(phrase):\n",
    "  cleaned_sentence = re.sub(r'(\\\\ u \\\\|u \\/|r \\/|\\\\ r \\\\)', '',phrase)\n",
    "  return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remove_reddit_tags(\"r / u /  \\\\ u \\\\  \\\\ r \\\\\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ~~Hashtags~~"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cerchiamo:\n",
    "- `#`\n",
    "- ``"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if ' # ' in string:\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       time.sleep(10)\n",
    "#       #break"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Formatting/control character removal"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rimozione dei seguenti control character:\n",
    "- `\\ n ` -> a capo\n",
    "- `\\ t ` -> tab\n",
    "- `\\ v ` -> vertical tab\n",
    "- `\\ m `\n",
    "- ` \\ `"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_control_chars(phrase):\n",
    "    cleaned_sentence = re.sub(r'(\\\\ u |\\\\ n |\\\\ v |\\\\ m | \\\\ )', ' ',phrase)\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i = 0\n",
    "\n",
    "# pattern = re.compile(r'(\\\\ u |\\\\ n |\\\\ v |\\\\ m | \\\\ )')\n",
    "\n",
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if re.search(pattern, string):\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       print(re.sub(r'(\\\\ u |\\\\ n |\\\\ v |\\\\ m | \\\\ )', ' ', string))\n",
    "#       time.sleep(5)\n",
    "#       #break\n",
    "\n",
    "# print(i)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### agef - agem to age famale / age male "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_age(phrase):\n",
    "    cleaned_sentence = re.sub(r'(\\b\\d{2})f\\b', r'\\1 female',phrase)\n",
    "    cleaned_sentence = re.sub(r'(\\b\\d{2})m\\b', r'\\1 male',phrase)\n",
    "    return cleaned_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "when i ( 21 female ) first met my boyfriend , he was one of the most mentally strong people i knew .\n"
     ]
    }
   ],
   "source": [
    "# #i = 0\n",
    "\n",
    "# pattern = re.compile(r'(\\b\\d{2})f\\b')\n",
    "\n",
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if re.search(pattern, string):\n",
    "#       clear_output(wait=True)\n",
    "#       #print(string)\n",
    "#       print(re.sub(r'(\\b\\d{2})f\\b', r'\\1 female', string))\n",
    "#       #time.sleep(0)\n",
    "#       #break\n",
    "\n",
    "# #print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ciao, ho 24 female gradi oggi e fa 45 female fuori.\n"
     ]
    }
   ],
   "source": [
    "# stringa = \"Ciao, ho 24f gradi oggi e fa 45f fuori.\"\n",
    "\n",
    "# nuova_stringa = re.sub(r'(\\b\\d{2})f\\b', r'\\1 female', stringa)\n",
    "# nuova_stringa = re.sub(r'\\bf(\\d{2})\\b', r'female \\1', nuova_stringa)\n",
    "# print(nuova_stringa)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Normalization - Word Processing"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "Um5d11nD7b-a"
   },
   "source": [
    "### Case Folding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "xMo7ZTKj7iEN"
   },
   "outputs": [],
   "source": [
    "def to_lower(phrase):\n",
    "  phrase = phrase.lower()\n",
    "  return phrase"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "ibcSQYqlSHCE"
   },
   "source": [
    "### Correction of words with 3 or more repeated characters within them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "HuH4xEnTSF8u"
   },
   "outputs": [],
   "source": [
    "# def remove_words_with_repeated_characters(sentence): \n",
    "#     pattern = re.compile(\"\\\\s*\\\\b(?=\\\\w*(\\\\w)\\\\1{2,})\\\\w*\\\\b\")\n",
    "#     cleaned_text  = re.sub(pattern,' ',sentence)\n",
    "#     return (cleaned_text)\n",
    "\n",
    "def character_repeatation(text):\n",
    "    # Pattern matching for all case alphabets\n",
    "    # \\1   It refers to the first capturing group.\n",
    "    # {2,} It means we are matching for repetition that occurs more than two times (or equal).\n",
    "    # r’\\1\\1' → It limits all the repetition to two characters.\n",
    "    Pattern_alpha = re.compile(r\"([A-Za-z])\\1{2,}\", re.DOTALL)\n",
    "    # Limiting all the  repeatation to two characters.\n",
    "    Formatted_text = Pattern_alpha.sub(r\"\\1\\1\", text) \n",
    "    # Pattern matching for all the punctuations that can occur\n",
    "    Pattern_Punct = re.compile(r'([.,/#!$%^&*?;:{}=_`~()+-])\\1{1,}')\n",
    "    # Limiting punctuations in previously formatted string to only one.\n",
    "    Combined_Formatted = Pattern_Punct.sub(r'\\1', Formatted_text)\n",
    "    return Combined_Formatted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before: Realllllllllyyyyy gooooood!!!!!\n",
      "After: Reallyy good!\n"
     ]
    }
   ],
   "source": [
    "# repeated = \"Realllllllllyyyyy gooooood!!!!!\"\n",
    "# print(\"Before:\",repeated)\n",
    "# print(\"After:\",character_repeatation(repeated))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "StjYT_ezRKAu"
   },
   "source": [
    "### Expansion of contractions"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Qui le contrazion hanno subito una separazione, nei documenti abbiamo di fatto parole separate da spazi. es. did n\\'t - your 're - i'm. Vanno quindi contemplate sia le versioni \"won\\'t\" che won \\'t"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for index, row in corpus['train'].iterrows():\n",
    "#   for string in row['document']:\n",
    "#     if 'won \\\\ t' in string:\n",
    "#       clear_output(wait=True)\n",
    "#       print(string)\n",
    "#       # print(pattern.sub(r'\\1', string))\n",
    "#       time.sleep(5)\n",
    "#       #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#contractions_dict "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alcune contrazioni vanno riattaccate per permettere una corretta espansione:\n",
    "- 'n\\'t'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tl7oZNoMNovF"
   },
   "outputs": [],
   "source": [
    "# def expand_eng_contradictions(phrase):\n",
    "#     phrase = re.sub(r\"won\\'t\", \"will not\", phrase)\n",
    "#     phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n",
    "#     phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
    "#     phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
    "#     phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
    "#     phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
    "#     phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
    "#     phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
    "#     phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
    "#     phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
    "#     return phrase"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Funzione"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Rimozione spazi bianchi prima dei suffissi nelle espressioni contratte\n",
    "2. Applicazione funzione di espansione da contractions\n",
    "   1. Utilizzo di slang=true che include espressioni "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_and_expand_eng_contradictions(phrase):\n",
    "    # Remove whitespaces before '\n",
    "    phrase = re.sub(r\"\\s+'\", \"'\", phrase)\n",
    "    # Avvicina parole tipo \"was n't\" a \"wasn't\" \n",
    "    phrase = re.sub(r\"\\s+n't\", \"n't\", phrase)\n",
    "    phrase = contractions.fix(phrase, slang=True)\n",
    "    return phrase\n",
    "    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stringa = \"yall cant , it 's been a week now ( besides that little incident on the weekend ) and it 's been driving me crazy , i would text her and she would n't respond , i would get home and she would act like im not there , i would go to bed and she would head out at 12 am and not tell me where she 's going , i do n't even know what time she 'd get home , i feel like i 've lost her completely and i just want her back , i just want to talk to her again , i just want her .\"\n",
    "# fix_and_expand_eng_contradictions(stringa)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spelling mistakes Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def correct_me(text):\n",
    "  textBlb = TextBlob(text)        \n",
    "  textCorrected = str(textBlb.correct())   # Correcting the text\n",
    "  return textCorrected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I my am is john'"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test = \"Hi my nam is jonh\"\n",
    "# test1 = str(correct_me(test))\n",
    "# test1"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "GG55fyh-SVCP"
   },
   "source": [
    "### Removal of punctuation and special characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "RQKaV7NQSaKy"
   },
   "outputs": [],
   "source": [
    "# Rimuove tutto ciò che non sia alfabetico\n",
    "def remove_special_characters_punctuations(sentence):\n",
    "    cleaned_text  = re.sub(r\"[^a-zA-Z]+\",' ',sentence).strip()\n",
    "    return cleaned_text\n",
    "\n",
    "# Rimuove solo i caratteri speciali e la punteggiatura\n",
    "# def remove_punctuation(text):\n",
    "#     punctuationfree=\"\".join([i for i in text if i not in string.punctuation])\n",
    "#     return punctuationfree"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.3 Normalization - Words Analysis"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenization"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Si utilizza la funzione già fatta `list(map(word_tokenize, prova1))`"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stop-words and 1-char words removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stop_words_1char_removal(text, stopwords_list = stopwords.words('english')):\n",
    "  return ([word for word in text if (word not in stopwords_list) and (len(word) > 1)])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmaSentence(token_words):\n",
    "    lemma_text=[]\n",
    "    for word in token_words:\n",
    "        lemma_text.append(WordNetLemmatizer().lemmatize(word))\n",
    "    return lemma_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dog'"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "WordNetLemmatizer().lemmatize('dogs')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### POS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\giorg\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package tagsets to\n",
      "[nltk_data]     C:\\Users\\giorg\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package tagsets is already up-to-date!\n",
      "[nltk_data] Downloading package universal_tagset to\n",
      "[nltk_data]     C:\\Users\\giorg\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package universal_tagset is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('tagsets')\n",
    "nltk.download('universal_tagset')\n",
    "\n",
    "def extract_tags(document_tags: list):\n",
    "  doc_tags = pd.Series(document_tags)\n",
    "  doc_tags = doc_tags.apply(lambda subList: pd.Series(subList))\n",
    "  doc_tags = doc_tags.applymap(lambda wordTagTuple: wordTagTuple[1] if type(wordTagTuple)==tuple else '')\n",
    "  return doc_tags.values\n",
    "\n",
    "def POS_tagging(document: list, tagset:str = 'universal', lang:str='eng'):\n",
    "  POS_tags = nltk.tag.pos_tag_sents(document, tagset=tagset, lang=lang)\n",
    "  POS_tags = extract_tags(POS_tags)\n",
    "  return [list(filter(None, l.tolist())) for l in POS_tags]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['VERB', 'NOUN', 'VERB', 'NOUN', 'NOUN'],\n",
       " ['NOUN', 'ADJ', 'NOUN', 'VERB', 'NOUN', 'NOUN', 'NOUN', 'ADJ', 'NOUN'],\n",
       " ['NOUN', 'NOUN', 'VERB', 'ADJ', 'NOUN', 'NOUN', 'NOUN', 'NOUN', 'VERB'],\n",
       " ['VERB', 'NOUN', 'NOUN', 'VERB', 'NOUN'],\n",
       " ['VERB', 'VERB', 'NOUN', 'NOUN', 'VERB', 'NOUN']]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lista = [['running', 'twenty', 'friend', 'family', 'group'], ['dad', 'best', 'friend', 'playing', 'briton', 'brother', 'chemist', 'charismatic', 'wedlock'], ['wedlock', 'dad', 'volunteered', 'stay', 'night', 'watch', 'town', 'people', 'spell'], ['asked', 'chemist', 'something', 'keep', 'awake'], ['turned', 'said', 'cock', 'plant', 'exist', 'world']]\n",
    "POS_tagging(lista)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "id": "GEkmE6MVTbuT"
   },
   "source": [
    "## Funzione completa"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Usa apply per applicare la macrofunzione agli elementi singoli\n",
    "2. Usa map per applicare le singole funzioni agli elementi della lista -> oppure list comprension\n",
    "\n",
    "- Aggiungere counter del tipo \"train - processing post x/n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "id": "-yAMfWsRTezi"
   },
   "outputs": [],
   "source": [
    "# def text_normalization(text):\n",
    "#   print(text)\n",
    "#   # Text Cleaning\n",
    "#   text = remove_html_tags(text)\n",
    "#   text = remove_html_entities(text)\n",
    "#   text = remove_extra_whitespaces(text)\n",
    "#   text = remove_urls(text)\n",
    "#   text = remove_emoji(text)\n",
    "#   text = process_numbers(text)\n",
    "#   text = remove_reddit_tags(text)\n",
    "#   text = remove_control_chars(text)\n",
    "#   text = process_age(text)\n",
    "#   text = remove_extra_whitespaces(text)\n",
    "#   # Word Processing\n",
    "#   text = text.lower()\n",
    "#   text = character_repeatation(text)\n",
    "#   text = fix_and_expand_eng_contradictions(text)\n",
    "#   text = correct_me(text)\n",
    "#   text = remove_special_characters_punctuations(text)\n",
    "#   print(text)\n",
    "#   return text\n",
    "\n",
    "def text_normalization(sentences):\n",
    "  # Text Cleaning\n",
    "  sentences = list(map(remove_html_tags, sentences))\n",
    "  sentences = list(map(remove_html_entities, sentences))\n",
    "  sentences = list(map(remove_extra_whitespaces, sentences))\n",
    "  sentences = list(map(remove_urls, sentences))\n",
    "  sentences = list(map(remove_emoji, sentences))\n",
    "  sentences = list(map(process_numbers, sentences))\n",
    "  sentences = list(map(remove_reddit_tags, sentences))\n",
    "  sentences = list(map(remove_control_chars, sentences))\n",
    "  sentences = list(map(process_age, sentences))\n",
    "  sentences = list(map(remove_extra_whitespaces, sentences))\n",
    "  # Word Processing\n",
    "  sentences = list(map(to_lower, sentences))\n",
    "  sentences = list(map(character_repeatation, sentences))\n",
    "  sentences = list(map(fix_and_expand_eng_contradictions, sentences))\n",
    "  #sentences = list(map(correct_me, sentences))\n",
    "  sentences = list(map(remove_special_characters_punctuations, sentences))\n",
    "  sentences = list(map(remove_extra_whitespaces, sentences))\n",
    "  return sentences\n",
    "\n",
    "def text_Tokenization(sentences):\n",
    "  # Tokenization\n",
    "  return list(map(word_tokenize, sentences))\n",
    "\n",
    "def text_stop_words_1char_removal(sentences):\n",
    "  # Stopwords removal\n",
    "  return list(map(stop_words_1char_removal, sentences))\n",
    "\n",
    "def text_lemmatizer(senteces):\n",
    "  return list(map(lemmaSentence, senteces))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "lista = [\"i 'm on the 8 ' th euler problem , the challenge is to find the thirteen adjacent digits in a 1000 digit number that has the highest product .\", \"i 've got some code that works for the example 4 adjacent digit product example but it 's not working for the 13 digits .\", \"here is the code i have ( forgive the poor naming i 'm not very good at it ) :\", 'long long get_product ( string ) ;', 'int convert_num ( char ) ;', '/ / to store the digits to be converted and checked', 'string start = \" \" ;', '/ / will eventuall store the largest product', 'long largest_product = 0 ;', \"/ / ca n't handle a 1000 digit number so store as string , convert as necessary\", 'string starting_number = \" 7316717653133062491922511967442657474235534919493496983520312774506326239578318016984801869478851843858615607891129494954595017379583319528532088055111254069874715852386305071569329096329522744304355766896648950445244523161731856403098711121722383113622298934233803081353362766142828064444866452387493035890729629049156044077239071381051585930796086670172427121883998797908792274921901699720888093776657273330010533678812202354218097512545405947522435258490771167055601360483958644670632441572215539753697817977846174064955149290862569321978468622482839722413756570560574902614079729686524145351004748216637048440319989000889524345065854122758866688116427171479924442928230863465674813919123162824586178664583591245665294765456828489128831426076900422421902267105562632111110937054421750694165896040807198403850962455444362981230987879927244284909188845801561660979191338754992005240636899125607176060588611646710940507754100225698315520005593572972571636269561882670428252483600823257530420752963450 \" ;', '/ / this should stop checking when there are less than 13 digits left to check', 'for ( int i = 0 ; i < starting_number .', 'size ( ) - 12 ; i + + )', '/ / add digits from the current position to the stirng', 'for ( int n = i ; n < i +13 ; n + + )', 'start + = starting_number [ n ] ;', '/ / convert the string to digits and get their product', 'long long product = get_product ( start ) ;', '/ / if the product is greater than the currently stored product replace it', 'if ( product > largest_product )', '/ / reset the start string', 'start = \" \" ;', '/ / output the largest product ( should be the answer )', 'cout < < \" length : \" < < endl < < largest_product ;', 'long long get_product ( string start )', \"/ / i do n't know why this works as opposed to having the starting number as ' 0 '\", 'long long total = convert_num ( start [ 0 ] ) ;', '/ / get the product of all the numbers in the string', 'for ( size_t i = 0 ; i < start.size ( ) - 1 ; i + + )', 'total = total * convert_num ( start [ i ] ) ;', 'int convert_num ( char value )', '/ / convert char to int', \"int num = value - ' 0 ' ;\", 'my eventual output is :', \"apparently that is wrong but i do n't know * why * given that this code works for the 4 digit version ( replacing starting_number .\", 'size ( ) - 12 with starting_number .', 'size ( ) - 3 and position +13 with position +4 ) .']\n",
    "prova1 = text_normalization(lista)\n",
    "prova2 = list(map(word_tokenize, prova1))\n",
    "prova3 = list(map(stop_words_1char_removal, prova2))\n",
    "prova4 = list(map(lemmaSentence, prova3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i was running a campaign for my friends and family group .', ' my dad and his best friend were playing triton brothers , an alchemist druid and a charismatic warlock .', ' the warlock ( my dad ) volunteered to stay up all night to watch a town of people who were under a spell .', ' he asked the alchemist for something to keep him awake .', ' he turned to me and said , \" does the coca plant exist in this world ? \"']\n",
      "\\n\n",
      "['i was running a campaign for my friends and family group', 'my dad and his best friend were playing briton brothers an chemist did and a charismatic wedlock', 'the wedlock my dad volunteered to stay up all night to watch a town of people who were under a spell', 'he asked the chemist for something to keep him awake', 'he turned to me and said does the cock plant exist in this world']\n",
      "\\n\n",
      "[['i', 'was', 'running', 'a', 'campaign', 'for', 'my', 'friends', 'and', 'family', 'group'], ['my', 'dad', 'and', 'his', 'best', 'friend', 'were', 'playing', 'briton', 'brothers', 'an', 'chemist', 'did', 'and', 'a', 'charismatic', 'wedlock'], ['the', 'wedlock', 'my', 'dad', 'volunteered', 'to', 'stay', 'up', 'all', 'night', 'to', 'watch', 'a', 'town', 'of', 'people', 'who', 'were', 'under', 'a', 'spell'], ['he', 'asked', 'the', 'chemist', 'for', 'something', 'to', 'keep', 'him', 'awake'], ['he', 'turned', 'to', 'me', 'and', 'said', 'does', 'the', 'cock', 'plant', 'exist', 'in', 'this', 'world']]\n",
      "\\n\n",
      "[['running', 'campaign', 'friends', 'family', 'group'], ['dad', 'best', 'friend', 'playing', 'briton', 'brothers', 'chemist', 'charismatic', 'wedlock'], ['wedlock', 'dad', 'volunteered', 'stay', 'night', 'watch', 'town', 'people', 'spell'], ['asked', 'chemist', 'something', 'keep', 'awake'], ['turned', 'said', 'cock', 'plant', 'exist', 'world']]\n",
      "\\n\n",
      "[['running', 'campaign', 'friend', 'family', 'group'], ['dad', 'best', 'friend', 'playing', 'briton', 'brother', 'chemist', 'charismatic', 'wedlock'], ['wedlock', 'dad', 'volunteered', 'stay', 'night', 'watch', 'town', 'people', 'spell'], ['asked', 'chemist', 'something', 'keep', 'awake'], ['turned', 'said', 'cock', 'plant', 'exist', 'world']]\n"
     ]
    }
   ],
   "source": [
    "print(val['document'][2])\n",
    "print(\"\\\\n\")\n",
    "print(prova1)\n",
    "print(\"\\\\n\")\n",
    "print(prova2)\n",
    "print(\"\\\\n\")\n",
    "print(prova3)\n",
    "print(\"\\\\n\")\n",
    "print(prova4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40413, 5)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_split = np.array_split(copy_val, 4)\n",
    "df_split[0].shape\n",
    "df = pd.concat(df_split)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "copy_val.to_json(\"ciao.json\", orient = \"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "text_normalization() got an unexpected keyword argument 'axis'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[65], line 4\u001b[0m\n\u001b[0;32m      1\u001b[0m copy_val \u001b[39m=\u001b[39m corpus[\u001b[39m'\u001b[39m\u001b[39mtest\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m.\u001b[39mcopy()\n\u001b[0;32m      2\u001b[0m \u001b[39m#tqdm_notebook().pandas()\u001b[39;00m\n\u001b[0;32m      3\u001b[0m \u001b[39m#copy_val['normalized_document'] = copy_val['document'].progress_apply(text_normalization)\u001b[39;00m\n\u001b[1;32m----> 4\u001b[0m copy_val[\u001b[39m'\u001b[39m\u001b[39mnormalized_document\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m copy_val[\u001b[39m'\u001b[39;49m\u001b[39mdocument\u001b[39;49m\u001b[39m'\u001b[39;49m]\u001b[39m.\u001b[39;49mapply(text_normalization, axis \u001b[39m=\u001b[39;49m \u001b[39m1\u001b[39;49m)\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\series.py:4771\u001b[0m, in \u001b[0;36mSeries.apply\u001b[1;34m(self, func, convert_dtype, args, **kwargs)\u001b[0m\n\u001b[0;32m   4661\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply\u001b[39m(\n\u001b[0;32m   4662\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[0;32m   4663\u001b[0m     func: AggFuncType,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4666\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[0;32m   4667\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m DataFrame \u001b[39m|\u001b[39m Series:\n\u001b[0;32m   4668\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m   4669\u001b[0m \u001b[39m    Invoke function on values of Series.\u001b[39;00m\n\u001b[0;32m   4670\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   4769\u001b[0m \u001b[39m    dtype: float64\u001b[39;00m\n\u001b[0;32m   4770\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 4771\u001b[0m     \u001b[39mreturn\u001b[39;00m SeriesApply(\u001b[39mself\u001b[39;49m, func, convert_dtype, args, kwargs)\u001b[39m.\u001b[39;49mapply()\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\apply.py:1105\u001b[0m, in \u001b[0;36mSeriesApply.apply\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1102\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapply_str()\n\u001b[0;32m   1104\u001b[0m \u001b[39m# self.f is Callable\u001b[39;00m\n\u001b[1;32m-> 1105\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mapply_standard()\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\apply.py:1156\u001b[0m, in \u001b[0;36mSeriesApply.apply_standard\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1154\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[0;32m   1155\u001b[0m         values \u001b[39m=\u001b[39m obj\u001b[39m.\u001b[39mastype(\u001b[39mobject\u001b[39m)\u001b[39m.\u001b[39m_values\n\u001b[1;32m-> 1156\u001b[0m         mapped \u001b[39m=\u001b[39m lib\u001b[39m.\u001b[39;49mmap_infer(\n\u001b[0;32m   1157\u001b[0m             values,\n\u001b[0;32m   1158\u001b[0m             f,\n\u001b[0;32m   1159\u001b[0m             convert\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconvert_dtype,\n\u001b[0;32m   1160\u001b[0m         )\n\u001b[0;32m   1162\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(mapped) \u001b[39mand\u001b[39;00m \u001b[39misinstance\u001b[39m(mapped[\u001b[39m0\u001b[39m], ABCSeries):\n\u001b[0;32m   1163\u001b[0m     \u001b[39m# GH#43986 Need to do list(mapped) in order to get treated as nested\u001b[39;00m\n\u001b[0;32m   1164\u001b[0m     \u001b[39m#  See also GH#25959 regarding EA support\u001b[39;00m\n\u001b[0;32m   1165\u001b[0m     \u001b[39mreturn\u001b[39;00m obj\u001b[39m.\u001b[39m_constructor_expanddim(\u001b[39mlist\u001b[39m(mapped), index\u001b[39m=\u001b[39mobj\u001b[39m.\u001b[39mindex)\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\_libs\\lib.pyx:2918\u001b[0m, in \u001b[0;36mpandas._libs.lib.map_infer\u001b[1;34m()\u001b[0m\n",
      "File \u001b[1;32md:\\Users\\giorg\\Documents\\venv\\NLP10\\lib\\site-packages\\pandas\\core\\apply.py:139\u001b[0m, in \u001b[0;36mApply.__init__.<locals>.f\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    138\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mf\u001b[39m(x):\n\u001b[1;32m--> 139\u001b[0m     \u001b[39mreturn\u001b[39;00m func(x, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "\u001b[1;31mTypeError\u001b[0m: text_normalization() got an unexpected keyword argument 'axis'"
     ]
    }
   ],
   "source": [
    "copy_val = corpus['test'].copy()\n",
    "#tqdm_notebook().pandas()\n",
    "#copy_val['normalized_document'] = copy_val['document'].progress_apply(text_normalization)\n",
    "copy_val['normalized_document'] = copy_val['document'].apply(text_normalization, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, value in corpus.items():\n",
    "    print(f\"{key.upper()} : Loading text preprocessing...\")\n",
    "    value['normalized_document'] = value['document'].apply(text_normalization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "#from tqdm.notebook import tqdm\n",
    "from tqdm import tqdm, tqdm_pandas, tqdm_notebook\n",
    "# print(\"Loading...\")\n",
    "# for i in tqdm(range(100)):\n",
    "#     #clear_output(wait=True)\n",
    "#     #print(i)\n",
    "#     time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLP10",
   "language": "python",
   "name": "nlp10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c7a68dc2e3110d80647fac8693cdc5243ade541bc79b3bfddbf1397505f434ab"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}