Pre-Processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Chatbot2.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ezn87osL7Qg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FDv-1YxrVKz9",
        "colab_type": "text"
      },
      "source": [
        "# Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SHtNqLPZMHq6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import collections\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Model\n",
        "from keras.layers import GRU, Input, Dense, TimeDistributed, Activation, RepeatVector, Bidirectional\n",
        "from keras.layers import Embedding, CuDNNLSTM, GlobalMaxPooling1D, GlobalAveragePooling1D, CuDNNGRU\n",
        "from keras.layers.embeddings import Embedding\n",
        "from keras.optimizers import Adam\n",
        "from keras.losses import sparse_categorical_crossentropy\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard\n",
        "from nltk.translate.bleu_score import sentence_bleu, corpus_bleu\n",
        "import helper\n",
        "\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "import re"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6oRC_bfxVfBd",
        "colab_type": "text"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nyleqlmNU2gw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('scrap_context.csv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uhX3-kLZfSUO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ecee25fd-e561-4e7b-f5df-3798f1a03bcc"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(19227, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xU81macQfQ3Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1df55454-7339-43d4-f9f0-33047bd56f35"
      },
      "source": [
        "df = df[(df['title'] != '[deleted]') & (df['response'] != '[deleted]')]\n",
        "df.shape"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(18758, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2_nlbXIWHrG-",
        "colab_type": "text"
      },
      "source": [
        "# Pre-Processing 1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aWK3UDYjfhVP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'u.s':'america', 'e.g':'for example'}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I2OQZO6wfi5z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean_contractions(text, mapping):\n",
        "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
        "    for s in specials:\n",
        "        text = text.replace(s, \"'\")\n",
        "    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n",
        "    return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HgDQwUAn__22",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['title'] = df['title'].apply(lambda x: x.lower())\n",
        "df['response'] = df['response'].apply(lambda x: x.lower())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-kT66EE_wGK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1ea2b54a-fc61-44ec-8558-6401b704afa8"
      },
      "source": [
        "df['title'] = df['title'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))\n",
        "df['response'] = df['response'].progress_apply(lambda x: clean_contractions(x, contraction_mapping))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:00<00:00, 40194.44it/s]\n",
            "100%|██████████| 18758/18758 [00:00<00:00, 33356.80it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBBDD9xZ_v44",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "punct = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
        " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
        " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
        " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
        " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZgHC0mB_fi3z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', '!':' '}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FQKSoV8sfi1P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def clean_special_chars(text, punct, mapping):\n",
        "    for p in mapping:\n",
        "        text = text.replace(p, mapping[p])\n",
        "    \n",
        "    for p in punct:\n",
        "        text = text.replace(p, f' {p} ')\n",
        "    \n",
        "    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  \n",
        "    for s in specials:\n",
        "        text = text.replace(s, specials[s])\n",
        "    \n",
        "    return text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0C500t6IG0uQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "f912b112-bbc7-4a5f-8062-1920ee328e10"
      },
      "source": [
        "df['title'] = df['title'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n",
        "df['response'] = df['response'].progress_apply(lambda x: clean_special_chars(x, punct, punct_mapping))"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:02<00:00, 9289.78it/s]\n",
            "100%|██████████| 18758/18758 [00:01<00:00, 10724.49it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2JTgbYlXG0rp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LlNEzgSjfizC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def correct_spelling(x, dic):\n",
        "    for word in dic.keys():\n",
        "        x = x.replace(word, dic[word])\n",
        "    return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u1xD4ut1HCNN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "37e95e37-d39e-4626-c413-3fcedc35d124"
      },
      "source": [
        "df['title'] = df['title'].progress_apply(lambda x: correct_spelling(x, mispell_dict))\n",
        "df['response'] = df['response'].progress_apply(lambda x: correct_spelling(x, mispell_dict))"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:00<00:00, 35159.70it/s]\n",
            "100%|██████████| 18758/18758 [00:00<00:00, 28474.27it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H1m_yIhOHCKe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "3b76af82-b1fe-4b3e-9a1f-869a0405a713"
      },
      "source": [
        "import re\n",
        "# text = re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', text, flags=re.MULTILINE)\n",
        "df['title'] = df['title'].progress_apply(lambda x: re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', x, flags=re.MULTILINE))\n",
        "df['response'] = df['response'].progress_apply(lambda x: re.sub(r'^https?:\\/\\/.*[\\r\\n]*', '', x, flags=re.MULTILINE))"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:00<00:00, 82587.87it/s]\n",
            "100%|██████████| 18758/18758 [00:00<00:00, 78057.06it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tUuAxpmAHBuU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "08fcf1fc-7344-47b2-f7f5-b979e5556813"
      },
      "source": [
        "check = 'hi this is /r/news yoo'\n",
        "re.sub(r'\\/r\\/[a-z]+ ', '', check, flags=re.MULTILINE)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'hi this is yoo'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aiEDLzC9HH4v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "f5084a5f-3a09-4597-f741-bb69ee6e43f2"
      },
      "source": [
        "df['title'] = df['title'].progress_apply(lambda x: re.sub(r'\\/r\\/[a-z]+ ', '', x, flags=re.MULTILINE))\n",
        "df['response'] = df['response'].progress_apply(lambda x: re.sub(r'\\/r\\/[a-z]+ ', '', x, flags=re.MULTILINE))"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:00<00:00, 150831.84it/s]\n",
            "100%|██████████| 18758/18758 [00:00<00:00, 136079.64it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AdZKN79mdNiB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess_sentence(w):\n",
        "\n",
        "    # creating a space between a word and the punctuation following it\n",
        "    # eg: \"he is a boy.\" => \"he is a boy .\"\n",
        "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
        "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
        "    w = re.sub(r'[\" \"]+', \" \", w)\n",
        "\n",
        "    # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
        "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
        "\n",
        "    w = w.rstrip().strip()\n",
        "    \n",
        "    # adding a start and an end token to the sentence\n",
        "    # so that the model know when to start and stop predicting.\n",
        "    w = '<start> ' + w + ' <end>'\n",
        "    \n",
        "    return w"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "74SH82JKHHzw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "56da4818-1e39-4e96-c0f2-b983c7832831"
      },
      "source": [
        "df['title'] = df['title'].progress_apply(lambda x: preprocess_sentence(x))\n",
        "df['response'] = df['response'].progress_apply(lambda x: preprocess_sentence(x))"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 18758/18758 [00:00<00:00, 22329.07it/s]\n",
            "100%|██████████| 18758/18758 [00:01<00:00, 15155.92it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KYgWWK2BHTk9",
        "colab_type": "text"
      },
      "source": [
        "# Pre-Processing 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H38ico-yVswk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# title list\n",
        "titles=df['title'].values\n",
        "import string\n",
        "title=[]\n",
        "for i in range(len(titles)):\n",
        "    title.append(titles[i].lower().strip())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nTTIJP9LWRWe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# response list\n",
        "responses=df['response'].values\n",
        "response=[]\n",
        "for i in range(len(responses)):\n",
        "    response.append(responses[i].lower().strip())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fy7DGmQPWVI4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "14bdf5f2-5354-4c27-e9a1-38f2e4eebd8f"
      },
      "source": [
        "# view title response pair\n",
        "for i in range(2):\n",
        "  print('Person 1 : ', title[i])\n",
        "  print('Person 2 : ', response[i])"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Person 1 :  <start> it kills me to see others in worse positions than i am . you matter . you all matter and it does not matter what your depression or suicidal thoughts tell you . <end>\n",
            "Person 2 :  <start> i do not know who you are , but i needed this and i thank you . i have been stuck ina runt for a couple months now . wanting to just die and let it all be over with . but after reading it , i have a different mindset . once again thank you kind stranger <end>\n",
            "Person 1 :  <start> i do not know who you are , but i needed this and i thank you . i have been stuck ina runt for a couple months now . wanting to just die and let it all be over with . but after reading it , i have a different mindset . once again thank you kind stranger <end>\n",
            "Person 2 :  <start> its no problem . i just know what its like . i have been in this rut for a year , and i hate how many people are in the same , if not a worse position . do not kill yourself . all the things that made you happy in the past , that can still happen . it takes time and effort , and a lot of it , but believe me when i say its worth living . its difficult to just get out of bed . but if you start turning depression into like , a source of motivation , it can help . x b for example we are all depressed , thats why we are here , but we do not want to stay here . i remember when i was happy although i do not remember the feeling . i do not care what you do , just do not give up . x b i ve split my brain and depression into two separate entities . try doing that . it helps me see and think clearer . <end>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0PbqQzRfIsP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "cdf7d425-3107-4aea-dd47-a725668f0316"
      },
      "source": [
        "title_counter = collections.Counter([word for sentence in title for word in sentence.split()])\n",
        "response_counter = collections.Counter([word for sentence in response for word in sentence.split()])\n",
        "\n",
        "print('{} Title sentences.'.format(len(title)))\n",
        "print('{} Title words.'.format(len([word for sentence in title for word in sentence.split()])))\n",
        "print('{} unique Title words.'.format(len(title_counter)))\n",
        "print('10 Most common words in the Title:')\n",
        "print('\"' + '\" \"'.join(list(zip(*title_counter.most_common(10)))[0]) + '\"')\n",
        "print()\n",
        "print('{} Response sentences.'.format(len(response)))\n",
        "print('{} Response words.'.format(len([word for sentence in response for word in sentence.split()])))\n",
        "print('{} unique Response words.'.format(len(response_counter)))\n",
        "print('10 Most common words in the Response dataset:')\n",
        "print('\"' + '\" \"'.join(list(zip(*response_counter.most_common(10)))[0]) + '\"')"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "18758 Title sentences.\n",
            "965914 Title words.\n",
            "20970 unique Title words.\n",
            "10 Most common words in the Title:\n",
            "\".\" \"i\" \",\" \"to\" \"you\" \"the\" \"and\" \"<start>\" \"<end>\" \"a\"\n",
            "\n",
            "18758 Response sentences.\n",
            "1496799 Response words.\n",
            "24906 unique Response words.\n",
            "10 Most common words in the Response dataset:\n",
            "\".\" \"i\" \",\" \"to\" \"you\" \"and\" \"the\" \"a\" \"is\" \"it\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EsPxUUKohV_i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "dac70c66-af67-4c4d-ea48-b482a7ab9ba4"
      },
      "source": [
        "def tokenize(x):\n",
        "\n",
        "    tok = Tokenizer()\n",
        "    tok.fit_on_texts(x)\n",
        "    return tok.texts_to_sequences(x), tok\n",
        "\n",
        "\n",
        "# Tokenize Example output\n",
        "text_sentences = [\n",
        "    'The quick brown fox jumps over the lazy dog .',\n",
        "    'By Jove , my quick study of lexicography won a prize .',\n",
        "    'This is a short sentence .']\n",
        "text_tokenized, text_tokenizer = tokenize(text_sentences)\n",
        "print(text_tokenizer.word_index)\n",
        "print()\n",
        "for sample_i, (sent, token_sent) in enumerate(zip(text_sentences, text_tokenized)):\n",
        "    print('Sequence {} in x'.format(sample_i + 1))\n",
        "    print('  Input:  {}'.format(sent))\n",
        "    print('  Output: {}'.format(token_sent))"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'the': 1, 'quick': 2, 'a': 3, 'brown': 4, 'fox': 5, 'jumps': 6, 'over': 7, 'lazy': 8, 'dog': 9, 'by': 10, 'jove': 11, 'my': 12, 'study': 13, 'of': 14, 'lexicography': 15, 'won': 16, 'prize': 17, 'this': 18, 'is': 19, 'short': 20, 'sentence': 21}\n",
            "\n",
            "Sequence 1 in x\n",
            "  Input:  The quick brown fox jumps over the lazy dog .\n",
            "  Output: [1, 2, 4, 5, 6, 7, 1, 8, 9]\n",
            "Sequence 2 in x\n",
            "  Input:  By Jove , my quick study of lexicography won a prize .\n",
            "  Output: [10, 11, 12, 2, 13, 14, 15, 16, 3, 17]\n",
            "Sequence 3 in x\n",
            "  Input:  This is a short sentence .\n",
            "  Output: [18, 19, 3, 20, 21]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-SV9Iva5e3AR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "9b22fcba-8c61-4717-8d5c-3f741d8c8a15"
      },
      "source": [
        "def pad(x, length=None):\n",
        "\n",
        "    if length is None:\n",
        "        length = max([len(sentence) for sentence in x])\n",
        "    return pad_sequences(x, maxlen=length, padding=\"post\")\n",
        "\n",
        "# Pad Tokenized output\n",
        "test_pad = pad(text_tokenized)\n",
        "for sample_i, (token_sent, pad_sent) in enumerate(zip(text_tokenized, test_pad)):\n",
        "    print('Sequence {} in x'.format(sample_i + 1))\n",
        "    print('  Input:  {}'.format(np.array(token_sent)))\n",
        "    print('  Output: {}'.format(pad_sent))"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sequence 1 in x\n",
            "  Input:  [1 2 4 5 6 7 1 8 9]\n",
            "  Output: [1 2 4 5 6 7 1 8 9 0]\n",
            "Sequence 2 in x\n",
            "  Input:  [10 11 12  2 13 14 15 16  3 17]\n",
            "  Output: [10 11 12  2 13 14 15 16  3 17]\n",
            "Sequence 3 in x\n",
            "  Input:  [18 19  3 20 21]\n",
            "  Output: [18 19  3 20 21  0  0  0  0  0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5GXKLp0dfau9",
        "colab_type": "text"
      },
      "source": [
        "# Pre-Process Pipeline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2GBv8yxfExi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def preprocess(x, y):\n",
        "\n",
        "    preprocess_x, x_tk = tokenize(x)\n",
        "    preprocess_y, y_tk = tokenize(y)\n",
        "\n",
        "    preprocess_x = pad(preprocess_x)\n",
        "    preprocess_y = pad(preprocess_y)\n",
        "\n",
        "    # Keras's sparse_categorical_crossentropy function requires the labels to be in 3 dimensions\n",
        "    preprocess_y = preprocess_y.reshape(*preprocess_y.shape, 1)\n",
        "\n",
        "    return preprocess_x, preprocess_y, x_tk, y_tk"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hZCNQuWHfjVt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "7fd56107-796d-42f8-8ea8-a346287f7bdf"
      },
      "source": [
        "preproc_title, preproc_response, title_tokenizer, response_tokenizer =\\\n",
        "    preprocess(title, response)\n",
        "    \n",
        "max_title_sequence_length = preproc_title.shape[1]\n",
        "max_response_sequence_length = preproc_response.shape[1]\n",
        "title_vocab_size = len(title_tokenizer.word_index)\n",
        "response_vocab_size = len(response_tokenizer.word_index)\n",
        "\n",
        "print('Data Preprocessed')\n",
        "print(\"Max Title sentence length:\", max_title_sequence_length)\n",
        "print(\"Max Response sentence length:\", max_response_sequence_length)\n",
        "print(\"Title vocabulary size:\", title_vocab_size)\n",
        "print(\"Response vocabulary size:\", response_vocab_size)"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Data Preprocessed\n",
            "Max Title sentence length: 1617\n",
            "Max Response sentence length: 1617\n",
            "Title vocabulary size: 20965\n",
            "Response vocabulary size: 24901\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MSMh3i2miyvf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}