Text_Preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Text Preprocessing",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMkexJSMM+/6k+2DyL2mc6i",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/koseii2122/Reddit-Vaccine-Data/blob/main/Text_Preprocessing.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IXAO3WRn53E3"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "import spacy\n",
        "import csv\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bX2jnmeldWGW",
        "outputId": "7ea7c2b4-c6ba-4eb9-889b-bd24ed97b12b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('comments_corpus_minscore_5.csv', engine='python', encoding='utf-8', error_bad_lines=False)\n",
        "df = df.loc[:, ~df.columns.str.contains('^Unnamed')]"
      ],
      "metadata": {
        "id": "tNu3_yiY6AdI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JBIMIKQYFz4g",
        "outputId": "ae9274f7-b270-4efb-e366-e1d49661cf22"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "220315"
            ]
          },
          "metadata": {},
          "execution_count": 144
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df[df['body'].notnull()]\n",
        "df.reset_index(drop=True, inplace=True)"
      ],
      "metadata": {
        "id": "E7fkP1iYonV2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u0TW47TbooDG",
        "outputId": "c322f6d7-1686-41de-b672-80497efb1776"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "220069"
            ]
          },
          "metadata": {},
          "execution_count": 146
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "collapsed": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "k21xP0fsnwaL",
        "outputId": "988e72ce-5dfb-42d0-9b15-2c2d1d6cc3d3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-845e6ddc-b913-4091-9855-6701087f0110\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>author</th>\n",
              "      <th>body</th>\n",
              "      <th>created_utc</th>\n",
              "      <th>id</th>\n",
              "      <th>link</th>\n",
              "      <th>parent_id</th>\n",
              "      <th>score</th>\n",
              "      <th>subreddit</th>\n",
              "      <th>title</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sealandair</td>\n",
              "      <td>Of course there are emails. Which is why Hunt ...</td>\n",
              "      <td>1.638895e+09</td>\n",
              "      <td>hnlyzwu</td>\n",
              "      <td>redd.it//r/CoronavirusDownunder/comments/razzw...</td>\n",
              "      <td>t3_razzwb</td>\n",
              "      <td>121.0</td>\n",
              "      <td>CoronavirusDownunder</td>\n",
              "      <td>Emails reveal Pfizer was 'enthusiastic' to eng...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>clementjohnson1963</td>\n",
              "      <td>But Scomo fucked up</td>\n",
              "      <td>1.638888e+09</td>\n",
              "      <td>hnlj73o</td>\n",
              "      <td>redd.it//r/CoronavirusDownunder/comments/razzw...</td>\n",
              "      <td>t3_razzwb</td>\n",
              "      <td>210.0</td>\n",
              "      <td>CoronavirusDownunder</td>\n",
              "      <td>Emails reveal Pfizer was 'enthusiastic' to eng...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>VolcanicBosnian</td>\n",
              "      <td>Imagine all the lockdowns we could have avoide...</td>\n",
              "      <td>1.638913e+09</td>\n",
              "      <td>hnn53y2</td>\n",
              "      <td>redd.it//r/CoronavirusDownunder/comments/razzw...</td>\n",
              "      <td>t3_razzwb</td>\n",
              "      <td>39.0</td>\n",
              "      <td>CoronavirusDownunder</td>\n",
              "      <td>Emails reveal Pfizer was 'enthusiastic' to eng...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AussieCollector</td>\n",
              "      <td>I really hope this fuck up costs scomo and the...</td>\n",
              "      <td>1.638914e+09</td>\n",
              "      <td>hnn6z8j</td>\n",
              "      <td>redd.it//r/CoronavirusDownunder/comments/razzw...</td>\n",
              "      <td>t3_razzwb</td>\n",
              "      <td>21.0</td>\n",
              "      <td>CoronavirusDownunder</td>\n",
              "      <td>Emails reveal Pfizer was 'enthusiastic' to eng...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>hu_he</td>\n",
              "      <td>It would be a lot easier to forgive the govern...</td>\n",
              "      <td>1.638911e+09</td>\n",
              "      <td>hnn0wc9</td>\n",
              "      <td>redd.it//r/CoronavirusDownunder/comments/razzw...</td>\n",
              "      <td>t3_razzwb</td>\n",
              "      <td>12.0</td>\n",
              "      <td>CoronavirusDownunder</td>\n",
              "      <td>Emails reveal Pfizer was 'enthusiastic' to eng...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220310</th>\n",
              "      <td>geckothegeek42</td>\n",
              "      <td>&gt;searching out positive male role models in hi...</td>\n",
              "      <td>1.639054e+09</td>\n",
              "      <td>hnugkof</td>\n",
              "      <td>redd.it//r/criticalrole/comments/rc2sqc/spoile...</td>\n",
              "      <td>t1_hnsd9xi</td>\n",
              "      <td>28.0</td>\n",
              "      <td>criticalrole</td>\n",
              "      <td>[Spoilers C3E6] Recurring themes with the cast...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220311</th>\n",
              "      <td>Muserallusion</td>\n",
              "      <td>&gt; All of Travis' characters in some way have b...</td>\n",
              "      <td>1.639140e+09</td>\n",
              "      <td>hnzaou0</td>\n",
              "      <td>redd.it//r/criticalrole/comments/rc2sqc/spoile...</td>\n",
              "      <td>t1_hnsd9xi</td>\n",
              "      <td>7.0</td>\n",
              "      <td>criticalrole</td>\n",
              "      <td>[Spoilers C3E6] Recurring themes with the cast...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220312</th>\n",
              "      <td>Jethro_McCrazy</td>\n",
              "      <td>Other than the hammer, yeah.</td>\n",
              "      <td>1.639026e+09</td>\n",
              "      <td>hntfndk</td>\n",
              "      <td>redd.it//r/criticalrole/comments/rc2sqc/spoile...</td>\n",
              "      <td>t1_hntdj12</td>\n",
              "      <td>7.0</td>\n",
              "      <td>criticalrole</td>\n",
              "      <td>[Spoilers C3E6] Recurring themes with the cast...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220313</th>\n",
              "      <td>Think-Ad-7612</td>\n",
              "      <td>Oh, yes, thanks. I noticed all that. Just neve...</td>\n",
              "      <td>1.639024e+09</td>\n",
              "      <td>hntah3e</td>\n",
              "      <td>redd.it//r/criticalrole/comments/rc2sqc/spoile...</td>\n",
              "      <td>t1_hnta92y</td>\n",
              "      <td>10.0</td>\n",
              "      <td>criticalrole</td>\n",
              "      <td>[Spoilers C3E6] Recurring themes with the cast...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>220314</th>\n",
              "      <td>CallistoWarriorQueen</td>\n",
              "      <td>Absolutely :)</td>\n",
              "      <td>1.639076e+09</td>\n",
              "      <td>hnvvym5</td>\n",
              "      <td>redd.it//r/criticalrole/comments/rc2sqc/spoile...</td>\n",
              "      <td>t1_hnugkof</td>\n",
              "      <td>5.0</td>\n",
              "      <td>criticalrole</td>\n",
              "      <td>[Spoilers C3E6] Recurring themes with the cast...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>220315 rows × 9 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-845e6ddc-b913-4091-9855-6701087f0110')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-845e6ddc-b913-4091-9855-6701087f0110 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-845e6ddc-b913-4091-9855-6701087f0110');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                      author  ...                                              title\n",
              "0                 sealandair  ...  Emails reveal Pfizer was 'enthusiastic' to eng...\n",
              "1         clementjohnson1963  ...  Emails reveal Pfizer was 'enthusiastic' to eng...\n",
              "2            VolcanicBosnian  ...  Emails reveal Pfizer was 'enthusiastic' to eng...\n",
              "3            AussieCollector  ...  Emails reveal Pfizer was 'enthusiastic' to eng...\n",
              "4                      hu_he  ...  Emails reveal Pfizer was 'enthusiastic' to eng...\n",
              "...                      ...  ...                                                ...\n",
              "220310        geckothegeek42  ...  [Spoilers C3E6] Recurring themes with the cast...\n",
              "220311         Muserallusion  ...  [Spoilers C3E6] Recurring themes with the cast...\n",
              "220312        Jethro_McCrazy  ...  [Spoilers C3E6] Recurring themes with the cast...\n",
              "220313         Think-Ad-7612  ...  [Spoilers C3E6] Recurring themes with the cast...\n",
              "220314  CallistoWarriorQueen  ...  [Spoilers C3E6] Recurring themes with the cast...\n",
              "\n",
              "[220315 rows x 9 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 133
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# #@title\n",
        "# df_10k = df.sample(n=10000)\n",
        "# df = df.loc[:, ~df.columns.str.contains('^Unnamed')]\n",
        "# df_10k.reset_index(drop=True, inplace=True)\n"
      ],
      "metadata": {
        "id": "kryvL055S2UY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.dtypes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FUDIOqubZ9xZ",
        "outputId": "e4849540-44cb-4872-9766-183c42a7d0a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "author          object\n",
              "body            object\n",
              "created_utc    float64\n",
              "id              object\n",
              "link            object\n",
              "parent_id       object\n",
              "score          float64\n",
              "subreddit       object\n",
              "title           object\n",
              "dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 134
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.body"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6UtZJ_RbbEJ0",
        "outputId": "a3c1027f-b5b3-4193-b20c-f969566388cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         Of course there are emails. Which is why Hunt ...\n",
              "1                                       But Scomo fucked up\n",
              "2         Imagine all the lockdowns we could have avoide...\n",
              "3         I really hope this fuck up costs scomo and the...\n",
              "4         It would be a lot easier to forgive the govern...\n",
              "                                ...                        \n",
              "220310    >searching out positive male role models in hi...\n",
              "220311    > All of Travis' characters in some way have b...\n",
              "220312                         Other than the hammer, yeah.\n",
              "220313    Oh, yes, thanks. I noticed all that. Just neve...\n",
              "220314                                        Absolutely :)\n",
              "Name: body, Length: 220315, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 135
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert to Lowercase\n",
        "\n",
        "df[\"body_pp_1\"] = df[\"body\"].str.lower()  # convert to lowercase"
      ],
      "metadata": {
        "id": "XldkPHBhHfXy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Whitespace removal\n",
        "\n",
        "df[\"body_pp_1\"] = df[\"body_pp_1\"].str.replace('\\n+', ' ', regex=True) # remove extra unnecessary \\n\n",
        "df[\"body_pp_1\"] = df[\"body_pp_1\"].str.replace('\\s+', ' ', regex=True) # remove extra unnecessary whitespaces\n",
        "df[\"body_pp_1\"] = df[\"body_pp_1\"].str.strip() # remove extra unnecessary whitespaces"
      ],
      "metadata": {
        "id": "iSc0dfeKJT2R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.body_pp_1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WcQfG78ybrII",
        "outputId": "d717d3a2-6d9c-40c7-d226-bd9c954e8d28"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         of course there are emails. which is why hunt ...\n",
              "1                                       but scomo fucked up\n",
              "2         imagine all the lockdowns we could have avoide...\n",
              "3         i really hope this fuck up costs scomo and the...\n",
              "4         it would be a lot easier to forgive the govern...\n",
              "                                ...                        \n",
              "220310    >searching out positive male role models in hi...\n",
              "220311    > all of travis' characters in some way have b...\n",
              "220312                         other than the hammer, yeah.\n",
              "220313    oh, yes, thanks. i noticed all that. just neve...\n",
              "220314                                        absolutely :)\n",
              "Name: body_pp_1, Length: 220315, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df[\"body_pp_2\"] = df[\"body_pp_1\"]"
      ],
      "metadata": {
        "id": "fr8SAVfCcmYo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing Stopwords\n",
        "\n",
        "from nltk.corpus import stopwords\n",
        "stop = stopwords.words('english')\n",
        "\n",
        "df['body_pp_2'] = df['body_pp_1'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n"
      ],
      "metadata": {
        "id": "tOrwzbn2eR8b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.body_pp_2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WG4yWUkvfCo9",
        "outputId": "64ec98dd-c2f1-4bdd-bc49-956131c50547"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         course emails. hunt resigned. knew would come ...\n",
              "1                                              scomo fucked\n",
              "2         imagine lockdowns could avoided slime minister...\n",
              "3         really hope fuck costs scomo lnp election. kno...\n",
              "4         would lot easier forgive government's stuff or...\n",
              "                                ...                        \n",
              "220064    >searching positive male role models life amaz...\n",
              "220065    > travis' characters way men struggling workin...\n",
              "220066                                        hammer, yeah.\n",
              "220067    oh, yes, thanks. noticed that. never heard ter...\n",
              "220068                                        absolutely :)\n",
              "Name: body_pp_2, Length: 220069, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 151
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['body_pp_3'] = df['body_pp_2']"
      ],
      "metadata": {
        "id": "HJp5u-wQeTco"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Lemmatization\n",
        "\n",
        "import nltk\n",
        "\n",
        "w_tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
        "lemmatizer = nltk.stem.WordNetLemmatizer()\n",
        "\n",
        "def lemmatize_text(text):\n",
        "    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]\n",
        "\n",
        "df['body_pp_3'] = df.body_pp_2.apply(lemmatize_text).str.join(' ').str.strip()"
      ],
      "metadata": {
        "id": "O6lcTHxXcPhS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.body_pp_3"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IvrHZoDifxeb",
        "outputId": "0f12b531-779d-454f-ce64-d136228a1aff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         course emails. hunt resigned. knew would come ...\n",
              "1                                              scomo fucked\n",
              "2         imagine lockdown could avoided slime minister ...\n",
              "3         really hope fuck cost scomo lnp election. know...\n",
              "4         would lot easier forgive government's stuff or...\n",
              "                                ...                        \n",
              "220064    >searching positive male role model life amazi...\n",
              "220065    > travis' character way men struggling working...\n",
              "220066                                        hammer, yeah.\n",
              "220067    oh, yes, thanks. noticed that. never heard ter...\n",
              "220068                                        absolutely :)\n",
              "Name: body_pp_3, Length: 220069, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 154
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['body_pp_4'] = df[\"body_pp_3\"]"
      ],
      "metadata": {
        "id": "uiN4h-XrmRdL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# selective punctuation removal\n",
        "\n",
        "remove_punct = ['.', '\\'', ',', '\\\\', '/', '‘', '’', '“', '”', '(', ')', '[', ']', ' -', '- '] # Hyphen - is a special case\n",
        "keep_punct = ['!', ':', ';', '?', '%'] # These are kept\n",
        "\n",
        "\n",
        "for punct in remove_punct:\n",
        "  df['body_pp_4'] = df[\"body_pp_4\"].str.replace(punct, '')\n",
        "  a = a.replace(punct, ' ')\n",
        "\n",
        "\n",
        "for punct in keep_punct:\n",
        "  st = ' ' + punct + ' '\n",
        "  df['body_pp_4'] = df[\"body_pp_4\"].str.replace(punct, st)\n",
        "\n",
        "df[\"body_pp_4\"] = df[\"body_pp_4\"].str.replace('\\s+', ' ', regex=True) # remove extra unnecessary whitespaces\n",
        "df[\"body_pp_4\"] = df[\"body_pp_4\"].str.strip()"
      ],
      "metadata": {
        "id": "N31kXSd2f7ue"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df.body_pp_4"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nSwWDQsXkgKK",
        "outputId": "bc532e3d-3e4e-4df1-eb58-02b6aff9f587"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0         course emails hunt resigned knew would come ev...\n",
              "1                                              scomo fucked\n",
              "2         imagine lockdown could avoided slime minister ...\n",
              "3         really hope fuck cost scomo lnp election knowi...\n",
              "4         would lot easier forgive governments stuff ord...\n",
              "                                ...                        \n",
              "220064    >searching positive male role model life amazi...\n",
              "220065    > travis character way men struggling working ...\n",
              "220066                                          hammer yeah\n",
              "220067    oh yes thanks noticed that never heard term \"m...\n",
              "220068                                         absolutely :\n",
              "Name: body_pp_4, Length: 220069, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 157
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.to_csv('final_comments_corpus_preprocessed_min_5.csv')"
      ],
      "metadata": {
        "id": "QQIEe1oRpUiK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfx = df\n",
        "\n",
        "dfx.drop_duplicates(keep='first', inplace=True, subset=['id'])\n",
        "dfx = dfx[dfx['score'] >= 20]    # Removing Submissions \n",
        "df = df.loc[:, ~df.columns.str.contains('^Unnamed')]\n",
        "dfx.reset_index(drop=True, inplace=True)\n",
        "\n",
        " \n",
        "print('Total Submissions scraped: ' + str(len(dfx)))\n",
        "dfx.to_csv('final_comments_corpus_preprocessed_min_20.csv')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C-CjX72lpjrU",
        "outputId": "184f24da-d559-4317-d019-33eb10c8985a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Submissions scraped: 64203\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(df['created_utc'].max())\n",
        "print(df['created_utc'].min())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5KvrxSnxriWf",
        "outputId": "326a1086-ab19-4925-f8d9-7c6a60d9bd18"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1640892154.0\n",
            "1604688258.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# GMT: Thursday, December 30, 2021 7:22:34 PM\n",
        "# GMT: Friday, November 6, 2020 6:44:18 PM"
      ],
      "metadata": {
        "id": "R1FKcmLVsAGv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# timetamps = [1601510402, 1604188802, ]\n",
        "\n",
        "len(df[\"created_utc\"].between(1601510402, 1604188802, inclusive = True))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        },
        "id": "HyGf4I13sZoJ",
        "outputId": "e0b0071c-aa0e-4f3a-c25f-ad9183044a68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-177-89039174965a>\"\u001b[0;36m, line \u001b[0;32m3\u001b[0m\n\u001b[0;31m    len(df[\"created_utc\"].between(160njn, 1604188802, inclusive = True))\u001b[0m\n\u001b[0m                                       ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    }
  ]
}