Question_2_NER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "n-DoFHDMGgQW",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3196,
     "status": "ok",
     "timestamp": 1731645286785,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "n-DoFHDMGgQW",
    "outputId": "79ffd2dd-5591-4650-e70f-c51f7b40b0f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting contractions\n",
      "  Downloading contractions-0.1.73-py2.py3-none-any.whl.metadata (1.2 kB)\n",
      "Collecting textsearch>=0.0.21 (from contractions)\n",
      "  Downloading textsearch-0.0.24-py2.py3-none-any.whl.metadata (1.2 kB)\n",
      "Collecting anyascii (from textsearch>=0.0.21->contractions)\n",
      "  Downloading anyascii-0.3.2-py3-none-any.whl.metadata (1.5 kB)\n",
      "Collecting pyahocorasick (from textsearch>=0.0.21->contractions)\n",
      "  Downloading pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl.metadata (13 kB)\n",
      "Downloading contractions-0.1.73-py2.py3-none-any.whl (8.7 kB)\n",
      "Downloading textsearch-0.0.24-py2.py3-none-any.whl (7.6 kB)\n",
      "Downloading anyascii-0.3.2-py3-none-any.whl (289 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.9/289.9 kB\u001b[0m \u001b[31m22.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (110 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.7/110.7 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: pyahocorasick, anyascii, textsearch, contractions\n",
      "Successfully installed anyascii-0.3.2 contractions-0.1.73 pyahocorasick-2.1.0 textsearch-0.0.24\n"
     ]
    }
   ],
   "source": [
    "!pip install contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c079012",
   "metadata": {
    "id": "8c079012"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import nltk\n",
    "import numpy as np\n",
    "import re\n",
    "import contractions\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import matplotlib.pyplot as plt\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Nycwznv_HWom",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 343,
     "status": "ok",
     "timestamp": 1731645555027,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "Nycwznv_HWom",
    "outputId": "daefb9bf-b6d5-448e-8017-4aaa62acd443"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt_tab to /root/nltk_data...\n",
      "[nltk_data]   Package punkt_tab is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger_eng to\n",
      "[nltk_data]     /root/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger_eng is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('punkt_tab')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('averaged_perceptron_tagger_eng')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928eed25",
   "metadata": {
    "id": "928eed25"
   },
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "LYV4LNcgG3It",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2460,
     "status": "ok",
     "timestamp": 1731645428933,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "LYV4LNcgG3It",
    "outputId": "b40bd350-ebf6-4a63-be07-a9c286533a2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "df = pd.read_csv('/content/drive/MyDrive/NLP/combined_file.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7341098f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "executionInfo": {
     "elapsed": 674,
     "status": "ok",
     "timestamp": 1731645436942,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "7341098f",
    "outputId": "8c3142a5-ce16-45bd-ef89-4cdb6ce4fd63"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"df\",\n  \"rows\": 92815,\n  \"fields\": [\n    {\n      \"column\": \"title\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 82753,\n        \"samples\": [\n          \"Really scared, please help\",\n          \"\\u05e0\\u05e9\\u05d9\\u05d0\\u05d9 \\u05d0\\u05e8\\u05e6\\u05d5\\u05ea \\u05d4\\u05d1\\u05e8\\u05d9\\u05ea \\u05d5\\u05e8\\u05d0\\u05e9\\u05d9 \\u05de\\u05de\\u05e9\\u05dc\\u05ea/\\u05e0\\u05e9\\u05d9\\u05d0\\u05d9 \\u05d9\\u05e9\\u05e8\\u05d0\\u05dc \\u05e9\\u05de\\u05e7\\u05d1\\u05d9\\u05dc\\u05d9\\u05dd \\u05d6\\u05d4 \\u05dc\\u05d6\\u05d4\",\n          \"Why don't we push to make Israel the 51st state.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 83945,\n        \"samples\": [\n          \"Is there a mention about any particular direction I shouldn\\u2019t keep my head towards while sleep?\",\n          \"hi, so i started uni this year and i live in an all girls accom. however recently a few of my flatmates have been bringing guys over. i brought this up but apparently it's allowed?? i'm not entirely sure what to do as my flatmates are currently a bit hostile (due to other disputes) and idk how to bring it up without causing issues. also im not confrontational or good at expressing things im unhappy with... help :(\",\n          \"Between the vandalism that at the UPenn Hillel and the antisemite convention going on over Yom Kippur it\\u2019s making me feel scared. It\\u2019s not even Yom Kippur yet and this is what\\u2019s happening.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"score\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 791,\n        \"samples\": [\n          \"101\",\n          \"113\",\n          \"411\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"upvote_ratio\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 101,\n        \"samples\": [\n          \"0.13\",\n          \"0.78\",\n          \"0.68\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"upvotes\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 788,\n        \"samples\": [\n          \"725\",\n          \"113\",\n          \"702\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unnamed: 5\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \" by organizations like Gush Emunim and the Jewish Agency/World Zionist Organization\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Unnamed: 6\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \" and intensified the settlement activities.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-573b91d5-aca3-4558-98e4-1b98ddf54fd5\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>score</th>\n",
       "      <th>upvote_ratio</th>\n",
       "      <th>upvotes</th>\n",
       "      <th>Unnamed: 5</th>\n",
       "      <th>Unnamed: 6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Looking for married Muslim men who have hijabi...</td>\n",
       "      <td>Don’t message me if you can’t live verify. Too...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Share your istikhara success stories. I need s...</td>\n",
       "      <td>Salaam everyone. I’m a F currently going throu...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fate?</td>\n",
       "      <td>In the Qur'an, I saw verses in these cases tha...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Good Thrift shop find. Highly reccomend</td>\n",
       "      <td>Holocaust book about family of Jewish Hungaria...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Wearing my kippah with tattoos</td>\n",
       "      <td>Shalom friends!\\n\\nI’m a Baal teshuva with man...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-573b91d5-aca3-4558-98e4-1b98ddf54fd5')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-573b91d5-aca3-4558-98e4-1b98ddf54fd5 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-573b91d5-aca3-4558-98e4-1b98ddf54fd5');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-1fd48386-5e82-4022-8f39-b53b96ed4dc3\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1fd48386-5e82-4022-8f39-b53b96ed4dc3')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-1fd48386-5e82-4022-8f39-b53b96ed4dc3 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Looking for married Muslim men who have hijabi...   \n",
       "1  Share your istikhara success stories. I need s...   \n",
       "2                                              Fate?   \n",
       "3            Good Thrift shop find. Highly reccomend   \n",
       "4                     Wearing my kippah with tattoos   \n",
       "\n",
       "                                                text score upvote_ratio  \\\n",
       "0  Don’t message me if you can’t live verify. Too...     1            1   \n",
       "1  Salaam everyone. I’m a F currently going throu...     1            1   \n",
       "2  In the Qur'an, I saw verses in these cases tha...     1            1   \n",
       "3  Holocaust book about family of Jewish Hungaria...     1            1   \n",
       "4  Shalom friends!\\n\\nI’m a Baal teshuva with man...     1            1   \n",
       "\n",
       "  upvotes Unnamed: 5 Unnamed: 6  \n",
       "0       1        NaN        NaN  \n",
       "1       1        NaN        NaN  \n",
       "2       1        NaN        NaN  \n",
       "3       1        NaN        NaN  \n",
       "4       1        NaN        NaN  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5e04557",
   "metadata": {
    "id": "d5e04557"
   },
   "outputs": [],
   "source": [
    "df = df.dropna(subset=['text'])\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "954818d3",
   "metadata": {
    "id": "954818d3"
   },
   "outputs": [],
   "source": [
    "df = df.drop(columns=['Unnamed: 5', 'Unnamed: 6'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b20993be",
   "metadata": {
    "id": "b20993be"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db514a9d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 311,
     "status": "ok",
     "timestamp": 1731645451651,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "db514a9d",
    "outputId": "e04a5b32-cf6a-439c-aa90-e98128131b2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentences with URLs:\n",
      "                                                    text\n",
      "9      i found an old bbc documentary show titled \"co...\n",
      "26     /r/worldnews\\n   \\n https://www.reuters.com/wo...\n",
      "31           https://www.youtube.com/watch?v=v3ga7skxino\n",
      "50     this is an automated weekly thread\\n\\nfeel fre...\n",
      "61     /r/worldnews\\n   \\n https://www.forbes.com/sit...\n",
      "...                                                  ...\n",
      "92653  israeli authorities have declined to release a...\n",
      "92656  /r/worldnews\\n   \\n https://www.businessinside...\n",
      "92657  it is important for those new to the palestini...\n",
      "92662  [https://bloggingtheology.net/2016/05/06/chris...\n",
      "92669  my previous post was removed by a mod because ...\n",
      "\n",
      "[17485 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "url_pattern = r'http[s]?://[^\\s]+'\n",
    "urls_found = df[df['text'].str.contains(url_pattern, regex=True)]\n",
    "\n",
    "print(\"Sentences with URLs:\")\n",
    "print(urls_found[['text']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ff08ead",
   "metadata": {
    "id": "7ff08ead"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.replace(url_pattern, '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbbd7e4e",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2653,
     "status": "ok",
     "timestamp": 1731645458654,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "bbbd7e4e",
    "outputId": "1410b65d-46bd-4588-8206-c099b50afc12"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sentences with email addresses:\n",
      "                                                    text\n",
      "7798   i'm running a speaker event on monday evening,...\n",
      "14458   hi i'm a disabled  dad if 2, london uk\\nmy wi...\n",
      "19960  i am excited that laasok - the new liberal bei...\n",
      "22489  i need a couple of volunteers for tomorrow.  s...\n",
      "25704  salamalaikum to everyone. i want to help out s...\n",
      "25962  hi all. i'm a reform rabbi, and founder of laa...\n",
      "25966  hello! we are a group of undergraduates lookin...\n",
      "26537  hello, my name is veronica, and i am a doctora...\n",
      "26540  hello, my name is veronica, and i am a doctora...\n",
      "26848  \\n\\nanasya's donation link: anastasiyaparaskev...\n",
      "40437  this is their official [post on fb:](\\n\\ndear ...\n",
      "41226  on 12/10/2023 on lbc at around 7:00pm-8:00pm t...\n",
      "42619  binance, the world's largest cryptocurrency ex...\n",
      "43429  \\n\\tplease use the sharing tools found via the...\n",
      "48964    \\nplease use the sharing tools found via the...\n",
      "55899  reform judaism is often referred to as \"non-ha...\n",
      "61884  to all those who reside in canada, please boos...\n",
      "62578  i would post to /r/canada and /r/onguardforthe...\n",
      "63400  “humanrightswatch - we are documenting censors...\n",
      "63991  has its hands ties since israel and the usa wo...\n",
      "64348  the washington post is interested in hearing f...\n",
      "65280  at faith to faithless (a service run by the ch...\n",
      "65561  hi friends. a lot of people are struggling rig...\n",
      "65746  any pro israel folks want to help get another ...\n",
      "65851  [screenshot of the tweet where he offers the r...\n",
      "66796  if you’re in biglaw and spend any time at all ...\n",
      "71555  hello!\\n\\ni am part of the peace and justice p...\n",
      "72488  i should profit of this but the war is on the ...\n",
      "75909  his name is brian kaplan, an employee at @ever...\n",
      "80015   hello r/askmiddleeast community,\\n\\ni hope th...\n",
      "80016   hello r/askmiddleeast community,\\n\\ni hope th...\n",
      "83493  hello! my name is adriana, i am a senior psych...\n"
     ]
    }
   ],
   "source": [
    "email_pattern = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}'\n",
    "emails_found = df[df['text'].str.contains(email_pattern, regex=True)]\n",
    "\n",
    "print(\"\\nSentences with email addresses:\")\n",
    "print(emails_found[['text']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14d82a2d",
   "metadata": {
    "id": "14d82a2d"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.replace(email_pattern, '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdceae7a",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1731645461393,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "fdceae7a",
    "outputId": "362aa3d0-f90a-4edd-c46a-7cfc1a26109c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sentences with hashtags:\n",
      "                                                    text\n",
      "5      slightly embarrassing question but also someth...\n",
      "6      i found this subreddit not too long ago but up...\n",
      "50     this is an automated weekly thread\\n\\nfeel fre...\n",
      "93     \\* you live in the west and are casually walki...\n",
      "96                                                #name?\n",
      "...                                                  ...\n",
      "92589                       &#x200b;\\n\\n\\n\\n&#x200b;\\n\\n\n",
      "92595                       &#x200b;\\n\\n\\n\\n&#x200b;\\n\\n\n",
      "92618  hi all,\\n\\ni wrote my congressman today (2nd t...\n",
      "92653  israeli authorities have declined to release a...\n",
      "92657  it is important for those new to the palestini...\n",
      "\n",
      "[4728 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "hashtag_pattern = r'#[\\w]+'\n",
    "hashtags_found = df[df['text'].str.contains(hashtag_pattern, regex=True)]\n",
    "\n",
    "print(\"\\nSentences with hashtags:\")\n",
    "print(hashtags_found[['text']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13d3a8d3",
   "metadata": {
    "id": "13d3a8d3"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.replace(hashtag_pattern, '', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33bf9b26",
   "metadata": {
    "id": "33bf9b26"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.strip().str.replace('\\s+', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e357ddd1",
   "metadata": {
    "id": "e357ddd1"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].apply(contractions.fix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28a22929",
   "metadata": {
    "id": "28a22929"
   },
   "outputs": [],
   "source": [
    "def remove_special_characters(text):\n",
    "    return re.sub(r'[^A-Za-z0-9\\s]', '', text)\n",
    "\n",
    "df['text'] = df['text'].apply(remove_special_characters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bebe1d5",
   "metadata": {
    "id": "9bebe1d5"
   },
   "outputs": [],
   "source": [
    "def remove_numbers(text):\n",
    "    return re.sub(r'\\d+', '', text)\n",
    "\n",
    "df['text'] = df['text'].apply(remove_numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd82aba6",
   "metadata": {
    "id": "dd82aba6"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].str.strip().str.replace('\\s+', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4c6d275",
   "metadata": {
    "id": "e4c6d275"
   },
   "outputs": [],
   "source": [
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99c9907e",
   "metadata": {
    "id": "99c9907e"
   },
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    tokens = word_tokenize(text)\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_tokens = [word for word in tokens if word not in stop_words]\n",
    "    pos_tags = pos_tag(filtered_tokens)\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    lemmatized_tokens = [lemmatizer.lemmatize(word, get_wordnet_pos(tag)) for word, tag in pos_tags]\n",
    "    return ' '.join(lemmatized_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a54d933",
   "metadata": {
    "id": "4a54d933"
   },
   "outputs": [],
   "source": [
    "df['text'] = df['text'].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26ddeba0",
   "metadata": {
    "id": "26ddeba0"
   },
   "outputs": [],
   "source": [
    "entity = [\"hamas\", \"israel\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8029df26",
   "metadata": {
    "id": "8029df26"
   },
   "outputs": [],
   "source": [
    "def extract_entity_context(text, entity_list):\n",
    "    doc = nlp(text)\n",
    "    sentences_with_entities = []\n",
    "\n",
    "    for sent in doc.sents:\n",
    "        if any(ent.text in entity_list for ent in sent.ents):\n",
    "            sentences_with_entities.append(sent.text)\n",
    "\n",
    "    return sentences_with_entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a05fc19",
   "metadata": {
    "id": "3a05fc19"
   },
   "outputs": [],
   "source": [
    "df['entity_context'] = df['text'].apply(lambda x: extract_entity_context(x, entity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tkAgMsWfNbE6",
   "metadata": {
    "id": "tkAgMsWfNbE6"
   },
   "outputs": [],
   "source": [
    "df = df[df['entity_context'].str.len() > 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wyJxyCG5OK0l",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 458
    },
    "executionInfo": {
     "elapsed": 1020,
     "status": "ok",
     "timestamp": 1731647310441,
     "user": {
      "displayName": "pradyumna kr",
      "userId": "10110331870836060516"
     },
     "user_tz": 300
    },
    "id": "wyJxyCG5OK0l",
    "outputId": "e0f4e99e-eea1-4baf-d40d-760e723f2565"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entity_context</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>[case germany right author claim jewish year p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>[melachim ii ii king chapter ahaziah twentytwo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>[hello interest know jewish israeli think arab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>[may move israel near future take job offer je...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>[expect mail israeli university outside israel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92650</th>\n",
       "      <td>[see uptick amount comment accuse people suppo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92653</th>\n",
       "      <td>[israeli authority decline release ahmad manas...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92657</th>\n",
       "      <td>[important new palestinian topic understand pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92661</th>\n",
       "      <td>[past two month leave frustrated upset native ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92667</th>\n",
       "      <td>[every month politician west across almost par...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17767 rows × 1 columns</p>\n",
       "</div><br><label><b>dtype:</b> object</label>"
      ],
      "text/plain": [
       "84       [case germany right author claim jewish year p...\n",
       "91       [melachim ii ii king chapter ahaziah twentytwo...\n",
       "102      [hello interest know jewish israeli think arab...\n",
       "105      [may move israel near future take job offer je...\n",
       "121      [expect mail israeli university outside israel...\n",
       "                               ...                        \n",
       "92650    [see uptick amount comment accuse people suppo...\n",
       "92653    [israeli authority decline release ahmad manas...\n",
       "92657    [important new palestinian topic understand pa...\n",
       "92661    [past two month leave frustrated upset native ...\n",
       "92667    [every month politician west across almost par...\n",
       "Name: entity_context, Length: 17767, dtype: object"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['entity_context']"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}