NLP with Wordnet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled4.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNOct5T7nBgufhVGgru5S97",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Aytijha/Caramel_Popcorn_Movie_Time/blob/main/NLP%20with%20Wordnet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tgd2lqiyyxfC",
        "outputId": "98383ec1-015a-4300-eadf-1757310d29fe"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import re, string\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import SnowballStemmer\n",
        "from nltk.corpus import wordnet\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
        "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
        "\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "import gensim\n",
        "from gensim.models import Word2Vec"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "vXlKoFkBk7wY",
        "outputId": "0be0686b-f28b-4900-b2ef-231a705f2d4c"
      },
      "source": [
        "import pandas as pd\n",
        "nlp_df = pd.read_csv('NLP DF.csv')\n",
        "nlp_df"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Name of movie</th>\n",
              "      <th>Description</th>\n",
              "      <th>Genre</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>The Shawshank Redemption</td>\n",
              "      <td>Two imprisoned men bond over a number of years...</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>The Godfather</td>\n",
              "      <td>An organized crime dynasty's aging patriarch t...</td>\n",
              "      <td>Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>Soorarai Pottru</td>\n",
              "      <td>Nedumaaran Rajangam \"Maara\" sets out to make t...</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>The Dark Knight</td>\n",
              "      <td>When the menace known as the Joker wreaks havo...</td>\n",
              "      <td>Action Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>The Godfather: Part II</td>\n",
              "      <td>The early life and career of Vito Corleone in ...</td>\n",
              "      <td>Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>995</td>\n",
              "      <td>Giant</td>\n",
              "      <td>Sprawling epic covering the life of a Texas ca...</td>\n",
              "      <td>Drama Western</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>996</td>\n",
              "      <td>From Here to Eternity</td>\n",
              "      <td>In Hawaii in 1941, a private is cruelly punish...</td>\n",
              "      <td>Drama Romance War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>997</td>\n",
              "      <td>Gilda</td>\n",
              "      <td>A small-time gambler hired to work in a Buenos...</td>\n",
              "      <td>Drama Film-Noir Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>998</td>\n",
              "      <td>Lifeboat</td>\n",
              "      <td>Several survivors of a torpedoed merchant ship...</td>\n",
              "      <td>Drama War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>999</td>\n",
              "      <td>The 39 Steps</td>\n",
              "      <td>A man in London tries to help a counter-espion...</td>\n",
              "      <td>Crime Mystery Thriller</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     Unnamed: 0  ...                    Genre\n",
              "0             0  ...                    Drama\n",
              "1             1  ...              Crime Drama\n",
              "2             2  ...                    Drama\n",
              "3             3  ...       Action Crime Drama\n",
              "4             4  ...              Crime Drama\n",
              "..          ...  ...                      ...\n",
              "995         995  ...            Drama Western\n",
              "996         996  ...        Drama Romance War\n",
              "997         997  ...  Drama Film-Noir Romance\n",
              "998         998  ...                Drama War\n",
              "999         999  ...   Crime Mystery Thriller\n",
              "\n",
              "[1000 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IyscKIXZlVoi"
      },
      "source": [
        "nlp_df.drop('Unnamed: 0', inplace=True, axis=1)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "gkjUz6K-ld9H",
        "outputId": "3d64e979-d87a-4aaf-cc0a-fa1685d91868"
      },
      "source": [
        "nlp_df"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name of movie</th>\n",
              "      <th>Description</th>\n",
              "      <th>Genre</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The Shawshank Redemption</td>\n",
              "      <td>Two imprisoned men bond over a number of years...</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>The Godfather</td>\n",
              "      <td>An organized crime dynasty's aging patriarch t...</td>\n",
              "      <td>Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Soorarai Pottru</td>\n",
              "      <td>Nedumaaran Rajangam \"Maara\" sets out to make t...</td>\n",
              "      <td>Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The Dark Knight</td>\n",
              "      <td>When the menace known as the Joker wreaks havo...</td>\n",
              "      <td>Action Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The Godfather: Part II</td>\n",
              "      <td>The early life and career of Vito Corleone in ...</td>\n",
              "      <td>Crime Drama</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>Giant</td>\n",
              "      <td>Sprawling epic covering the life of a Texas ca...</td>\n",
              "      <td>Drama Western</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>From Here to Eternity</td>\n",
              "      <td>In Hawaii in 1941, a private is cruelly punish...</td>\n",
              "      <td>Drama Romance War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>Gilda</td>\n",
              "      <td>A small-time gambler hired to work in a Buenos...</td>\n",
              "      <td>Drama Film-Noir Romance</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>Lifeboat</td>\n",
              "      <td>Several survivors of a torpedoed merchant ship...</td>\n",
              "      <td>Drama War</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>The 39 Steps</td>\n",
              "      <td>A man in London tries to help a counter-espion...</td>\n",
              "      <td>Crime Mystery Thriller</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name of movie  ...                    Genre\n",
              "0    The Shawshank Redemption  ...                    Drama\n",
              "1               The Godfather  ...              Crime Drama\n",
              "2             Soorarai Pottru  ...                    Drama\n",
              "3             The Dark Knight  ...       Action Crime Drama\n",
              "4      The Godfather: Part II  ...              Crime Drama\n",
              "..                        ...  ...                      ...\n",
              "995                     Giant  ...            Drama Western\n",
              "996     From Here to Eternity  ...        Drama Romance War\n",
              "997                     Gilda  ...  Drama Film-Noir Romance\n",
              "998                  Lifeboat  ...                Drama War\n",
              "999              The 39 Steps  ...   Crime Mystery Thriller\n",
              "\n",
              "[1000 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-eqYcPcSlfis"
      },
      "source": [
        "d = dict(nlp_df['Genre'].value_counts())\n",
        "#d"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "lRQos7gTlopr",
        "outputId": "e44f0e49-ef7e-4e85-bc15-63841d802e9e"
      },
      "source": [
        "nlp_df['Text'] = nlp_df['Name of movie'] + ' ' + nlp_df['Description']\n",
        "nlp_df"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name of movie</th>\n",
              "      <th>Description</th>\n",
              "      <th>Genre</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The Shawshank Redemption</td>\n",
              "      <td>Two imprisoned men bond over a number of years...</td>\n",
              "      <td>Drama</td>\n",
              "      <td>The Shawshank Redemption Two imprisoned men bo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>The Godfather</td>\n",
              "      <td>An organized crime dynasty's aging patriarch t...</td>\n",
              "      <td>Crime Drama</td>\n",
              "      <td>The Godfather An organized crime dynasty's agi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Soorarai Pottru</td>\n",
              "      <td>Nedumaaran Rajangam \"Maara\" sets out to make t...</td>\n",
              "      <td>Drama</td>\n",
              "      <td>Soorarai Pottru Nedumaaran Rajangam \"Maara\" se...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The Dark Knight</td>\n",
              "      <td>When the menace known as the Joker wreaks havo...</td>\n",
              "      <td>Action Crime Drama</td>\n",
              "      <td>The Dark Knight When the menace known as the J...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The Godfather: Part II</td>\n",
              "      <td>The early life and career of Vito Corleone in ...</td>\n",
              "      <td>Crime Drama</td>\n",
              "      <td>The Godfather: Part II The early life and care...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>Giant</td>\n",
              "      <td>Sprawling epic covering the life of a Texas ca...</td>\n",
              "      <td>Drama Western</td>\n",
              "      <td>Giant Sprawling epic covering the life of a Te...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>From Here to Eternity</td>\n",
              "      <td>In Hawaii in 1941, a private is cruelly punish...</td>\n",
              "      <td>Drama Romance War</td>\n",
              "      <td>From Here to Eternity In Hawaii in 1941, a pri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>Gilda</td>\n",
              "      <td>A small-time gambler hired to work in a Buenos...</td>\n",
              "      <td>Drama Film-Noir Romance</td>\n",
              "      <td>Gilda A small-time gambler hired to work in a ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>Lifeboat</td>\n",
              "      <td>Several survivors of a torpedoed merchant ship...</td>\n",
              "      <td>Drama War</td>\n",
              "      <td>Lifeboat Several survivors of a torpedoed merc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>The 39 Steps</td>\n",
              "      <td>A man in London tries to help a counter-espion...</td>\n",
              "      <td>Crime Mystery Thriller</td>\n",
              "      <td>The 39 Steps A man in London tries to help a c...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name of movie  ...                                               Text\n",
              "0    The Shawshank Redemption  ...  The Shawshank Redemption Two imprisoned men bo...\n",
              "1               The Godfather  ...  The Godfather An organized crime dynasty's agi...\n",
              "2             Soorarai Pottru  ...  Soorarai Pottru Nedumaaran Rajangam \"Maara\" se...\n",
              "3             The Dark Knight  ...  The Dark Knight When the menace known as the J...\n",
              "4      The Godfather: Part II  ...  The Godfather: Part II The early life and care...\n",
              "..                        ...  ...                                                ...\n",
              "995                     Giant  ...  Giant Sprawling epic covering the life of a Te...\n",
              "996     From Here to Eternity  ...  From Here to Eternity In Hawaii in 1941, a pri...\n",
              "997                     Gilda  ...  Gilda A small-time gambler hired to work in a ...\n",
              "998                  Lifeboat  ...  Lifeboat Several survivors of a torpedoed merc...\n",
              "999              The 39 Steps  ...  The 39 Steps A man in London tries to help a c...\n",
              "\n",
              "[1000 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "StYW7Zstnl65"
      },
      "source": [
        "# converting text to lowercase, stripping and removing punctuations\n",
        "def preprocess(text):\n",
        "    text = text.lower() \n",
        "    text=text.strip()  \n",
        "    text=re.compile('<.*?>').sub('', text) \n",
        "    text = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', text)  \n",
        "    text = re.sub('\\s+', ' ', text)  \n",
        "    text = re.sub(r'\\[[0-9]*\\]',' ',text) \n",
        "    text=re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
        "    text = re.sub(r'\\d',' ',text) \n",
        "    text = re.sub(r'\\s+',' ',text) \n",
        "    return text"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ix4VNvYCv3U1"
      },
      "source": [
        "# removing stopwords\n",
        "def stopword(string):\n",
        "    a= [i for i in string.split() if i not in stopwords.words('english')]\n",
        "    return ' '.join(a)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OtSrzspyv9LI"
      },
      "source": [
        "# lemmatization\n",
        "wl = WordNetLemmatizer()\n",
        " \n",
        "# helper function to map NTLK position tags\n",
        "def get_wordnet_pos(tag):\n",
        "    if tag.startswith('J'):\n",
        "        return wordnet.ADJ\n",
        "    elif tag.startswith('V'):\n",
        "        return wordnet.VERB\n",
        "    elif tag.startswith('N'):\n",
        "        return wordnet.NOUN\n",
        "    elif tag.startswith('R'):\n",
        "        return wordnet.ADV\n",
        "    else:\n",
        "        return wordnet.NOUN\n",
        "\n",
        "# to tokenize the sentence\n",
        "def lemmatizer(string):\n",
        "    word_pos_tags = nltk.pos_tag(word_tokenize(string)) # Get position tags\n",
        "    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)] # Map the position tag and lemmatize the word/token\n",
        "    return \" \".join(a)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jwJ2TApI3J-i",
        "outputId": "40179afa-0526-456f-9f1e-bcda7fb295dd"
      },
      "source": [
        "nltk.download('stopwords')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 585
        },
        "id": "b1n_5YSuym0i",
        "outputId": "bd19d0e0-5894-469a-86d5-ba6fc352d8ae"
      },
      "source": [
        "def finalpreprocess(string):\n",
        "    return lemmatizer(stopword(preprocess(string)))\n",
        "nlp_df['clean_text'] = nlp_df['Text'].apply(lambda x: finalpreprocess(x))\n",
        "nlp_df"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name of movie</th>\n",
              "      <th>Description</th>\n",
              "      <th>Genre</th>\n",
              "      <th>Text</th>\n",
              "      <th>clean_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The Shawshank Redemption</td>\n",
              "      <td>Two imprisoned men bond over a number of years...</td>\n",
              "      <td>Drama</td>\n",
              "      <td>The Shawshank Redemption Two imprisoned men bo...</td>\n",
              "      <td>shawshank redemption two imprisoned men bond n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>The Godfather</td>\n",
              "      <td>An organized crime dynasty's aging patriarch t...</td>\n",
              "      <td>Crime Drama</td>\n",
              "      <td>The Godfather An organized crime dynasty's agi...</td>\n",
              "      <td>godfather organize crime dynasty age patriarch...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Soorarai Pottru</td>\n",
              "      <td>Nedumaaran Rajangam \"Maara\" sets out to make t...</td>\n",
              "      <td>Drama</td>\n",
              "      <td>Soorarai Pottru Nedumaaran Rajangam \"Maara\" se...</td>\n",
              "      <td>soorarai pottru nedumaaran rajangam maara set ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The Dark Knight</td>\n",
              "      <td>When the menace known as the Joker wreaks havo...</td>\n",
              "      <td>Action Crime Drama</td>\n",
              "      <td>The Dark Knight When the menace known as the J...</td>\n",
              "      <td>dark knight menace know joker wreaks havoc cha...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The Godfather: Part II</td>\n",
              "      <td>The early life and career of Vito Corleone in ...</td>\n",
              "      <td>Crime Drama</td>\n",
              "      <td>The Godfather: Part II The early life and care...</td>\n",
              "      <td>godfather part ii early life career vito corle...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>Giant</td>\n",
              "      <td>Sprawling epic covering the life of a Texas ca...</td>\n",
              "      <td>Drama Western</td>\n",
              "      <td>Giant Sprawling epic covering the life of a Te...</td>\n",
              "      <td>giant sprawl epic cover life texas cattle ranc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>From Here to Eternity</td>\n",
              "      <td>In Hawaii in 1941, a private is cruelly punish...</td>\n",
              "      <td>Drama Romance War</td>\n",
              "      <td>From Here to Eternity In Hawaii in 1941, a pri...</td>\n",
              "      <td>eternity hawaii private cruelly punish box uni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>Gilda</td>\n",
              "      <td>A small-time gambler hired to work in a Buenos...</td>\n",
              "      <td>Drama Film-Noir Romance</td>\n",
              "      <td>Gilda A small-time gambler hired to work in a ...</td>\n",
              "      <td>gilda small time gambler hire work buenos air ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>Lifeboat</td>\n",
              "      <td>Several survivors of a torpedoed merchant ship...</td>\n",
              "      <td>Drama War</td>\n",
              "      <td>Lifeboat Several survivors of a torpedoed merc...</td>\n",
              "      <td>lifeboat several survivor torpedo merchant shi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>The 39 Steps</td>\n",
              "      <td>A man in London tries to help a counter-espion...</td>\n",
              "      <td>Crime Mystery Thriller</td>\n",
              "      <td>The 39 Steps A man in London tries to help a c...</td>\n",
              "      <td>step man london try help counter espionage age...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name of movie  ...                                         clean_text\n",
              "0    The Shawshank Redemption  ...  shawshank redemption two imprisoned men bond n...\n",
              "1               The Godfather  ...  godfather organize crime dynasty age patriarch...\n",
              "2             Soorarai Pottru  ...  soorarai pottru nedumaaran rajangam maara set ...\n",
              "3             The Dark Knight  ...  dark knight menace know joker wreaks havoc cha...\n",
              "4      The Godfather: Part II  ...  godfather part ii early life career vito corle...\n",
              "..                        ...  ...                                                ...\n",
              "995                     Giant  ...  giant sprawl epic cover life texas cattle ranc...\n",
              "996     From Here to Eternity  ...  eternity hawaii private cruelly punish box uni...\n",
              "997                     Gilda  ...  gilda small time gambler hire work buenos air ...\n",
              "998                  Lifeboat  ...  lifeboat several survivor torpedo merchant shi...\n",
              "999              The 39 Steps  ...  step man london try help counter espionage age...\n",
              "\n",
              "[1000 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qnzt-avB2-MM",
        "outputId": "9d388b95-46a0-4c67-99b5-f0bdf98ca1df"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(nlp_df['clean_text'], nlp_df['Genre'], test_size=0.15)\n",
        "print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(850,) (150,) (850,) (150,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BTmqMgbU4gBk"
      },
      "source": [
        "#Tf-Idf\n",
        "tfidf_vectorizer = TfidfVectorizer(use_idf=True)\n",
        "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) \n",
        "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9dfDniUB4w2w",
        "outputId": "fcf47883-85de-4e6c-e158-95bb9e613571"
      },
      "source": [
        "#building Word2Vec model\n",
        "class MeanEmbeddingVectorizer(object):\n",
        "    def __init__(self, word2vec):\n",
        "        self.word2vec = word2vec\n",
        "        # if a text is empty we should return a vector of zeros\n",
        "        # with the same dimensionality as all the other vectors\n",
        "        self.dim = len(next(iter(word2vec.values())))\n",
        "def fit(self, X, y):\n",
        "        return self\n",
        "def transform(self, X):\n",
        "        return np.array([\n",
        "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
        "                    or [np.zeros(self.dim)], axis=0)\n",
        "            for words in X\n",
        "        ])\n",
        "\n",
        "nlp_df['clean_text_tok']=[nltk.word_tokenize(i) for i in nlp_df['clean_text']]\n",
        "model = Word2Vec(nlp_df['clean_text_tok'],min_count=1)\n",
        "w2v = dict(zip(model.wv.index2word, model.wv.syn0))   \n",
        "modelw = MeanEmbeddingVectorizer(w2v)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:19: DeprecationWarning: Call to deprecated `syn0` (Attribute will be removed in 4.0.0, use self.wv.vectors instead).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u_OFhfyA44aN"
      },
      "source": [
        "#Word2Vec\n",
        "# Word2Vec runs on tokenized sentences\n",
        "X_train_tok= [nltk.word_tokenize(i) for i in X_train]  \n",
        "X_test_tok= [nltk.word_tokenize(i) for i in X_test]"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}