code/datasetGeneration.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "datasetGeneration.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1Q-1sIow9eWSTQRD5eZvLSKz-ySuX_um8",
      "authorship_tag": "ABX9TyMCJLyOBpErljAR1Xgk5HMZ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/itissandeep98/StackOverFlow-Tag-Predictor/blob/master/datasetGeneration.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9zB1i4CdraKR"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G2FsyGWTJcaU"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd \n",
        "import csv,nltk,random,pickle\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.stem import PorterStemmer,WordNetLemmatizer,SnowballStemmer\n",
        "import re\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from wordcloud import WordCloud\n",
        "from sklearn import metrics\n",
        "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n",
        "from sklearn.model_selection import train_test_split,ShuffleSplit,GridSearchCV\n",
        "from sklearn.multiclass import OneVsRestClassifier\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.naive_bayes import GaussianNB,MultinomialNB,BernoulliNB\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.gaussian_process import GaussianProcessClassifier\n",
        "import pickle\n",
        "import joblib\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOj3niNqMhyE"
      },
      "source": [
        "af=pd.read_csv('/content/drive/MyDrive/Ml Project/Train.zip')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vEfckXZVc_Lx"
      },
      "source": [
        "# Pre-Process Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rRq3Gd3miqu-"
      },
      "source": [
        "def generate(n):\n",
        "  print(n)\n",
        "  \n",
        "  df=af[:n]\n",
        "\n",
        "  duplicate_pairs = df.sort_values('Title', ascending=False).duplicated('Title')\n",
        "  df = df[~duplicate_pairs]\n",
        "\n",
        "  df[\"Tags\"]=df[\"Tags\"].astype(str)\n",
        "  df[\"tag_count\"] = df[\"Tags\"].apply(lambda x : len(x.split()))\n",
        "  qus_list=[]\n",
        "\n",
        "  for index,row in df.iterrows():\n",
        "      title, body, tags = row[\"Title\"], row[\"Body\"], row[\"Tags\"]   \n",
        "      body=re.sub('<code>(.*?)</code>', '', body, flags=re.MULTILINE|re.DOTALL)\n",
        "      body = re.sub('<.*?>', ' ', str(body.encode('utf-8')))\n",
        "      title=title.encode('utf-8')\n",
        "      question=str(title)+\" \"+str(body)\n",
        "      question=re.sub(r'[^A-Za-z]+',' ',question)\n",
        "      words=word_tokenize(str(question.lower()))\n",
        "      stop_words = set(stopwords.words('english'))\n",
        "      stemmer = SnowballStemmer(\"english\")\n",
        "      question=' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))\n",
        "      qus_list.append(question)\n",
        "    \n",
        "  df[\"question\"] = qus_list\n",
        "  preprocessed_df = df[[\"question\",\"Tags\"]]\n",
        "  preprocessed_df.to_csv(\"preprocessed_\"+str(n)+\"k.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgW7lmS9-gmE"
      },
      "source": [
        "for i in range(10000,100001,10000):\n",
        "  generate(i)\n",
        "  !cp /content/preprocessed_\"$i\"k.csv /content/drive/MyDrive/MlProject/preprocessed_data/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OHX3sqngdEgn"
      },
      "source": [
        "# Vectorised Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2-6urnJcN2XZ"
      },
      "source": [
        "def tags_to_consider(n,multilabel_y):\n",
        "    tag_i_sum = multilabel_y.sum(axis=0).tolist()[0]\n",
        "    sorted_tags_i = sorted(range(len(tag_i_sum)), key=lambda i: tag_i_sum[i], reverse=True)\n",
        "    yn_multilabel=multilabel_y[:,sorted_tags_i[:n]]\n",
        "    return yn_multilabel"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c7onZfOpNiIB"
      },
      "source": [
        "def vectorize(n):\n",
        "  print(n)\n",
        "  preprocessed_df=pd.read_csv(\"/content/preprocessed_\"+str(n)+\"k.csv\")\n",
        "  preprocessed_df[\"tag_count\"] = preprocessed_df[\"Tags\"].apply(lambda text: len(text.split(\" \")))\n",
        "  vectorizer = CountVectorizer(tokenizer = lambda x: x.split())\n",
        "  multilabel_y = vectorizer.fit_transform(preprocessed_df['Tags'])\n",
        "  yx_multilabel = tags_to_consider(50,multilabel_y)\n",
        "  preprocessed_df.to_csv(\"vectorised_X_\"+str(n)+\"k.csv\")\n",
        "  joblib.dump(yx_multilabel, 'vectorised_y_'+str(n)+'.pkl') \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ZfUDqTjQfZ7"
      },
      "source": [
        "for i in range(10000,100001,10000):\n",
        "  vectorize(i)\n",
        "  !cp /content/vectorised_X_\"$i\"k.csv /content/drive/MyDrive/MlProject/data/\n",
        "  !cp /content/vectorised_y_\"$i\".pkl /content/drive/MyDrive/MlProject/data/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zlI5bw77UJNV"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}