sns_data_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# SNS Sentiment Analysis\n",
    "\n",
    "Analyze social media texts and measure potential inflammatory / offensive language."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Data Pre-processing\n",
    "\n",
    "From the selected datasets, extract the text and labels from all of them, then combine into one large CSV dataset.\n",
    "\n",
    "(Install NLTK data if not already installed)."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import nltk, os\n",
    "\n",
    "# Run this if you are locally accessing the NLTK data\n",
    "nltk.data.path.append('./nltk_data/')\n",
    "if not os.path.exists('./nltk_data'):\n",
    "    nltk.download('punkt', download_dir='./nltk_data/')\n",
    "    nltk.download('stopwords', download_dir='./nltk_data/')\n",
    "    nltk.download('words', download_dir='./nltk_data/')\n",
    "    nltk.download('brown', download_dir='./nltk_data/')"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-08T05:55:50.053395Z",
     "start_time": "2024-04-08T05:55:49.164737Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "hate_speech_dataset_path = \"./datasets/hate_speech_detect/HateSpeechDatasetBalanced.csv\"\n",
    "malignant_dataset_path = \"./datasets/malignant/train.csv\""
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-08T05:56:00.299479Z",
     "start_time": "2024-04-08T05:56:00.298531Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# Process malignant train data\n",
    "m_train_df = pd.read_csv(malignant_dataset_path)\n",
    "m_train_df_no_id = m_train_df.drop(columns=m_train_df.columns[0])\n",
    "\n",
    "processed_m_train_df = pd.DataFrame({\n",
    "    \"text\": m_train_df_no_id[m_train_df_no_id.columns[0]],\n",
    "    \"label\": m_train_df_no_id[m_train_df_no_id.columns[1:]].max(axis=1)\n",
    "})"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "import data_util as du"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-04-08T05:51:38.159203Z",
     "start_time": "2024-04-08T05:51:38.143993Z"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "hs_tuples = du.generate_tuples_from_file(hate_speech_dataset_path)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "m_tuples = du.generate_tuples_from_df(processed_m_train_df)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Combine and save the data to a CSV\n",
    "processed_data_save = pd.DataFrame({\n",
    "    \"text\": hs_tuples[0] + m_tuples[0],\n",
    "    \"label\": hs_tuples[1] + m_tuples[1]\n",
    "})\n",
    "\n",
    "processed_data_save.to_csv(\"./datasets/processed/all_data.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Load the saved data, only to fetch from the previous state\n",
    "import ast\n",
    "import pandas as pd\n",
    "\n",
    "complete_df = pd.read_csv(\"./datasets/processed/all_data.csv\")\n",
    "complete_df[complete_df.columns[0]] = complete_df[complete_df.columns[0]].apply(ast.literal_eval)\n",
    "complete_df[complete_df.columns[1]] = complete_df[complete_df.columns[1]].astype(int)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Our current dataset is too large, so we'll be using a portion of it in our actual model\n",
    "# Training data will be a size of 8,000\n",
    "# Test data will be a size of 2,000\n",
    "# This is a balanced dataset with half being labels 0 and 1\n",
    "df_0 = complete_df[complete_df[complete_df.columns[1]] == 0].sample(n=int(5e3), random_state=1).reset_index(drop=True)\n",
    "df_1 = complete_df[complete_df[complete_df.columns[1]] == 1].sample(n=int(5e3), random_state=1).reset_index(drop=True)\n",
    "\n",
    "assert len(df_0[df_0[df_0.columns[1]] == 0]) > 0 and len(df_0[df_0[df_0.columns[1]] == 1]) == 0\n",
    "assert len(df_1[df_1[df_1.columns[1]] == 1]) > 0 and len(df_1[df_1[df_1.columns[1]] == 0]) == 0\n",
    "\n",
    "combined_data = []\n",
    "for i in range(len(df_0)):\n",
    "    combined_data.append([df_0.iloc[i, 0], 0])\n",
    "    if i < len(df_1):\n",
    "        combined_data.append([df_1.iloc[i, 0], 1])\n",
    "\n",
    "combined_df = pd.DataFrame(combined_data, columns=[complete_df.columns[0], complete_df.columns[1]])\n",
    "\n",
    "training_text, test_text, training_labels, test_labels = train_test_split(combined_df.text, combined_df.label, test_size=0.2, random_state=42)\n",
    "\n",
    "training_data = pd.DataFrame({\n",
    "    \"text\": training_text,\n",
    "    \"label\": training_labels\n",
    "})\n",
    "\n",
    "test_data = pd.DataFrame({\n",
    "    \"text\": test_text,\n",
    "    \"label\": test_labels\n",
    "})\n",
    "#\n",
    "training_data.to_csv(\"./datasets/processed/train.csv\", index=False)\n",
    "test_data.to_csv(\"./datasets/processed/test.csv\", index=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}