Confusion Matrix AthonBot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1922ecc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import re\n",
    "import nltk\n",
    "import random\n",
    "import string\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "3c384535",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1003, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>Label</th>\n",
       "      <th>Username</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>none</td>\n",
       "      <td>chyrisalys</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>indirect complaint</td>\n",
       "      <td>woiidal</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>remark</td>\n",
       "      <td>ranieaw</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>negative remark</td>\n",
       "      <td>daeguv_</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>indirect complaint</td>\n",
       "      <td>fiorincha</td>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>998</td>\n",
       "      <td>campaign</td>\n",
       "      <td>sangwarior</td>\n",
       "      <td>Bisa sharing bareng sobat indihome gini kan as...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>999</td>\n",
       "      <td>campaign</td>\n",
       "      <td>sangwarior</td>\n",
       "      <td>Sekarang nyantai dulu bentar sambil scroll sos...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>1000</td>\n",
       "      <td>negative remark</td>\n",
       "      <td>untextend</td>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1001</th>\n",
       "      <td>1001</td>\n",
       "      <td>campaign</td>\n",
       "      <td>sangwarior</td>\n",
       "      <td>Ini barusan slesei sob... Lumayan buat nyari k...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>1002</td>\n",
       "      <td>inquiry</td>\n",
       "      <td>narshashila</td>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1003 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        No               Label     Username  \\\n",
       "0        0                none   chyrisalys   \n",
       "1        1  indirect complaint      woiidal   \n",
       "2        2              remark      ranieaw   \n",
       "3        3     negative remark      daeguv_   \n",
       "4        4  indirect complaint    fiorincha   \n",
       "...    ...                 ...          ...   \n",
       "998    998            campaign   sangwarior   \n",
       "999    999            campaign   sangwarior   \n",
       "1000  1000     negative remark    untextend   \n",
       "1001  1001            campaign   sangwarior   \n",
       "1002  1002             inquiry  narshashila   \n",
       "\n",
       "                                                  Tweet  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                       indihome ada masalah apasih??!!  \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...  \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...  \n",
       "4                                   INDIHOME NGAPASIIII  \n",
       "...                                                 ...  \n",
       "998   Bisa sharing bareng sobat indihome gini kan as...  \n",
       "999   Sekarang nyantai dulu bentar sambil scroll sos...  \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...  \n",
       "1001  Ini barusan slesei sob... Lumayan buat nyari k...  \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?  \n",
       "\n",
       "[1003 rows x 4 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('dataset/tweetlabels1000_labeled.xlsx - Sheet2.csv')\n",
    "print(df.shape)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "73271a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>Label</th>\n",
       "      <th>Username</th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>none</td>\n",
       "      <td>chyrisalys</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>indirect complaint</td>\n",
       "      <td>woiidal</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>remark</td>\n",
       "      <td>ranieaw</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>negative remark</td>\n",
       "      <td>daeguv_</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>indirect complaint</td>\n",
       "      <td>fiorincha</td>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>994</td>\n",
       "      <td>inquiry</td>\n",
       "      <td>pecintamochi</td>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>995</td>\n",
       "      <td>inquiry</td>\n",
       "      <td>tetehaisyah51</td>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>997</td>\n",
       "      <td>direct complaint</td>\n",
       "      <td>untextend</td>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>1000</td>\n",
       "      <td>negative remark</td>\n",
       "      <td>untextend</td>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>1002</td>\n",
       "      <td>inquiry</td>\n",
       "      <td>narshashila</td>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        No               Label       Username  \\\n",
       "0        0                none     chyrisalys   \n",
       "1        1  indirect complaint        woiidal   \n",
       "2        2              remark        ranieaw   \n",
       "3        3     negative remark        daeguv_   \n",
       "4        4  indirect complaint      fiorincha   \n",
       "...    ...                 ...            ...   \n",
       "994    994             inquiry   pecintamochi   \n",
       "995    995             inquiry  tetehaisyah51   \n",
       "997    997    direct complaint      untextend   \n",
       "1000  1000     negative remark      untextend   \n",
       "1002  1002             inquiry    narshashila   \n",
       "\n",
       "                                                  Tweet  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                       indihome ada masalah apasih??!!  \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...  \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...  \n",
       "4                                   INDIHOME NGAPASIIII  \n",
       "...                                                 ...  \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...  \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...  \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...  \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...  \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?  \n",
       "\n",
       "[722 rows x 4 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop rows with label = campaign, own tweet, & incomplete\n",
    "df = df.loc[~df['Label'].isin(['campaign','own tweet','incomplete'])].copy()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d3ca38c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label\n",
       "0                            wifi watcha pasti indihome      5\n",
       "1                       indihome ada masalah apasih??!!      0\n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1\n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2\n",
       "4                                   INDIHOME NGAPASIIII      0\n",
       "...                                                 ...    ...\n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6\n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6\n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4\n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2\n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6\n",
       "\n",
       "[722 rows x 2 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Label Encoding & drop columns\n",
    "df['label'] = df['Label'].map({'indirect complaint': 0, 'remark': 1, 'negative remark': 2, 'direct compliment' : 3 , 'direct complaint' : 4, 'none' : 5, 'inquiry' : 6})\n",
    "df.drop(['No', 'Label', 'Username'], axis=1, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d81e946b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>76</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Tweet\n",
       "label       \n",
       "0         68\n",
       "1        153\n",
       "2        128\n",
       "3         64\n",
       "4        110\n",
       "5        123\n",
       "6         76"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby( by='label').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0eea237b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet_lower</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, kalo jatuh tempo pembayaran...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, 1 indihome tv bisa gak berl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label  \\\n",
       "0                            wifi watcha pasti indihome      5   \n",
       "1                       indihome ada masalah apasih??!!      0   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2   \n",
       "4                                   INDIHOME NGAPASIIII      0   \n",
       "...                                                 ...    ...   \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6   \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6   \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4   \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2   \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6   \n",
       "\n",
       "                                            tweet_lower  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                       indihome ada masalah apasih??!!  \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...  \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...  \n",
       "4                                   indihome ngapasiiii  \n",
       "...                                                 ...  \n",
       "994   @indihomecare min, kalo jatuh tempo pembayaran...  \n",
       "995   @indihomecare min, 1 indihome tv bisa gak berl...  \n",
       "997   @indihomecare @fauzindrianto dari semalem down...  \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...  \n",
       "1002          @indihomecare daftar indihome lewat mana?  \n",
       "\n",
       "[722 rows x 3 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# to lowercase\n",
    "df['tweet_lower'] = df['Tweet'].str.lower()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c0c2c8e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_cleansing(text):\n",
    "    \n",
    "       \n",
    "    # remove non ASCII (emoticon, chinese word, etc)\n",
    "    text = text.encode('ascii', 'replace').decode('ascii')\n",
    "    \n",
    "    # remove digits (using regex) -> subtitute\n",
    "    text = re.sub('\\d+', '', text)\n",
    "    \n",
    "    # remove punctuation, reference: https://stackoverflow.com/a/34294398\n",
    "    #text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    \n",
    "    # remove punctuation, except mention tag @\n",
    "    text = re.sub(r'[^\\w\\s@]', '', text)\n",
    "    \n",
    "    # remove whitespace in the beginning and end of sentence\n",
    "    text = text.strip()\n",
    "    \n",
    "    # remove extra whitespace in the middle of sentence (using regex)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    \n",
    "    # remove url in tweet (using regex)\n",
    "    text = re.sub(r\"\\bhttp\\w+\", \"\", text)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "8ecebf05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet_lower</th>\n",
       "      <th>tweet_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>indihome ada masalah apasih</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>indihome plis untuk tanggal jangan kesurupan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>@indihomecare min kalo jatuh tempo pembayaran ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, 1 indihome tv bisa gak berl...</td>\n",
       "      <td>@indihomecare min indihome tv bisa gak berlang...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana?</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label  \\\n",
       "0                            wifi watcha pasti indihome      5   \n",
       "1                       indihome ada masalah apasih??!!      0   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2   \n",
       "4                                   INDIHOME NGAPASIIII      0   \n",
       "...                                                 ...    ...   \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6   \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6   \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4   \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2   \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6   \n",
       "\n",
       "                                            tweet_lower  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                       indihome ada masalah apasih??!!   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min, kalo jatuh tempo pembayaran...   \n",
       "995   @indihomecare min, 1 indihome tv bisa gak berl...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002          @indihomecare daftar indihome lewat mana?   \n",
       "\n",
       "                                            tweet_clean  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                           indihome ada masalah apasih  \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...  \n",
       "3          indihome plis untuk tanggal jangan kesurupan  \n",
       "4                                   indihome ngapasiiii  \n",
       "...                                                 ...  \n",
       "994   @indihomecare min kalo jatuh tempo pembayaran ...  \n",
       "995   @indihomecare min indihome tv bisa gak berlang...  \n",
       "997   @indihomecare @fauzindrianto dari semalem down...  \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...  \n",
       "1002           @indihomecare daftar indihome lewat mana  \n",
       "\n",
       "[722 rows x 4 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# text_cleansing()\n",
    "df['tweet_clean'] = df['tweet_lower'].apply(lambda x: text_cleansing(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "b9a1bafe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15166"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# normalization indonesian words\n",
    "normalized_word = pd.read_csv('dataset/new_kamusalay.csv', header=None)\n",
    "data_dict = dict(zip(normalized_word[0], normalized_word[1]))\n",
    "len(data_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "ace2c8a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_text(text):\n",
    "    return ' '.join(data_dict.get(word, word) for word in text.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7b4b36d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet_lower</th>\n",
       "      <th>tweet_clean</th>\n",
       "      <th>tweet_normalized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>indihome ada masalah apasih</td>\n",
       "      <td>indihome ada masalah apa sih</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocah cilik men...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>indihome plis untuk tanggal jangan kesurupan</td>\n",
       "      <td>indihome tolong untuk tanggal jangan kesurupan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>@indihomecare min kalo jatuh tempo pembayaran ...</td>\n",
       "      <td>@indihomecare min kalau jatuh tempo pembayaran...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, 1 indihome tv bisa gak berl...</td>\n",
       "      <td>@indihomecare min indihome tv bisa gak berlang...</td>\n",
       "      <td>@indihomecare min indihome televisi bisa tidak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalam down...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dari semalam j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana?</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label  \\\n",
       "0                            wifi watcha pasti indihome      5   \n",
       "1                       indihome ada masalah apasih??!!      0   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2   \n",
       "4                                   INDIHOME NGAPASIIII      0   \n",
       "...                                                 ...    ...   \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6   \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6   \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4   \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2   \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6   \n",
       "\n",
       "                                            tweet_lower  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                       indihome ada masalah apasih??!!   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min, kalo jatuh tempo pembayaran...   \n",
       "995   @indihomecare min, 1 indihome tv bisa gak berl...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002          @indihomecare daftar indihome lewat mana?   \n",
       "\n",
       "                                            tweet_clean  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                           indihome ada masalah apasih   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3          indihome plis untuk tanggal jangan kesurupan   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min kalo jatuh tempo pembayaran ...   \n",
       "995   @indihomecare min indihome tv bisa gak berlang...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002           @indihomecare daftar indihome lewat mana   \n",
       "\n",
       "                                       tweet_normalized  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                          indihome ada masalah apa sih  \n",
       "2     sore sore hujan rebahan bareng bocah cilik men...  \n",
       "3        indihome tolong untuk tanggal jangan kesurupan  \n",
       "4                                   indihome ngapasiiii  \n",
       "...                                                 ...  \n",
       "994   @indihomecare min kalau jatuh tempo pembayaran...  \n",
       "995   @indihomecare min indihome televisi bisa tidak...  \n",
       "997   @indihomecare @fauzindrianto dari semalam down...  \n",
       "1000  @jefrihandri sini indihome down dari semalam j...  \n",
       "1002           @indihomecare daftar indihome lewat mana  \n",
       "\n",
       "[722 rows x 5 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tweet_normalized'] = df['tweet_clean'].apply(lambda x: normalize_text(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "cae9c292",
   "metadata": {},
   "outputs": [],
   "source": [
    "# stemming 'kata berimbuhan'\n",
    "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory\n",
    "\n",
    "# create stemmer\n",
    "factory = StemmerFactory()\n",
    "stemmer = factory.create_stemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bbaa386f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet_lower</th>\n",
       "      <th>tweet_clean</th>\n",
       "      <th>tweet_normalized</th>\n",
       "      <th>tweet_stemmed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>indihome ada masalah apasih</td>\n",
       "      <td>indihome ada masalah apa sih</td>\n",
       "      <td>indihome ada masalah apa sih</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocah cilik men...</td>\n",
       "      <td>sore sore hujan rebah bareng bocah cilik tonto...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>indihome plis untuk tanggal jangan kesurupan</td>\n",
       "      <td>indihome tolong untuk tanggal jangan kesurupan</td>\n",
       "      <td>indihome tolong untuk tanggal jangan surup</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>@indihomecare min kalo jatuh tempo pembayaran ...</td>\n",
       "      <td>@indihomecare min kalau jatuh tempo pembayaran...</td>\n",
       "      <td>indihomecare min kalau jatuh tempo bayar indih...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, 1 indihome tv bisa gak berl...</td>\n",
       "      <td>@indihomecare min indihome tv bisa gak berlang...</td>\n",
       "      <td>@indihomecare min indihome televisi bisa tidak...</td>\n",
       "      <td>indihomecare min indihome televisi bisa tidak ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalam down...</td>\n",
       "      <td>indihomecare fauzindrianto dari malam down tid...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dari semalam j...</td>\n",
       "      <td>jefrihandri sini indihome down dari malam jam ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana?</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "      <td>indihomecare daftar indihome lewat mana</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label  \\\n",
       "0                            wifi watcha pasti indihome      5   \n",
       "1                       indihome ada masalah apasih??!!      0   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2   \n",
       "4                                   INDIHOME NGAPASIIII      0   \n",
       "...                                                 ...    ...   \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6   \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6   \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4   \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2   \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6   \n",
       "\n",
       "                                            tweet_lower  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                       indihome ada masalah apasih??!!   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min, kalo jatuh tempo pembayaran...   \n",
       "995   @indihomecare min, 1 indihome tv bisa gak berl...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002          @indihomecare daftar indihome lewat mana?   \n",
       "\n",
       "                                            tweet_clean  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                           indihome ada masalah apasih   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3          indihome plis untuk tanggal jangan kesurupan   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min kalo jatuh tempo pembayaran ...   \n",
       "995   @indihomecare min indihome tv bisa gak berlang...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002           @indihomecare daftar indihome lewat mana   \n",
       "\n",
       "                                       tweet_normalized  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                          indihome ada masalah apa sih   \n",
       "2     sore sore hujan rebahan bareng bocah cilik men...   \n",
       "3        indihome tolong untuk tanggal jangan kesurupan   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min kalau jatuh tempo pembayaran...   \n",
       "995   @indihomecare min indihome televisi bisa tidak...   \n",
       "997   @indihomecare @fauzindrianto dari semalam down...   \n",
       "1000  @jefrihandri sini indihome down dari semalam j...   \n",
       "1002           @indihomecare daftar indihome lewat mana   \n",
       "\n",
       "                                          tweet_stemmed  \n",
       "0                            wifi watcha pasti indihome  \n",
       "1                          indihome ada masalah apa sih  \n",
       "2     sore sore hujan rebah bareng bocah cilik tonto...  \n",
       "3            indihome tolong untuk tanggal jangan surup  \n",
       "4                                   indihome ngapasiiii  \n",
       "...                                                 ...  \n",
       "994   indihomecare min kalau jatuh tempo bayar indih...  \n",
       "995   indihomecare min indihome televisi bisa tidak ...  \n",
       "997   indihomecare fauzindrianto dari malam down tid...  \n",
       "1000  jefrihandri sini indihome down dari malam jam ...  \n",
       "1002            indihomecare daftar indihome lewat mana  \n",
       "\n",
       "[722 rows x 6 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tweet_stemmed'] = df['tweet_normalized'].apply(lambda x: stemmer.stem(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "e8f54c72",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to compute the max length of sequence\n",
    "def max_length(sequences):\n",
    "    '''\n",
    "    input:\n",
    "        sequences: a 2D list of integer sequences\n",
    "    output:\n",
    "        max_length: the max length of the sequences\n",
    "    '''\n",
    "    max_length = 0\n",
    "    for i, seq in enumerate(sequences):\n",
    "        length = len(seq)\n",
    "        if max_length < length:\n",
    "            max_length = length\n",
    "    return max_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "91fc601c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.constraints import MaxNorm\n",
    "\n",
    "def define_model(input_dim = None, output_dim=300, max_length = None ):\n",
    "    \n",
    "    model = tf.keras.models.Sequential([\n",
    "        tf.keras.layers.Embedding(input_dim=input_dim, \n",
    "                                  mask_zero= True,\n",
    "                                  output_dim=output_dim, \n",
    "                                  input_length=max_length, \n",
    "                                  input_shape=(max_length, )),\n",
    "        tf.keras.layers.Bidirectional(tf.keras.layers.GRU(64, return_sequences=True)),\n",
    "        tf.keras.layers.Bidirectional(tf.keras.layers.GRU(64, return_sequences=False)),\n",
    "        tf.keras.layers.Dropout(0.5),\n",
    "        tf.keras.layers.Dense(units=7, activation='sigmoid')\n",
    "    ])\n",
    "    \n",
    "    model.compile( loss = 'categorical_crossentropy', optimizer = 'adam', metrics = ['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f1649451",
   "metadata": {},
   "outputs": [],
   "source": [
    "class myCallback(tf.keras.callbacks.Callback):\n",
    "    # Overide the method on_epoch_end() for our benefit\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if (logs.get('accuracy') > 0.93):\n",
    "            print(\"\\nReached 93% accuracy so cancelling training!\")\n",
    "            self.model.stop_training=True\n",
    "\n",
    "\n",
    "callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', min_delta=0, \n",
    "                                             patience=10, verbose=2, \n",
    "                                             mode='auto', restore_best_weights=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "9f2ce60c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training 1: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 22s 413ms/step - loss: 1.9012 - accuracy: 0.2558 - val_loss: 2.0164 - val_accuracy: 0.0685\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 161ms/step - loss: 1.7113 - accuracy: 0.3282 - val_loss: 1.9348 - val_accuracy: 0.2603\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 164ms/step - loss: 1.2766 - accuracy: 0.5778 - val_loss: 2.0649 - val_accuracy: 0.3288\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 4s 183ms/step - loss: 0.7592 - accuracy: 0.7535 - val_loss: 2.0824 - val_accuracy: 0.3425\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 4s 174ms/step - loss: 0.4177 - accuracy: 0.8706 - val_loss: 2.3121 - val_accuracy: 0.2466\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 4s 169ms/step - loss: 0.2182 - accuracy: 0.9368 - val_loss: 2.5871 - val_accuracy: 0.2603\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 160ms/step - loss: 0.1158 - accuracy: 0.9738 - val_loss: 3.1034 - val_accuracy: 0.2329\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 162ms/step - loss: 0.0775 - accuracy: 0.9753 - val_loss: 3.0243 - val_accuracy: 0.2329\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 164ms/step - loss: 0.0658 - accuracy: 0.9861 - val_loss: 3.2735 - val_accuracy: 0.2740\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 4s 169ms/step - loss: 0.0531 - accuracy: 0.9831 - val_loss: 3.3670 - val_accuracy: 0.2466\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 160ms/step - loss: 0.0593 - accuracy: 0.9753 - val_loss: 3.6642 - val_accuracy: 0.2329\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 165ms/step - loss: 0.0492 - accuracy: 0.9846 - val_loss: 3.6027 - val_accuracy: 0.2466\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 163ms/step - loss: 0.0310 - accuracy: 0.9877 - val_loss: 3.6670 - val_accuracy: 0.2466\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0311 - accuracy: 0.9938Restoring model weights from the end of the best epoch: 4.\n",
      "21/21 [==============================] - 4s 168ms/step - loss: 0.0311 - accuracy: 0.9938 - val_loss: 3.9003 - val_accuracy: 0.2740\n",
      "Epoch 00014: early stopping\n",
      "Test Accuracy: 34.24657583236694\n",
      "Training 2: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 23s 441ms/step - loss: 1.9046 - accuracy: 0.2327 - val_loss: 1.9087 - val_accuracy: 0.1918\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 4s 186ms/step - loss: 1.7158 - accuracy: 0.3898 - val_loss: 1.7746 - val_accuracy: 0.3425\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 4s 168ms/step - loss: 1.2854 - accuracy: 0.5639 - val_loss: 1.7557 - val_accuracy: 0.3425\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 4s 167ms/step - loss: 0.7809 - accuracy: 0.7442 - val_loss: 1.7385 - val_accuracy: 0.3973\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 4s 195ms/step - loss: 0.4240 - accuracy: 0.8659 - val_loss: 2.0240 - val_accuracy: 0.3699\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 4s 177ms/step - loss: 0.2201 - accuracy: 0.9368 - val_loss: 2.3550 - val_accuracy: 0.4110\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 4s 181ms/step - loss: 0.1462 - accuracy: 0.9538 - val_loss: 2.4708 - val_accuracy: 0.3425\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 4s 185ms/step - loss: 0.0910 - accuracy: 0.9753 - val_loss: 2.7286 - val_accuracy: 0.3288\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 4s 198ms/step - loss: 0.0632 - accuracy: 0.9831 - val_loss: 2.7637 - val_accuracy: 0.3973\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 4s 175ms/step - loss: 0.0529 - accuracy: 0.9831 - val_loss: 2.9565 - val_accuracy: 0.3562\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 4s 176ms/step - loss: 0.0494 - accuracy: 0.9861 - val_loss: 3.0749 - val_accuracy: 0.3699\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 4s 177ms/step - loss: 0.0511 - accuracy: 0.9831 - val_loss: 3.2953 - val_accuracy: 0.3836\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 4s 174ms/step - loss: 0.0494 - accuracy: 0.9892 - val_loss: 3.3122 - val_accuracy: 0.3425\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 6s 272ms/step - loss: 0.0301 - accuracy: 0.9923 - val_loss: 3.4595 - val_accuracy: 0.3562\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 5s 233ms/step - loss: 0.0297 - accuracy: 0.9892 - val_loss: 3.4964 - val_accuracy: 0.3425\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0282 - accuracy: 0.9938Restoring model weights from the end of the best epoch: 6.\n",
      "21/21 [==============================] - 5s 233ms/step - loss: 0.0282 - accuracy: 0.9938 - val_loss: 3.6220 - val_accuracy: 0.3425\n",
      "Epoch 00016: early stopping\n",
      "Test Accuracy: 41.09589159488678\n",
      "Training 3: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 21s 378ms/step - loss: 1.9065 - accuracy: 0.2154 - val_loss: 1.8987 - val_accuracy: 0.2083\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 147ms/step - loss: 1.7339 - accuracy: 0.3677 - val_loss: 1.7729 - val_accuracy: 0.3611\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 151ms/step - loss: 1.2359 - accuracy: 0.5985 - val_loss: 1.6770 - val_accuracy: 0.3611\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 3s 144ms/step - loss: 0.6808 - accuracy: 0.7738 - val_loss: 1.7687 - val_accuracy: 0.3889\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 143ms/step - loss: 0.3354 - accuracy: 0.9077 - val_loss: 1.8990 - val_accuracy: 0.4306\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 144ms/step - loss: 0.1728 - accuracy: 0.9492 - val_loss: 2.4509 - val_accuracy: 0.3889\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 144ms/step - loss: 0.0944 - accuracy: 0.9815 - val_loss: 2.4698 - val_accuracy: 0.4167\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 157ms/step - loss: 0.0660 - accuracy: 0.9862 - val_loss: 2.9243 - val_accuracy: 0.4167\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 146ms/step - loss: 0.0736 - accuracy: 0.9785 - val_loss: 2.9180 - val_accuracy: 0.4167\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 4s 174ms/step - loss: 0.0457 - accuracy: 0.9862 - val_loss: 3.1532 - val_accuracy: 0.4306\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 150ms/step - loss: 0.0373 - accuracy: 0.9892 - val_loss: 3.4628 - val_accuracy: 0.4306\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 164ms/step - loss: 0.0335 - accuracy: 0.9892 - val_loss: 3.4709 - val_accuracy: 0.4306\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 6s 269ms/step - loss: 0.0355 - accuracy: 0.9892 - val_loss: 3.5350 - val_accuracy: 0.4306\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 4s 207ms/step - loss: 0.0386 - accuracy: 0.9831 - val_loss: 3.7044 - val_accuracy: 0.4167\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0329 - accuracy: 0.9908Restoring model weights from the end of the best epoch: 5.\n",
      "21/21 [==============================] - 4s 178ms/step - loss: 0.0329 - accuracy: 0.9908 - val_loss: 3.6507 - val_accuracy: 0.4306\n",
      "Epoch 00015: early stopping\n",
      "Test Accuracy: 43.05555522441864\n",
      "Training 4: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 20s 365ms/step - loss: 1.9052 - accuracy: 0.2615 - val_loss: 1.8499 - val_accuracy: 0.3750\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 147ms/step - loss: 1.7110 - accuracy: 0.3892 - val_loss: 1.6369 - val_accuracy: 0.4028\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 153ms/step - loss: 1.2718 - accuracy: 0.5569 - val_loss: 1.4571 - val_accuracy: 0.3889\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 3s 153ms/step - loss: 0.7504 - accuracy: 0.7477 - val_loss: 1.7711 - val_accuracy: 0.4167\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 4s 170ms/step - loss: 0.4124 - accuracy: 0.8846 - val_loss: 1.8400 - val_accuracy: 0.3611\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 162ms/step - loss: 0.2026 - accuracy: 0.9508 - val_loss: 2.1253 - val_accuracy: 0.3889\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 146ms/step - loss: 0.1207 - accuracy: 0.9569 - val_loss: 2.2953 - val_accuracy: 0.3611\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 0.1028 - accuracy: 0.9754 - val_loss: 2.7976 - val_accuracy: 0.3333\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 139ms/step - loss: 0.0953 - accuracy: 0.9708 - val_loss: 2.6297 - val_accuracy: 0.3889\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 134ms/step - loss: 0.0688 - accuracy: 0.9785 - val_loss: 2.7559 - val_accuracy: 0.3750\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 135ms/step - loss: 0.0523 - accuracy: 0.9877 - val_loss: 2.8804 - val_accuracy: 0.3889\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 136ms/step - loss: 0.0412 - accuracy: 0.9862 - val_loss: 3.0637 - val_accuracy: 0.4028\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 138ms/step - loss: 0.0298 - accuracy: 0.9908 - val_loss: 3.2368 - val_accuracy: 0.3889\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0307 - accuracy: 0.9908Restoring model weights from the end of the best epoch: 4.\n",
      "21/21 [==============================] - 3s 136ms/step - loss: 0.0307 - accuracy: 0.9908 - val_loss: 3.3327 - val_accuracy: 0.3889\n",
      "Epoch 00014: early stopping\n",
      "Test Accuracy: 41.66666567325592\n",
      "Training 5: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 18s 321ms/step - loss: 1.9120 - accuracy: 0.2431 - val_loss: 1.8561 - val_accuracy: 0.4583\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 130ms/step - loss: 1.7304 - accuracy: 0.4415 - val_loss: 1.6764 - val_accuracy: 0.4583\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 135ms/step - loss: 1.2254 - accuracy: 0.5723 - val_loss: 1.5635 - val_accuracy: 0.4306\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 3s 143ms/step - loss: 0.7146 - accuracy: 0.7662 - val_loss: 1.7852 - val_accuracy: 0.4722\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 129ms/step - loss: 0.3490 - accuracy: 0.9046 - val_loss: 2.2373 - val_accuracy: 0.4306\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 142ms/step - loss: 0.2059 - accuracy: 0.9492 - val_loss: 2.6382 - val_accuracy: 0.3750\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 137ms/step - loss: 0.1447 - accuracy: 0.9538 - val_loss: 2.6705 - val_accuracy: 0.4028\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 136ms/step - loss: 0.1022 - accuracy: 0.9738 - val_loss: 2.7212 - val_accuracy: 0.4167\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 142ms/step - loss: 0.0735 - accuracy: 0.9831 - val_loss: 2.9268 - val_accuracy: 0.4028\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 136ms/step - loss: 0.0631 - accuracy: 0.9815 - val_loss: 3.1506 - val_accuracy: 0.3889\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 135ms/step - loss: 0.0576 - accuracy: 0.9831 - val_loss: 3.1989 - val_accuracy: 0.4028\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 133ms/step - loss: 0.0509 - accuracy: 0.9846 - val_loss: 3.2664 - val_accuracy: 0.4167\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 0.0501 - accuracy: 0.9862 - val_loss: 3.4488 - val_accuracy: 0.4028\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0541 - accuracy: 0.9877Restoring model weights from the end of the best epoch: 4.\n",
      "21/21 [==============================] - 3s 134ms/step - loss: 0.0541 - accuracy: 0.9877 - val_loss: 3.3416 - val_accuracy: 0.4167\n",
      "Epoch 00014: early stopping\n",
      "Test Accuracy: 47.22222089767456\n",
      "Training 6: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 19s 359ms/step - loss: 1.9064 - accuracy: 0.2031 - val_loss: 1.8549 - val_accuracy: 0.1944\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 156ms/step - loss: 1.7272 - accuracy: 0.3200 - val_loss: 1.6651 - val_accuracy: 0.3333\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 156ms/step - loss: 1.2563 - accuracy: 0.5662 - val_loss: 1.4157 - val_accuracy: 0.4583\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 4s 166ms/step - loss: 0.7266 - accuracy: 0.7815 - val_loss: 1.4474 - val_accuracy: 0.4306\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 149ms/step - loss: 0.3808 - accuracy: 0.9015 - val_loss: 1.5444 - val_accuracy: 0.4583\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 155ms/step - loss: 0.1876 - accuracy: 0.9508 - val_loss: 1.7226 - val_accuracy: 0.4583\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 150ms/step - loss: 0.1232 - accuracy: 0.9723 - val_loss: 1.9709 - val_accuracy: 0.5139\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 155ms/step - loss: 0.0931 - accuracy: 0.9738 - val_loss: 2.0517 - val_accuracy: 0.4722\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.0718 - accuracy: 0.9769 - val_loss: 2.3088 - val_accuracy: 0.4306\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 4s 178ms/step - loss: 0.1031 - accuracy: 0.9754 - val_loss: 2.2345 - val_accuracy: 0.4306\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 146ms/step - loss: 0.0618 - accuracy: 0.9877 - val_loss: 2.4647 - val_accuracy: 0.5000\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 4s 175ms/step - loss: 0.0710 - accuracy: 0.9769 - val_loss: 2.3899 - val_accuracy: 0.4583\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 153ms/step - loss: 0.0625 - accuracy: 0.9815 - val_loss: 2.5159 - val_accuracy: 0.4444\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 3s 145ms/step - loss: 0.0391 - accuracy: 0.9923 - val_loss: 2.6630 - val_accuracy: 0.4583\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 3s 160ms/step - loss: 0.0441 - accuracy: 0.9877 - val_loss: 2.6784 - val_accuracy: 0.4722\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - 4s 168ms/step - loss: 0.0420 - accuracy: 0.9862 - val_loss: 2.6869 - val_accuracy: 0.4444\n",
      "Epoch 17/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0382 - accuracy: 0.9846Restoring model weights from the end of the best epoch: 7.\n",
      "21/21 [==============================] - 4s 189ms/step - loss: 0.0382 - accuracy: 0.9846 - val_loss: 2.7477 - val_accuracy: 0.4306\n",
      "Epoch 00017: early stopping\n",
      "Test Accuracy: 51.38888955116272\n",
      "Training 7: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 21s 394ms/step - loss: 1.9173 - accuracy: 0.2123 - val_loss: 1.8316 - val_accuracy: 0.2500\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 137ms/step - loss: 1.7590 - accuracy: 0.3862 - val_loss: 1.6584 - val_accuracy: 0.3889\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 4s 181ms/step - loss: 1.3033 - accuracy: 0.5631 - val_loss: 1.3988 - val_accuracy: 0.5000\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 4s 174ms/step - loss: 0.7465 - accuracy: 0.7369 - val_loss: 1.5647 - val_accuracy: 0.3750\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 163ms/step - loss: 0.3790 - accuracy: 0.8954 - val_loss: 1.6316 - val_accuracy: 0.4722\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 162ms/step - loss: 0.2706 - accuracy: 0.9138 - val_loss: 1.7553 - val_accuracy: 0.3611\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 163ms/step - loss: 0.1630 - accuracy: 0.9523 - val_loss: 1.8068 - val_accuracy: 0.5000\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 158ms/step - loss: 0.1228 - accuracy: 0.9708 - val_loss: 2.1089 - val_accuracy: 0.4444\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 158ms/step - loss: 0.0902 - accuracy: 0.9800 - val_loss: 2.2935 - val_accuracy: 0.4167\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 165ms/step - loss: 0.0677 - accuracy: 0.9846 - val_loss: 2.3644 - val_accuracy: 0.3750\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 4s 173ms/step - loss: 0.0524 - accuracy: 0.9862 - val_loss: 2.8710 - val_accuracy: 0.3750\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 4s 195ms/step - loss: 0.0448 - accuracy: 0.9923 - val_loss: 2.3978 - val_accuracy: 0.4167\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0397 - accuracy: 0.9877Restoring model weights from the end of the best epoch: 3.\n",
      "21/21 [==============================] - 4s 178ms/step - loss: 0.0397 - accuracy: 0.9877 - val_loss: 2.9064 - val_accuracy: 0.3611\n",
      "Epoch 00013: early stopping\n",
      "Test Accuracy: 50.0\n",
      "Training 8: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 21s 391ms/step - loss: 1.9078 - accuracy: 0.2462 - val_loss: 1.8318 - val_accuracy: 0.3333\n",
      "Epoch 2/30\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21/21 [==============================] - 3s 156ms/step - loss: 1.7073 - accuracy: 0.3292 - val_loss: 1.6021 - val_accuracy: 0.3750\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 4s 169ms/step - loss: 1.2712 - accuracy: 0.5446 - val_loss: 1.4161 - val_accuracy: 0.4722\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 3s 145ms/step - loss: 0.8103 - accuracy: 0.7354 - val_loss: 1.4995 - val_accuracy: 0.4722\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 0.4590 - accuracy: 0.8646 - val_loss: 1.9689 - val_accuracy: 0.4167\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 155ms/step - loss: 0.2417 - accuracy: 0.9308 - val_loss: 1.7187 - val_accuracy: 0.5417\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 149ms/step - loss: 0.1533 - accuracy: 0.9508 - val_loss: 2.3040 - val_accuracy: 0.4444\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 151ms/step - loss: 0.1150 - accuracy: 0.9646 - val_loss: 2.0346 - val_accuracy: 0.5139\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.0778 - accuracy: 0.9785 - val_loss: 2.4553 - val_accuracy: 0.4722\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 142ms/step - loss: 0.0633 - accuracy: 0.9815 - val_loss: 2.5915 - val_accuracy: 0.5139\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 4s 169ms/step - loss: 0.0581 - accuracy: 0.9831 - val_loss: 2.6742 - val_accuracy: 0.4028\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 151ms/step - loss: 0.0528 - accuracy: 0.9923 - val_loss: 2.6635 - val_accuracy: 0.4861\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.0405 - accuracy: 0.9892 - val_loss: 2.6104 - val_accuracy: 0.4306\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.0335 - accuracy: 0.9908 - val_loss: 2.6041 - val_accuracy: 0.4167\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.0447 - accuracy: 0.9862 - val_loss: 2.6729 - val_accuracy: 0.4583\n",
      "Epoch 16/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0317 - accuracy: 0.9923Restoring model weights from the end of the best epoch: 6.\n",
      "21/21 [==============================] - 3s 147ms/step - loss: 0.0317 - accuracy: 0.9923 - val_loss: 2.8225 - val_accuracy: 0.5000\n",
      "Epoch 00016: early stopping\n",
      "Test Accuracy: 54.16666865348816\n",
      "Training 9: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 19s 337ms/step - loss: 1.9127 - accuracy: 0.2908 - val_loss: 1.8421 - val_accuracy: 0.2222\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 1.7321 - accuracy: 0.3938 - val_loss: 1.7204 - val_accuracy: 0.3056\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 168ms/step - loss: 1.2754 - accuracy: 0.5385 - val_loss: 1.6872 - val_accuracy: 0.3750\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 4s 189ms/step - loss: 0.7570 - accuracy: 0.7246 - val_loss: 1.6791 - val_accuracy: 0.3889\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 148ms/step - loss: 0.4218 - accuracy: 0.8662 - val_loss: 1.9579 - val_accuracy: 0.4028\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 143ms/step - loss: 0.2360 - accuracy: 0.9246 - val_loss: 2.2510 - val_accuracy: 0.3333\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 138ms/step - loss: 0.1705 - accuracy: 0.9508 - val_loss: 2.3455 - val_accuracy: 0.3889\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 139ms/step - loss: 0.1137 - accuracy: 0.9692 - val_loss: 2.5552 - val_accuracy: 0.3611\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 135ms/step - loss: 0.0845 - accuracy: 0.9800 - val_loss: 2.6133 - val_accuracy: 0.3889\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 146ms/step - loss: 0.0723 - accuracy: 0.9815 - val_loss: 2.7475 - val_accuracy: 0.3750\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 156ms/step - loss: 0.0469 - accuracy: 0.9877 - val_loss: 2.9850 - val_accuracy: 0.3333\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 155ms/step - loss: 0.0553 - accuracy: 0.9877 - val_loss: 3.1472 - val_accuracy: 0.3472\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - 3s 146ms/step - loss: 0.0489 - accuracy: 0.9846 - val_loss: 3.1715 - val_accuracy: 0.3472\n",
      "Epoch 14/30\n",
      "21/21 [==============================] - 3s 139ms/step - loss: 0.0484 - accuracy: 0.9831 - val_loss: 3.3720 - val_accuracy: 0.3333\n",
      "Epoch 15/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0379 - accuracy: 0.9862Restoring model weights from the end of the best epoch: 5.\n",
      "21/21 [==============================] - 3s 138ms/step - loss: 0.0379 - accuracy: 0.9862 - val_loss: 3.3242 - val_accuracy: 0.3472\n",
      "Epoch 00015: early stopping\n",
      "Test Accuracy: 40.27777910232544\n",
      "Training 10: \n",
      "Epoch 1/30\n",
      "21/21 [==============================] - 23s 508ms/step - loss: 1.9069 - accuracy: 0.2246 - val_loss: 1.8866 - val_accuracy: 0.1667\n",
      "Epoch 2/30\n",
      "21/21 [==============================] - 3s 152ms/step - loss: 1.7075 - accuracy: 0.3369 - val_loss: 1.6223 - val_accuracy: 0.4028\n",
      "Epoch 3/30\n",
      "21/21 [==============================] - 3s 154ms/step - loss: 1.2737 - accuracy: 0.5569 - val_loss: 1.5738 - val_accuracy: 0.4306\n",
      "Epoch 4/30\n",
      "21/21 [==============================] - 3s 142ms/step - loss: 0.7903 - accuracy: 0.7323 - val_loss: 1.8106 - val_accuracy: 0.4167\n",
      "Epoch 5/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 0.4470 - accuracy: 0.8631 - val_loss: 2.2107 - val_accuracy: 0.4167\n",
      "Epoch 6/30\n",
      "21/21 [==============================] - 3s 147ms/step - loss: 0.2550 - accuracy: 0.9415 - val_loss: 2.5081 - val_accuracy: 0.4028\n",
      "Epoch 7/30\n",
      "21/21 [==============================] - 3s 131ms/step - loss: 0.1517 - accuracy: 0.9646 - val_loss: 3.0964 - val_accuracy: 0.3472\n",
      "Epoch 8/30\n",
      "21/21 [==============================] - 3s 142ms/step - loss: 0.1038 - accuracy: 0.9738 - val_loss: 2.9040 - val_accuracy: 0.4306\n",
      "Epoch 9/30\n",
      "21/21 [==============================] - 3s 135ms/step - loss: 0.0896 - accuracy: 0.9769 - val_loss: 3.1637 - val_accuracy: 0.4167\n",
      "Epoch 10/30\n",
      "21/21 [==============================] - 3s 140ms/step - loss: 0.0642 - accuracy: 0.9908 - val_loss: 3.5712 - val_accuracy: 0.3333\n",
      "Epoch 11/30\n",
      "21/21 [==============================] - 3s 143ms/step - loss: 0.0545 - accuracy: 0.9846 - val_loss: 3.4995 - val_accuracy: 0.4028\n",
      "Epoch 12/30\n",
      "21/21 [==============================] - 3s 145ms/step - loss: 0.0442 - accuracy: 0.9877 - val_loss: 3.8193 - val_accuracy: 0.3750\n",
      "Epoch 13/30\n",
      "21/21 [==============================] - ETA: 0s - loss: 0.0437 - accuracy: 0.9831Restoring model weights from the end of the best epoch: 3.\n",
      "21/21 [==============================] - 3s 139ms/step - loss: 0.0437 - accuracy: 0.9831 - val_loss: 3.6749 - val_accuracy: 0.3750\n",
      "Epoch 00013: early stopping\n",
      "Test Accuracy: 43.05555522441864\n",
      "\n",
      "        acc1       acc2       acc3       acc4       acc5      acc6  acc7  \\\n",
      "0  34.246576  41.095892  43.055555  41.666666  47.222221  51.38889  50.0   \n",
      "\n",
      "        acc8       acc9      acc10       AVG  \n",
      "0  54.166669  40.277779  43.055555  44.61758  \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\970068\\AppData\\Local\\Temp\\ipykernel_10804\\3147406012.py:82: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  record = record.append(temp, ignore_index=True)\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "# Parameter Initialization\n",
    "trunc_type='post'\n",
    "padding_type='post'\n",
    "oov_tok = \"<UNK>\"\n",
    "\n",
    "columns = ['acc1', 'acc2', 'acc3', 'acc4', 'acc5', 'acc6', 'acc7', 'acc8', 'acc9', 'acc10', 'AVG']\n",
    "record = pd.DataFrame(columns = columns)\n",
    "\n",
    "# prepare cross validation with 10 splits and shuffle = True\n",
    "kfold = KFold(n_splits=10, shuffle=False, random_state=None)\n",
    "\n",
    "# Separate the tweet and label\n",
    "tweets, labels = list(df.tweet_stemmed), list(df.label)\n",
    "\n",
    "exp=0\n",
    "\n",
    "# kfold.split() will return set indices for each split\n",
    "acc_list = []\n",
    "for train, test in kfold.split(tweets):\n",
    "    \n",
    "    exp+=1\n",
    "    print('Training {}: '.format(exp))\n",
    "    \n",
    "    train_x, test_x = [], []\n",
    "    train_y, test_y = [], []\n",
    "\n",
    "    for i in train:\n",
    "        train_x.append(tweets[i])\n",
    "        train_y.append(labels[i])\n",
    "\n",
    "    for i in test:\n",
    "        test_x.append(tweets[i])\n",
    "        test_y.append(labels[i])\n",
    "\n",
    "    # Turn the labels into a numpy array\n",
    "    train_y = np.array(train_y)\n",
    "    test_y = np.array(test_y)\n",
    "    \n",
    "    # one-hot\n",
    "    onehot_train_y = to_categorical(train_y)\n",
    "    onehot_test_y = to_categorical(test_y)\n",
    "\n",
    "    # encode data using Cleaning and Tokenization\n",
    "    tokenizer = Tokenizer(oov_token=oov_tok)\n",
    "    tokenizer.fit_on_texts(train_x)\n",
    "\n",
    "    # Turn the text into sequence\n",
    "    training_sequences = tokenizer.texts_to_sequences(train_x)\n",
    "    test_sequences = tokenizer.texts_to_sequences(test_x)\n",
    "\n",
    "    max_len = max_length(training_sequences)\n",
    "\n",
    "    # Pad the sequence to have the same size\n",
    "    Xtrain = pad_sequences(training_sequences, maxlen=max_len, padding=padding_type, truncating=trunc_type)\n",
    "    Xtest = pad_sequences(test_sequences, maxlen=max_len, padding=padding_type, truncating=trunc_type)\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "    vocab_size = len(word_index)+1\n",
    "\n",
    "    # Define the input shape\n",
    "    model = define_model(input_dim=vocab_size, max_length=max_len)\n",
    "\n",
    "    # Train the model\n",
    "    model.fit(Xtrain, onehot_train_y, batch_size=32, epochs=30, verbose=1, \n",
    "              callbacks=[callbacks], validation_data=(Xtest, onehot_test_y))\n",
    "\n",
    "    # evaluate the model\n",
    "    loss, acc = model.evaluate(Xtest, onehot_test_y, verbose=0)\n",
    "    print('Test Accuracy: {}'.format(acc*100))\n",
    "\n",
    "    acc_list.append(acc*100)\n",
    "\n",
    "mean_acc = np.array(acc_list).mean()\n",
    "entries = acc_list + [mean_acc]\n",
    "\n",
    "temp = pd.DataFrame([entries], columns=columns)\n",
    "record = record.append(temp, ignore_index=True)\n",
    "print()\n",
    "print(record)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "661e6b72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc1</th>\n",
       "      <th>acc2</th>\n",
       "      <th>acc3</th>\n",
       "      <th>acc4</th>\n",
       "      <th>acc5</th>\n",
       "      <th>acc6</th>\n",
       "      <th>acc7</th>\n",
       "      <th>acc8</th>\n",
       "      <th>acc9</th>\n",
       "      <th>acc10</th>\n",
       "      <th>AVG</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>34.246576</td>\n",
       "      <td>41.095892</td>\n",
       "      <td>43.055555</td>\n",
       "      <td>41.666666</td>\n",
       "      <td>47.222221</td>\n",
       "      <td>51.38889</td>\n",
       "      <td>50.0</td>\n",
       "      <td>54.166669</td>\n",
       "      <td>40.277779</td>\n",
       "      <td>43.055555</td>\n",
       "      <td>44.61758</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        acc1       acc2       acc3       acc4       acc5      acc6  acc7  \\\n",
       "0  34.246576  41.095892  43.055555  41.666666  47.222221  51.38889  50.0   \n",
       "\n",
       "        acc8       acc9      acc10       AVG  \n",
       "0  54.166669  40.277779  43.055555  44.61758  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "80143bb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the preprocessing pipeline\n",
    "def text_preprocessing(text):\n",
    "    # case folding\n",
    "    text = text.lower()\n",
    "    \n",
    "    # text cleansing\n",
    "    text = text_cleansing(text)\n",
    "    \n",
    "    # text normalization\n",
    "    text = normalize_text(text)\n",
    "    \n",
    "    # stemming\n",
    "    text = stemmer.stem(text)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "8f040f9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# predicting a text\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "def text_predict(text):\n",
    "    \n",
    "    # preprocess\n",
    "    text_cleaned = text_preprocessing(text)\n",
    "    text_sequence = tokenizer.texts_to_sequences([text_cleaned])\n",
    "\n",
    "    max_len= max_length(training_sequences)\n",
    "\n",
    "    seq_padded = pad_sequences(text_sequence, maxlen=max_len, padding=padding_type, truncating=trunc_type)\n",
    "\n",
    "    # prediction\n",
    "    pred_proba = model.predict(seq_padded)\n",
    "\n",
    "    result = np.argmax(pred_proba)\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f88f3ab1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction test\n",
    "text_predict('Indihome sampah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ba5b6e20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet_lower</th>\n",
       "      <th>tweet_clean</th>\n",
       "      <th>tweet_normalized</th>\n",
       "      <th>tweet_stemmed</th>\n",
       "      <th>tweet_predict</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>wifi watcha pasti indihome</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ada masalah apasih??!!</td>\n",
       "      <td>indihome ada masalah apasih</td>\n",
       "      <td>indihome ada masalah apa sih</td>\n",
       "      <td>indihome ada masalah apa sih</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>1</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocil nonton nu...</td>\n",
       "      <td>sore sore hujan rebahan bareng bocah cilik men...</td>\n",
       "      <td>sore sore hujan rebah bareng bocah cilik tonto...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>2</td>\n",
       "      <td>indihome plis untuk tanggal 10,13 jangan kesur...</td>\n",
       "      <td>indihome plis untuk tanggal jangan kesurupan</td>\n",
       "      <td>indihome tolong untuk tanggal jangan kesurupan</td>\n",
       "      <td>indihome tolong untuk tanggal jangan surup</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>INDIHOME NGAPASIIII</td>\n",
       "      <td>0</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>indihome ngapasiiii</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994</th>\n",
       "      <td>@IndiHomeCare Min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, kalo jatuh tempo pembayaran...</td>\n",
       "      <td>@indihomecare min kalo jatuh tempo pembayaran ...</td>\n",
       "      <td>@indihomecare min kalau jatuh tempo pembayaran...</td>\n",
       "      <td>indihomecare min kalau jatuh tempo bayar indih...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>@IndiHomeCare Min, 1 IndiHome TV bisa gak berl...</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare min, 1 indihome tv bisa gak berl...</td>\n",
       "      <td>@indihomecare min indihome tv bisa gak berlang...</td>\n",
       "      <td>@indihomecare min indihome televisi bisa tidak...</td>\n",
       "      <td>indihomecare min indihome televisi bisa tidak ...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>@IndiHomeCare @fauzindrianto Dari semalem down...</td>\n",
       "      <td>4</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalem down...</td>\n",
       "      <td>@indihomecare @fauzindrianto dari semalam down...</td>\n",
       "      <td>indihomecare fauzindrianto dari malam down tid...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>@JefriHandri Sini indihome down dr semalem jam...</td>\n",
       "      <td>2</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dr semalem jam...</td>\n",
       "      <td>@jefrihandri sini indihome down dari semalam j...</td>\n",
       "      <td>jefrihandri sini indihome down dari malam jam ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1002</th>\n",
       "      <td>@IndiHomeCare daftar indihome lewat mana?</td>\n",
       "      <td>6</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana?</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "      <td>@indihomecare daftar indihome lewat mana</td>\n",
       "      <td>indihomecare daftar indihome lewat mana</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>722 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Tweet  label  \\\n",
       "0                            wifi watcha pasti indihome      5   \n",
       "1                       indihome ada masalah apasih??!!      0   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...      1   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...      2   \n",
       "4                                   INDIHOME NGAPASIIII      0   \n",
       "...                                                 ...    ...   \n",
       "994   @IndiHomeCare Min, kalo jatuh tempo pembayaran...      6   \n",
       "995   @IndiHomeCare Min, 1 IndiHome TV bisa gak berl...      6   \n",
       "997   @IndiHomeCare @fauzindrianto Dari semalem down...      4   \n",
       "1000  @JefriHandri Sini indihome down dr semalem jam...      2   \n",
       "1002          @IndiHomeCare daftar indihome lewat mana?      6   \n",
       "\n",
       "                                            tweet_lower  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                       indihome ada masalah apasih??!!   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3     indihome plis untuk tanggal 10,13 jangan kesur...   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min, kalo jatuh tempo pembayaran...   \n",
       "995   @indihomecare min, 1 indihome tv bisa gak berl...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002          @indihomecare daftar indihome lewat mana?   \n",
       "\n",
       "                                            tweet_clean  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                           indihome ada masalah apasih   \n",
       "2     sore sore hujan rebahan bareng bocil nonton nu...   \n",
       "3          indihome plis untuk tanggal jangan kesurupan   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min kalo jatuh tempo pembayaran ...   \n",
       "995   @indihomecare min indihome tv bisa gak berlang...   \n",
       "997   @indihomecare @fauzindrianto dari semalem down...   \n",
       "1000  @jefrihandri sini indihome down dr semalem jam...   \n",
       "1002           @indihomecare daftar indihome lewat mana   \n",
       "\n",
       "                                       tweet_normalized  \\\n",
       "0                            wifi watcha pasti indihome   \n",
       "1                          indihome ada masalah apa sih   \n",
       "2     sore sore hujan rebahan bareng bocah cilik men...   \n",
       "3        indihome tolong untuk tanggal jangan kesurupan   \n",
       "4                                   indihome ngapasiiii   \n",
       "...                                                 ...   \n",
       "994   @indihomecare min kalau jatuh tempo pembayaran...   \n",
       "995   @indihomecare min indihome televisi bisa tidak...   \n",
       "997   @indihomecare @fauzindrianto dari semalam down...   \n",
       "1000  @jefrihandri sini indihome down dari semalam j...   \n",
       "1002           @indihomecare daftar indihome lewat mana   \n",
       "\n",
       "                                          tweet_stemmed  tweet_predict  \n",
       "0                            wifi watcha pasti indihome              5  \n",
       "1                          indihome ada masalah apa sih              2  \n",
       "2     sore sore hujan rebah bareng bocah cilik tonto...              1  \n",
       "3            indihome tolong untuk tanggal jangan surup              2  \n",
       "4                                   indihome ngapasiiii              2  \n",
       "...                                                 ...            ...  \n",
       "994   indihomecare min kalau jatuh tempo bayar indih...              6  \n",
       "995   indihomecare min indihome televisi bisa tidak ...              6  \n",
       "997   indihomecare fauzindrianto dari malam down tid...              6  \n",
       "1000  jefrihandri sini indihome down dari malam jam ...              4  \n",
       "1002            indihomecare daftar indihome lewat mana              6  \n",
       "\n",
       "[722 rows x 7 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predicting data set\n",
    "\n",
    "df['tweet_predict'] = df['tweet_stemmed'].apply(lambda x: text_predict(x))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f5da455c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEGCAYAAABFBX+4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA19ElEQVR4nO3dd3gU5drH8e+9SSihY+hBwAP2o6AUAelNmqByAA/4oqLYD6io2JCDXbGgB5EONoooIEUEsYFSlUhXQxFCqNIhQLK53z92wUDaJuxmdsj98Zoru7O78/yyLneefWbmGVFVjDHGuIfH6QDGGGNyxgq3Mca4jBVuY4xxGSvcxhjjMla4jTHGZSKdDpCZMiUucd3hLodOHHM6Qo60KnuV0xFyLCk12ekIOfbD7rVOR8iRskVKOh0hx7bvXyvnuo3kvZsCrjlRMRedc3vnImwLtzHG5KlUr9MJAmaF2xhjADTV6QQBs8JtjDEAqVa4jTHGVdR63MYY4zLeFKcTBMwKtzHGgO2cNMYY17GhEmOMcRnbOWmMMe5iOyeNMcZtrMdtjDEu43XPdAr5pnD/o3o1Ro976/T9KlUr8+pL7zBi+AQHU2UtNrYCY8a8TbmyMagqY8Z8wv+GjXU61hliKsTwyFuPUrJMSVSVrz6ZyxdjvwCgw+0daf9/7UlNTWXFN8sZ99I4h9P6lKlQhgFDH6NUTClUldmfzOHzMdO5vX8vGrapT2qqcmDvAV575HX+2rXP6bjpjBr5Bu3atWT3nr3UqtXC6TgB6X1PT/7dqwuC8MkHUxn9/odOR0rPRUMlEq6XLgvlJFMej4fVG36gTYuuJGxLDNp2gz3JVPnyZSlfvixxcWsoWrQISxbPocu/7mLDhj+Csv1gTDJVqmwpSpctzcY1GylcpDBvzx7KC3c/T8mYUnR7qBuDbn+OlJMplLigBAf/OnjO7QVjkqnSZUtzQdnS/LEmnsJFCvP+l8MY2HsQe3bs5dgR3//Dm+7sTJUaF/L2k++cc3vBnmTq+uvrcfTIUcaOGxqSwh3sSaYuuaw6740ZQvsW3Uk+mczHU0cw4JHBbNm8NWhtBGOSqRNrFwRccwpe0cLRSaby5bSujZvWZ8vmbUEt2qGwc+du4uLWAHDkyFE2bIinUqXyDqc60/7d+9m4ZiMASUeT2Ba/jQvKX0C729rx6XufknLSd1JDMIp2sOzbvY8/1sQDvsx//rGVmPIxp4s2QKHChQjXTs2iRUvZt/+A0zECVuPii1i5YhXHk47j9XpZ8uMK2nZs6XSs9DQ18MVhISvcInKpiDwhIu/4lydE5LJQtZcTN93cns+nznI6Ro5UqRLL1TWvYNmylU5HyVTZ2LJcdMVF/LbyNypVq8QVda/gjRlv8vKUV6hxVQ2n42WoXGw5ql9ZnfUrNwBw5+O3M3HZx7S4qTnjh3zgcLrzw4b18dSrfy2lSpWgUOFCNG/ViIph1gEBfDsnA10cFpLCLSJPAJMAAZb5FwEmisiALF7XR0RWiMiK4ycPhCIaUVFRtGnXnC+mzw3J9kOhSJFoJk0cQf/+gzh8+IjTcTJUKLoQT414mlH/HUXSkSQiIj0UK1GMRzs9wrgXx/LEe5n+b3dMoehCDBo5kPcGDT/d2x772nhurduDBdO+ofMdNzqc8PwQ//smhg0dwyefj+LjqSNYu2YDqV7ni9/ZNDU54MVpoepx9wbqqOorqvqRf3kFqOt/LEOqOlJVa6tq7UIFSoYkWItWjVn161r27PkrJNsPtsjISCZPGsmkSdOZMSM8/9hEREbw1Iin+G7atyye+xMAe3f8xU/+27//+juqSvHSxZ2MeYaIyAgGjRzIgmnfsOjLH9M9vmDaAhq1beRAsvPTpI8+p22zrtzSvhcHDxxi08YtTkdKL4g9bhEZKyK7RWRNmnWvi8gGEVklItNEpGSax54UkXgR+U1E2mS3/VAV7lSgYgbrK/gfc8zNXdozbepsJyPkyIgRr7Nhwx8MfWeU01Ey1ff1vmyL38b00dNPr1sybzFX1fft/KxYrSKRUZEc2nfIoYTp9R/yCFvjtzJ11Gen11Wq9vdHtkGbBmzbuM2JaOelC2JKA1AxtgJtO7Rk2qdh+G8wuGPc44Ebzlo3H7hSVa8CfgeeBBCRy4HuwBX+17wnIhFZbTxUhwP2AxaIyB/AqU//hUB14MEQtZmt6OjCNGnWgEf7DXQqQo40aFCHnj26sHr1epYt9fW2Bw58lblffetwsr9dXudymt/Sgs3rN/POl+8C8MFrE5g/eT59X+/HsPnDSD6ZwluPvOlw0r9dWecKWndpxab1mxjx1XAAxrw6lrbdb6DyRZVRTWVXwm7efnKow0kz9uGHw2jSuD4xMaXZvGkFgwcPYdz4SU7HytKoD96mVKmSpKSk8PRjL3Do0GGnI6UXxEmmVPUHEal61rp5ae4uAbr4b3cCJqnqCWCziMTjG51YnNn2Q3Y4oIh4/I1X8q/aDixX1YDeHbvmZOjZNSfzhl1zMvSCcTjg8WWfBlxzCtfreg/QJ82qkao6Mu1z/IV7lqpeefbrRWQmMFlVPxKR/wFLVPUj/2NjgC9VdWpm7YfsBBz1nfi/JFTbN8aYoMrB0SL+Ij0y2ydmQESeBlKAj3PzeshHZ04aY0yW8uBCCiJyO9ABaKF/D3dsByqneVqsf12m8uUJOMYYk06Ij+MWkRuAx4EbVTXtuOoXQHcRKSgi1YAa+A6hzpT1uI0xBghw91tARGQi0BSIEZEE4Dl8R5EUBOaLCPjGte9V1bUiMgVYh28I5YHs9gVa4TbGGAjqGZGqemsGq8dk8fwXgRcD3b4VbmOMgbCYgyRQVriNMQbCYg6SQFnhNsYYyJOjSoLFCrcxxoANlRhjjOvYUMm5258UntOXZiUpcaHTEXLk0ku7ZP+kMJNwZI/TEXIsukAhpyPkyN6k8JkMLE9Z4TbGGJexoRJjjHEZ2zlpjDEuY0MlxhjjMjZUYowxLmM9bmOMcRkr3MYY4zIhuhpYKFjhNsYYgBQ7qsQYY9zFdk4aY4zL2Bi3Mca4jI1xh6c2rZvy5puDifB4GDtuIq+9Piwk7Tzz0pv88OMySpcqyfSP3k/3+DcLF/PuqA/wiIeIiAgG9O3DNVdfeU5tHjx0mEeffZnEnbuoWL4cbzz/JCWKF2PWV98w5uNPQSE6ujDP9n+QS2tcdE5tne2Voc/RvHUj/tq7j7aNugIwYFA/mrdpRPLJFLZu2cbjDw3i8KHwm38mNrYCY8a8TbmyMagqY8Z8wv+GjXU6VrZWrf2eI0eO4vV68aZ4adq4s9ORMuWa99hFPW7RMP0rE1mgUlCDeTwe1q9dyA3tbiUhYQdLFs+h5233s379H0Fr49QkUyviVhNduDBPPT8kw8J97FgShQsXQkT4LX4z/Z99iZkTRwXUxrJfVjFjznxefObRM9a/MWwMJYoX467bujL6wykcOnyYR+7vzcrV67ioSmVKFC/GwsXLeW/sx0wc9TYQvEmm6tS/hmNHjzFk2ODThfv6ptexeOFyvF4vjw/8DwCvDX7nnNsK9iRT5cuXpXz5ssTFraFo0SIsWTyHLv+6iw0bgve5KBRZIGjbOmXV2u9p2rgz+/7aH/RtH085GdTt5cV7fOL4NjnXbSSN6R9wzSnce8g5t3cu8s1V3uvWqcXGjVvYvHkrycnJTJkygxs7tglJW7Vr/pMSxYtl+nh0dGH8Fwsl6fhxkL8/A2M/nkq33v/hpv+7j/+N/jDgNr9duJhObVsC0KltS775YTEAtf55+eksV11xKbt2783x75Od5Yt/4cD+g2esW/TdErxe3/VO41aspnzFskFvNxh27txNXNwaAI4cOcqGDfFUqlTe4VTnF7e8x+r1Brw4Ld8MlVSsVJ5tCYmn7yds30HdOrUcy/P19z8y9P3x/LX/AO8NGQzAj0t/ZmvCdiaNHoqq8uAT/2VF3Gpq1/xnttv7a/8BysSUBiDmglL8tf9Auud8Pusrrr+udlB/j0B06dGJ2dPn5Xm7OVWlSixX17yCZctWOh0le6pMnzEeVRg3diLjx01yOlFAwvo9dtFQSZ4XbhG5Q1XHZfJYH6APgESUwOMpkqfZ8lLLJg1p2aQhK+JW879RHzB66Mv8tPwXflr2C11ufxCAY0lJ/Lktkdo1/8mtd/fj5MlkjiUlcfDQYW7p9QAAj9x/Jw3rXXvGtkXkdI/+lGU//8rns+bx4fAhefML+t3/cG+8KSnM+HROnrabU0WKRDNp4gj69x/E4cPhNxZ/tjaturFjxy5iylzA9C8m8PvvG/npx+VOx8pS2L/Hdjhglv4LZFi4VXUkMBKCP8aduH0nlWMrnr4fW6kCiYk7g9lErtSu+U8SEney/8BBULjrtm507dwu3fNOjUtnNsZ9QamS7Nm7jzIxpdmzdx+lS5Y4/dhv8ZsZ+MrbvP/G85QsUTykv09at3TvSLPWjbjt5nvzrM3ciIyMZPKkkUyaNJ0ZM+Y6HScgO3bsAmDvnr+YNXMe1157dVgXble8x6nhub8vIyEZ4xaRVZksq4FyoWgzO8tXxFG9ejWqVq1MVFQUXbt2YuYsZ76+b01I5NRO4XW/xXPyZDIlSxSnQd1rmDZ7HseOJQGwa8/eDIc8MtL0+uuY8eXXAMz48muaNaoPwI6du+n31PO8PPAxql4YG/xfJhONmzfg7od6cU/PfhxPOp5n7ebGiBGvs2HDHwx9J7AdxE6Lji5M0aJFTt9u3rwR69b97nCqrLniPU5NDXxxWKh63OWANsDZu7wF+ClEbWbJ6/XSt98zzJn9CREeD+MnTA7Zh/2x515h+cpVHDhwiBade3J/79tI8Z9O2+2m9sz/bhFffLmAyMhIChUswJDBAxARGta7lk1/bqPHPY8AEF24EC8PfIwLSpXMts27buvKo8++xOezvqJi+bK88fxTAAwf9wkHDx3mhSG+Qx8jIiKYMvbcj+5I6+2RL1Gv4bWUKl2SRau+ZOir73Nf3zspUDCKCVOHAxD382qe7f9SUNsNhgYN6tCzRxdWr17PsqW+nuDAga8y96tvHU6WubJlY/hoou99jYyMYOqUmSz4+geHU2XONe9xEHc6ishYoAOwW1Wv9K8rDUwGqgJbgK6qul9845pDgXbAMeB2Vf0ly+2H4nBAERkDjFPVRRk89omq/ju7bQR7qCQv2DUnQ8+N15wMxeGAoRTswwHzQjAOBzz25t0B15zoR0Zl2Z6INAaOAB+kKdyvAftU9RURGQCUUtUnRKQd8BC+wl0PGKqq9bLafkiGSlS1d0ZF2/9YtkXbGGPyXKoGvmRDVX8A9p21uhMwwX97AtA5zfoP1GcJUFJEKmS1/XxzHLcxxmRJUwNeRKSPiKxIs/QJoIVyqrrDf3snf+/vqwRsS/O8BP+6TOWb47iNMSZLOTiqJO0RcLmhqioiuR4OtsJtjDGAhv5okV0iUkFVd/iHQnb7128HKqd5Xqx/XaZsqMQYY8B3VEmgS+58AfTy3+4FzEiz/v/E5zrgYJohlQxZj9sYYyCoJ+CIyESgKRAjIgnAc8ArwBQR6Q38CXT1P30OviNK4vEdDnhHdtu3wm2MMRDUE2tU9dZMHmqRwXMVeCAn27fCbYwx4KpT3q1wG2MM2CRTxhjjOtbjzp+iKzZyOkKOHBp3p9MRcqz6g587HeG8d+xkeE8KFiqa4vwFEgJlhdsYY8B63MYY4zo2xm2MMS5jPW5jjHEXtcJtjDEuYzsnjTHGZazHbYwxLmOF2xhj3CUUl3EMFSvcxhgD1uM2xhjXscJtjDHuoinuOQEnX10Bp03rpqxd8wMb1i3i8cdyNP2tI0aNfIPtCb+ycuWCkLbz3BfLaDZkBrcMn5vh45v3HuL/xiygzotTmfDThqC0eTLFy+NTF9Px3Tn0HP012w8cBWDxxp3cOmo+Xd7/iltHzWfZ5l1BaS8rve/pyYKfpvPNTzO4697bQt5eMLgtc159ls9Jag4Wh+Wbwu3xeHhn6It06NiTf17djG7dOnPZZTWcjpWlCR9MoUOHHiFv58arq/Fej8aZPl6icAEev6EW/1f/khxve/uBo/Se8G269dNWbqZ44ShmPtSOntddzNCvVwFQKrogQ7tfz9R72/B8p7o8PX1ZjtvMiUsuq86/e3WhfYvutGp0My3bNKFqtQtD2ua5cmPmvPosnwtN1YAXp+Wbwl23Ti02btzC5s1bSU5OZsqUGdzYsY3TsbK0aNFS9u0/EPJ2rq1ShuKFC2T6eOkihbiyUmkiPZLusdmr/qTH6K/pOmIez89agTfAq4h899t2Ol5VFYCWl8eybPMuVJVLK5SibLHCAPyjTHFOJHs5GcITI2pcfBErV6zieNJxvF4vS35cQduOLUPWXjC4MXNefZbPSaoGvjgsZIVbRC4VkRYiUvSs9TeEqs2sVKxUnm0JiafvJ2zfQcWK5Z2Ict7YtOcQX63dyvg7mjPlntZ4PMKc1VsDeu3uw0mULxENQKTHQ9FCURxIOnnGc75en8BlFUpSIDIi6NlP2bA+nnr1r6VUqRIUKlyI5q0aUbFSeH8u3JjZFVw0VBKSnZMi8h9811BbD4wRkb6qeuqKxi8BGQ6mikgfoA+ARJTA4ykSingmSJZt3sX6HfvpMfprAE6keCkdXRCAhyf/yPYDR0nxprLj4DG6jpgHwL/r1aBzzWrZbjt+90GGLljF8B5NQvcLAPG/b2LY0DF88vkojh1LYu2aDaR6w+BfZhbcmNkNwmEIJFChOqrkbuBaVT0iIlWBqSJSVVWHAum/b/up6khgJEBkgUpBfRcTt++kcmzF0/djK1UgMXFnMJvIdxToeHVV/tPiqnSPvdWtIeAb4x44YxljejU74/GyxQqz8+AxyhWPJiU1lSPHkynpH67ZdegYj0z5kec71aNy6aLpth1skz76nEkf+S7QMODZvuxIDP0O0XPlxszhTlPcU7hDNVTiUdUjAKq6Bd9l6tuKyJtkUbhDafmKOKpXr0bVqpWJioqia9dOzJw1z4ko54261coyf30C+476rphyMOkEif6jQ7LT5JKKzFy1BYCv1yVQp1pZRIRDx0/y0MSF9G1xFbUujAlV9DNcEFMagIqxFWjboSXTPp2dJ+2eCzdmDnv5fagE2CUiNVU1DsDf8+4AjAX+GaI2s+T1eunb7xnmzP6ECI+H8RMms27d705ECdiHHw6jSeP6xMSUZvOmFQwePIRx4ycFvZ0Bny1mxZ97OHDsBK3fmsl9Ta8gxf/V+1+1q7P3SBL/HvU1R08kIyJ8vPQPPr//Bv5RpgQPNruSez/6AVUlMsLDk22voWLJ7Ie4bqp1EU9PW0rHd+dQvHABXr3lOgAmL4tn674jjPhhHSN+WAfA+z0bU7pIoaD/3qeM+uBtSpUqSUpKCk8/9gKHDh0OWVvB4rbMefVZPhcuuo4CEorz80UkFkhR1XRjESLSUFV/zG4bwR4qyQuOfJU4B3bNSZOR3UcPOB0hx5JPbj/nf35/tW8ScM25YPb3jv5zD8lQiaomZFS0/Y9lW7SNMSavaWrgS3ZE5GERWSsia0RkoogUEpFqIrJUROJFZLKIZH4MbjbyzXHcxhiTFU0JfMmKiFQC/gPUVtUrgQigO/Aq8JaqVgf2A71zm9UKtzHGENweN779h4VFJBKIBnYAzYGp/scnAJ1zm9UKtzHGkLPCLSJ9RGRFmqXP6e2obgeGAFvxFeyDwM/AAdXT/fUEoFJus9rsgMYYA6CB729Me87J2USkFNAJqAYcAD4FgnrGuBVuY4whqIcDtgQ2q+oeABH5HGgIlBSRSH+vOxbYntsGbKjEGGMATZWAl2xsBa4TkWgREaAFsA74Fujif04vYEYmr8+W9biNMQZI9Qbn0GxVXSoiU4FfgBRgJb5hldnAJBF5wb9uTG7bsMJtjDEE98xJVX0OeO6s1ZuAusHYvhVuY4yBQIZAwoYVbmOMAUIw+0fIWOEOogcrNnI6Qo5ceP8UpyPkWOGIXJ8l7Jh9x484HSFHoguEbkKvcGY9bmOMcZlg7ZzMC1a4jTGG86THLSLv4rvISYZU9T8hSWSMMQ7QHJw56bSsetwr8iyFMcY4zE0XUsi0cKvqhLwMYowxTko9T3rcAIhIGeAJ4HLg9O5mVW0ewlzGGJOn3DRUEshcJR8D6/HNdPVfYAuwPISZjDEmz6V6JeDFaYEU7gtUdQyQrKrfq+qd+CYEN8aY80YQJ5kKuUAOB0z2/9whIu2BRKB06CIZY0zeO6/GuIEXRKQE8CjwLlAceDikqYwxJo+5aYw728KtqrP8Nw8CzUIbJ7TatG7Km28OJsLjYey4ibz2+jCnI6UTWTCKByc/R2TBKCIiPPz65VLmvjWVGg2u5ManeiAe4cTR40zsP5y9f+5yOu4Z/lG9GqPHvXX6fpWqlXn1pXcYMTy8DlB6/d3BtGjdmL/27qNVw5sBKFGyOO+NHUJs5YokbEvk/jv6c/DgIYeTZs7j8bDwxy9ITNzJv265y+k42Vq19nuOHDmK1+vFm+KlaePOTkdK57yaq0RExpHBiTj+sW7X8Hg8vDP0RW5odysJCTtYsngOM2fNY/36P5yOdoaUE8m89+/nOXnsBJ7ICP4z9b+s/y6OLi/0Zszdr7N7YyINe7ai1UM3M7H/cKfjnmFj/GaaNeoM+N7v1Rt+YPas+c6GysCnn8xgwqiJvDX8xdPrHujXmx+/X8p7Q8dwf9/e3N+vNy//960stuKs+x+4g982xFOseFGnowSsQ7se7Ptrv9MxMuWmoZJAdk7OwjcB+GxgAb6hEnfNmgPUrVOLjRu3sHnzVpKTk5kyZQY3dmzjdKwMnTx2AoCIyAgiIiN8PQFVChWLBqBQ8WgO7grffwAAjZvWZ8vmbSRsS3Q6SjrLFv/Mgf0Hz1jXqm0zpk7yXZBk6qQZtG4Xvl8uK1Yqzw03NGPC+MlORzmvpKZKwIvTAhkq+SztfRGZCCzK7nUiUtf3cl0uIpfju1jmBlWdk9uw56JipfJsS/i7iCRs30HdOrWciJIt8QiPznqZmCrlWfThPLbGxTN5wEj6jHuC5OMnOX4kibdvetbpmFm66eb2fD51VvZPDBMxZS9g9669AOzetZeYshc4nChzr702kGeeeYViRYs4HSVwqkyfMR5VGDd2IuPHTXI6UTrnW4/7bDWAslk9QUSeA94BhovIy8D/gCLAABF5OovXnb7kfWrq0VxEOz9oqjKk3QAG1b+fC6/+B+UvjqVJ73aMvONV/lv/AZZ9+h2dn7nN6ZiZioqKok275nwxfa7TUXIvTMc7b2jbnD179hK3co3TUXKkTatuNL6+E7fcfCd39elJg4Z1nI6UjqoEvDgt28ItIodF5NCpBZiJ70zKrHTBd1XjxsADQGdVfR5oA3TL7EWqOlJVa6tqbY8nuL2JxO07qRxb8fT92EoVSEzcGdQ2gu34oWPEL17LZU1rUvGyKmyNiwdg5azFVL32YofTZa5Fq8as+nUte/b85XSUgO3d/Rdly8UAULZcDHvDNPt1111Lu/YtWbt+IeM/eJcmTRowekz4jsWfsmOHb0f63j1/MWvmPK699mqHE6WXqhLw4rRsC7eqFlPV4mmWi88ePslAiqp6VfUYsFFVD/m3lQQ4MpXL8hVxVK9ejapVKxMVFUXXrp2YOWueE1GyVKR0MQoV941lRxWM4pLrr2JXfCKFihWmTLUKAP51252MmaWbu7Rn2tTZTsfIkflzv6NL904AdOneiflffutwoowNeu51LqnRgCsua8Tt//cQ33//E3f1Du+jc6OjC1PUP6wTHV2Y5s0bsW7d7w6nSk9zsDgtkKNKFqhqi+zWneWkiET7C/e1aV5XAocKt9frpW+/Z5gz+xMiPB7GT5gclh+e4mVL8e837sPj8SAeD3GzF7Pum1+Y8uQobh/+MKpK0sGjTHrsfaejZig6ujBNmjXg0X4DnY6SqXdHvUr9hnUodUFJlq75mjdfGcZ7b49h+NghdOt5E9u37eC+Ox91OuZ5o2zZGD6a6DsCKjIygqlTZrLg6x8cTpWeNzU3I8fOEM3k4EURKQREA98CTYFT3w+KA3NV9dJMNypSUFVPZLA+BqigqquzCxZZoFI4/GHLkYdcdumyj/avdDpCjtmly0IvwuOeAnbKwSMbz3n8YmH5LgHXnEY7pzo6XpJVj/seoB9QEfiZvwv3IXw7GzOVUdH2r98L7M1xSmOMCTHF+bHrQGU1H/dQYKiIPKSq7+ZhJmOMyXOpLvqOH8h3olQRKXnqjoiUEpH7QxfJGGPyXioS8JIdESkpIlNFZIOIrBeR+iJSWkTmi8gf/p+lcps1kMJ9t6oeOHVHVfcDd+e2QWOMCUeKBLwEYCh/7wu8Gt81DQYAC1S1Br6z0AfkNmsghTtCRE4nFZEIwH17iIwxJgteJOAlK/6j5xoDYwBU9aS/89sJODXj2gSgc26zBlK45wKTRaSFiLQAJgJf5rZBY4wJR6k5WNKe5e1f+qTZVDVgDzBORFaKyGgRKQKUU9Ud/ufsBMrlNmsg83E/AfQB7vXfXwWUz22DxhgTjnJygomqjgRGZvJwJHAN8JCqLhWRoZw1LKKqKiK53h0ayJmTqcBSfNearIvvsmXrc9ugMcaEoyCOcScACaq61H9/Kr5CvktEKgD4f+7ObdZMe9wicjFwq3/ZC0wGUNXwne/SGGNyKViztarqThHZJiKXqOpvQAtgnX/pBbzi/zkjt21kNVSyAVgIdFDVeAARCe9JEYwxJpcCOcwvBx4CPhaRAsAm4A58IxxTRKQ38CfQNbcbz6pw3wx0B74VkbnAJAjub2aMMeHCG8RtqWocUDuDh7Ka4ylgWZ05OR2Y7t8b2gnf6e9lRWQ4ME1VQzq1nhv/QvwvcaHTEXLkwuK53qntmLju7tsvHjM626l5wopX3TdXSTCkinuqTiA7J4+q6ieq2hGIBVaS/XzcxhjjKm6a1jVHf1pVdb//YgdB6e4bY0y4yMlx3E4L5DhuY4w574XBNYADZoXbGGMg21PZw4kVbmOMwXrcxhjjOuEwdh0oK9zGGEN4HC0SKCvcxhiDDZUYY4zr2FCJMca4jNdFPe58c27rqJFvsD3hV1auXOB0lIC5JfMrQ59j2fqv+XLhlNPrBgzqx7zFnzH7+8kMnzCEYsWLOpgwvaimnYh+6j2inx5OVNNOAETWup7op4dT9J1ZeC6s4XDCzMXGVuCrryYTt3IBK3/5mgcfuNPpSFkqWLAgCxfOYOnSL/n55/k880x4zlXnphNw8k3hnvDBFDp06OF0jBxxS+bPJs3kjm4PnrFu0XdLaHt9V9o36cbmjVu5r1/4FBdPhSpENWjDsdcf5tjLDxB5ZV0kpgKpiX+SNOoFvBvXOB0xSykpXp544nlq1mpBo8aduPfeXlx6afj+oTlx4gQ33HAr9eq1pV69trRu3YS6dWs5HSsdK9xhaNGipezbf8DpGDnilszLF//Cgf0Hz1i36LsleL2++dbiVqymfMWyTkTLkKd8ZbxbfoPkE5Caijd+DVE1G5K6axu6e7vT8bK1c+du4uJ8f1yOHDnKhg3xVKoU3pNvHT16DICoqEgiI6NQDb9jOM7buUrOhYh8kFdtmfDSpUcnvl/wk9MxTktN/JPI6ldCkWIQVZDIK2ojpWKcjpUrVarEcnXNK1i2bKXTUbLk8XhYsmQOW7f+wjffLGT58jinI6WTKoEvTgvJzkkR+eLsVUAzESkJoKo3ZvK6Pviub4knogQeT5FQxDN56P6He+NNSWHGp3OcjnJa6q5tnJz/KdEPvICePIE3YROkhsMX4JwpUiSaSRNH0L//IA4fPuJ0nCylpqZy3XXtKFGiOJMnj+Tyyy9m3brfnY51Bjd9AkJ1VEksvsv0jMb3zULwTSr+RlYvSnsBzqgClcLhG4k5B7d070iz1o247eZ7s39yHktePI/kxb4p5Qt07IUe2OtwopyJjIxk8qSRTJo0nRkz5jodJ2AHDx7i++9/onXrpmFXuIN5IYVQC9VQSW3gZ+Bp4KCqfgckqer3qvp9iNo0YaRx8wbc/VAv7unZj+NJx52Ok44ULeH7WaoMkVc3IHnFd84GyqERI15nw4Y/GPrOKKejZCsmpjQlShQHoFChgrRo0Yjffot3OFV6+X6oxH9l+LdE5FP/z12haitQH344jCaN6xMTU5rNm1YwePAQxo2f5GSkbLkl89sjX6Jew2spVboki1Z9ydBX3+e+vndSoGAUE6YOByDu59U82/8lh5P+rdBdTyNFioM3hRNT3oOko0ReVZ+C/7oPKVqCwvcOInX7JpKGPet01HQaNKhDzx5dWL16PcuW+nrbAwe+ytyvvnU4WcbKly/LqFFvEhHhwePx8Nlns/jyy2+cjpWOm4ZKJC/27opIe6Chqj4V6GtsqCT07NJlecNtly7ziPsONktK+vOc+8EvV+kZcM158s+PHO1350kvWFVnA7Pzoi1jjMmN1LA40C8wdsq7Mcbgrp2TVriNMQZ3jXFb4TbGGMLjaJFAuW8vhDHGhEAqGvASCBGJEJGVIjLLf7+aiCwVkXgRmSwiBXKb1Qq3McYQkrlK+gLr09x/FXhLVasD+4Heuc1qhdsYYwju7IAiEgu0x3f2OCIiQHNgqv8pE4DOuc1qhdsYYwAvGvAiIn1EZEWapc9Zm3sbeJy/6/wFwAFVTfHfTwAq5Tar7Zw0xhhydlRJ2nmVziYiHYDdqvqziDQNQrR0rHAbYwxBPQGnIXCjiLQDCgHFgaFASRGJ9Pe6Y4FcT/5uQyXGGEPwdk6q6pOqGquqVYHuwDeq2gP4Fujif1ovYEZus4Ztj9s9J5/+rYrL5v4oHlnY6Qg5Vnrkr05HyLF9fa52OkKOuG1ulWDJgxNwngAmicgLwEpgTG43FLaF2xhj8pI3BN1F/5TW3/lvbwLqBmO7VriNMQabZMoYY1zHPWXbCrcxxgDW4zbGGNex2QGNMcZl1HrcxhjjLqE4qiRUrHAbYww2VGKMMa6TmgcXTg8WK9zGGIMdDhi22rRuyptvDibC42HsuIm89vowpyOl88rQ52jeuhF/7d1H20ZdARgwqB/N2zQi+WQKW7ds4/GHBnH40BGHk/7tubeepHGrhuzbu59/Nb0NgFdGDKbqPy4EoFiJohw+eITuLW93MGXGRo18g3btWrJ7z15q1WrhdJxMRTXtRFSDNiBC8o9zSf5uBpG1rqdAux54ylXm2JCHSd36h9MxMxQbW4ExY96mXNkYVJUxYz7hf8PGOh0rHTcdDphvJpnyeDy8M/RFOnTsyT+vbka3bp257LIaTsdK57NJM7mj24NnrFv03RLaXt+V9k26sXnjVu7rd6dD6TI2c/IcHrj1kTPWDbhnIN1b3k73lrezYPZ3fDPne4fSZW3CB1Po0KGH0zGy5KlQhagGbTj2+sMce/kBIq+si8RUIDXxT5JGvYB34xqnI2YpJcXLE088T81aLWjUuBP33tuLSy8Nv397moP/nJZvCnfdOrXYuHELmzdvJTk5mSlTZnBjxzZOx0pn+eJfOLD/4BnrFn23BK/XC0DcitWUr1jWiWiZ+mXJrxw8cCjTx1t1bM7cafPzMFHgFi1ayr79B5yOkSVP+cp4t/wGyScgNRVv/BqiajYkddc2dHeuZwbNMzt37iYuzvfH5ciRo2zYEE+lSuUdTpVeChrw4rQ8Kdwicr2IPCIirfOivYxUrFSebQmJp+8nbN9BxYrh9+HJTpcenfh+wU9OxwjYNdddzb69+9m6OcHpKK6VmvgnkdWvhCLFIKogkVfURkrFOB0rV6pUieXqmlewbNlKp6Ok46Yed0jGuEVkmarW9d++G3gAmAY8JyLXqOormbyuD9AHQCJK4PEUCUU817r/4d54U1KY8ekcp6ME7IabWoVtb9stUndt4+T8T4l+4AX05Am8CZsg1U0Hr/kUKRLNpIkj6N9/EIcPh88+mlPc9I6GaudkVJrbfYBWqrpHRIYAS4AMC3faywFFFqgU1D9ridt3Ujm24un7sZUqkJi4M5hNhNQt3TvSrHUjbrv5XqejBCwiIoLm7Zrw79bhNSbvRsmL55G8eB4ABTr2Qg/sdThRzkRGRjJ50kgmTZrOjBlznY6TIXXR4YChGirxiEgpEbkAEFXdA6CqR4GUrF8aGstXxFG9ejWqVq1MVFQUXbt2YuaseU5EybHGzRtw90O9uKdnP44nHXc6TsDqNa7Nlvg/2b1jj9NRXE+KlvD9LFWGyKsbkLziO2cD5dCIEa+zYcMfDH1nlNNRMpWKBrw4LVQ97hLAz4AAKiIVVHWHiBT1r8tzXq+Xvv2eYc7sT4jweBg/YTLr1v3uRJQsvT3yJeo1vJZSpUuyaNWXDH31fe7reycFCkYxYepwAOJ+Xs2z/V9yOOnfXh4+iGsb1KJk6ZLM/WUa778+hukTZ9Gmc0vmTvva6XhZ+vDDYTRpXJ+YmNJs3rSCwYOHMG78JKdjpVPorqeRIsXBm8KJKe9B0lEir6pPwX/dhxQtQeF7B5G6fRNJw551Omo6DRrUoWePLqxevZ5lS3297YEDX2XuV986nOxMbjrlXfLy64GIRAPlVHVzds8N9lBJXrBLl4Xemn1bnI6QY3bpstA7cXzbOXcI213YLuCaM2frHEc6oKfk6Qk4qnoMyLZoG2NMXnPTGHe+OnPSGGMyY0eVGGOMy4TD8dmBssJtjDG4a64SK9zGGAN41T2DJVa4jTEGdw2V5JtJpowxJiupqgEvWRGRyiLyrYisE5G1ItLXv760iMwXkT/8P0vlNqsVbmOMwXchhUCXbKQAj6rq5cB1wAMicjkwAFigqjWABf77uWKF2xhjCN4p76q6Q1V/8d8+DKwHKgGdgAn+p00AOuc2qxVuY4whZ4VbRPqIyIo0S5+MtikiVYFawFJ8Z43v8D+0E8j1qda2c9IYY8jZUSVpZzLNjH9ups+Afqp6SOTvs+RVVUUk13tDrXAH0dZDu5yOcN5rXPYKpyPkWKmRvzodIUeGl23mdARHBPOoEhGJwle0P1bVz/2rd6WZcK8CsDu327ehEmOMwTdXSaBLVsTXtR4DrFfVN9M89AXQy3+7FzAjt1mtx22MMQT1zMmGwG3AahGJ8697Ct8FZKaISG/gT6Brbhuwwm2MMQRvdkBVXUTm1x1oEYw2rHAbYwzgddH8gFa4jTEGsj0jMpxY4TbGGNw1V4kVbmOMwXrcxhjjOtbjNsYYl7EetzHGuIybLqSQr86cbNO6KWvX/MCGdYt4/LEHnI6TrVEj32B7wq+sXLnA6SgBc0PmMhXK8MaU1xj7zSjGLBjJzb07A3B7/16Mmv8+I74azqsfv8wF5Uo7GzQLbvgsd1/8Frd8/TI3f/UinWcPBqBa+7p0WfAKd239gJirqjmc8Eyag/+clm8Kt8fj4Z2hL9KhY0/+eXUzunXrzGWX1XA6VpYmfDCFDh16OB0jR9yQ2ev18v7gkdzZ/G4evLEvnXrdSJUaFzLl/U+5u9W93NPmPpYsWMpt/Xo6HTVDbvosz/rXi3ze5mmmtx8IwP7fEph/91B2LP3N4WTpqaYGvDgt3xTuunVqsXHjFjZv3kpycjJTpszgxo5tnI6VpUWLlrJv/wGnY+SIGzLv272PP9bEA5B0NIk//9hKTPkYjh05dvo5hQoXCtqZdMHmxs/yKQfiEzm4aUf2T3RAsObjzgshGeMWkXr4Jlg5JCKF8V3p4RpgHfCSqh4MRbtZqVipPNsSEk/fT9i+g7p1auV1DBNmysWWo/qV1Vm/cgMAdz5+O626tOLooaM82vUxh9NlzDWfZVXafTIAVWXDx9+w4eNvnU6UpXD9Q52RUPW4xwKnui9DgRLAq/514zJ7UdrJyVNTj4YomjE+haILMWjkQN4bNPx0b3vsa+O5tW4PFkz7hs533OhwQnf74ubnmdb2Gebe9jqX92pJ+XqXOB0pS27qcYeqcHtUNcV/u7aq9lPVRar6X+CizF6kqiNVtbaq1vZ4igQ1UOL2nVSOrXj6fmylCiQm7gxqG8Y9IiIjGDRyIAumfcOiL39M9/iCaQto1LaRA8my55bP8rGd+wE4/tchtsz9mTI1/+Fwoqx5U1MDXpwWqsK9RkTu8N/+VURqA4jIxUByiNrM0vIVcVSvXo2qVSsTFRVF166dmDlrnhNRTBjoP+QRtsZvZeqoz06vq1Tt72LYoE0Dtm3c5kS0bLnhsxxZuCBRRQqdvh3b+Er2/5bgcKqsuemoklAdx30XMFREngH2AotFZBuwzf9YnvN6vfTt9wxzZn9ChMfD+AmTWbfudyeiBOzDD4fRpHF9YmJKs3nTCgYPHsK48ZOcjpUlN2S+ss4VtO7Sik3rNzHiq+EAjHl1LG2730DliyqjmsquhN28/eRQh5NmzA2f5cJlitNqdD8APBERxE//iYTvVlH1htrUf/7/KFy6GG0m9Gff2j/5sudrzob1c9MYt4QyrIgUB6rh+wORoKoBX9srskAl97yLfplNwGuCx42XLvt+91qnI+SIGy9ddnfCR+f8z69MiUsCrjl7Dv7m6D/3kJ45qaqHAHddcM8Yky+5qcdtp7wbYwyExU7HQFnhNsYYgnrNyZCzwm2MMdhQiTHGuI5N62qMMS4TDsdnB8oKtzHGYD1uY4xxndQwmK41UPlmWldjjMmKqga8ZEdEbhCR30QkXkQGBDur9biNMYbgHVUiIhHAMKAVkAAsF5EvVHVdUBrAetzGGAOA5mDJRl0gXlU3qepJYBLQKZhZw7bHnXJye8jmAhCRPqo6MlTbDza35QX3ZXZbXrDMwZaTmiMifYA+aVaNTPN7VcI3od4pCUC9c0/4t/za4+6T/VPCitvygvsyuy0vWGbHpL12gH/J0z9G+bVwG2NMqGwHKqe5H+tfFzRWuI0xJriWAzVEpJqIFAC6A18Es4GwHeMOsbAcY8uC2/KC+zK7LS9Y5rCkqiki8iDwFRABjFXVoE7KHtILKRhjjAk+GyoxxhiXscJtjDEuk68Kd6hPQw02ERkrIrtFZI3TWQIhIpVF5FsRWScia0Wkr9OZAiUiESKyUkRmOZ0lECKyRURWi0iciKxwOk92RKSkiEwVkQ0isl5E6judyc3yzRi3/zTU30lzGipwazBPQw02EWkMHAE+UNUrnc6THRGpAFRQ1V9EpBjwM9A5nN/jU0TkEaA2UFxVOzidJzsisgWorap7nc4SCBGZACxU1dH+Iy2iVfWAw7FcKz/1uEN+GmqwqeoPwD6ncwRKVXeo6i/+24eB9fjOIgtrIhILtAdGO53lfCQiJYDGwBgAVT1pRfvc5KfCndFpqGFfVNxKRKoCtYClDkcJxNvA44B75vX0TZkxT0R+9p9+Hc6qAXuAcf7hqNEiUsTpUG6Wnwq3ySMiUhT4DOinqoeczpMVEekA7FbVn53OkkPXq+o1QFvgAf+wWriKBK4BhqtqLeAoEPb7mMJZfircIT8N1YCIROEr2h+r6udO5wlAQ+BG/5jxJKC5iHzkbKTsqep2/8/dwDR8Q4HhKgFIUNVT376m4ivkJpfyU+EO+Wmo+Z2ICL5xzPWq+qbTeQKhqk+qaqyqVsX3mfhGVXs6HCtLIlLEv/MX/5BDayBsjzxS1Z3ANhG5xL+qBRD2O6zDWb4p3KqaApw6DXU9MCXYp6EGm4hMBBYDl4hIgoj0djpTNhoCt+Hrtcb5l3ZOhzoPlQMWicivwDJgtqrOdThTdh4CPhaRVUBN4CVn47hbvjkc0Bhjzhf5psdtjDHnCyvcxhjjMla4jTHGZaxwG2OMy1jhNsYYl7HCbUJCRLz+wwHXiMinIhJ9DtsaLyJd/LdHi8jlWTy3qYg0yEUbW0QkJrcZjclLVrhNqCSpak3/rIYngXvTPigiubpsnqrelc1sg02BHBduY9zECrfJCwuB6v7e8EIR+QJY558D+3URWS4iq0TkHvCdgSki//PPnf41UPbUhkTkOxGp7b99g4j8IiK/isgC/8RW9wIP+3v7jUSkjIh85m9juYg09L/2AhGZ5583fDQgefyeGJNr+fViwSaP+HvWbYFTZ/ZdA1ypqpv9s9odVNU6IlIQ+FFE5uGbVfAS4HJ8ZwmuA8aetd0ywCigsX9bpVV1n4i8DxxR1SH+530CvKWqi0TkQnxnzl4GPAcsUtXBItIeCPezUo05zQq3CZXCIhLnv70Q3xwmDYBlqrrZv741cNWp8WugBFAD39zNE1XVCySKyDcZbP864IdT21LVzOYtbwlc7ptGBYDi/tkLGwM3+187W0T25+7XNCbvWeE2oZKkqjXTrvAXz6NpVwEPqepXZz0vmPObeIDrVPV4BlmMcSUb4zZO+gq4zz8VLCJysX+2ux+Abv4x8ApAswxeuwRoLCLV/K8t7V9/GCiW5nnz8E1whP95Nf03fwD+7V/XFigVrF/KmFCzwm2cNBrf+PUv4rsg8gh83wKnAX/4H/sA3wyJZ1DVPUAf4HP/LHmT/Q/NBG46tXMS+A9Q27/zcx1/H93yX3yFfy2+IZOtIfodjQk6mx3QGGNcxnrcxhjjMla4jTHGZaxwG2OMy1jhNsYYl7HCbYwxLmOF2xhjXMYKtzHGuMz/A8zCoa6eIJUfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix  \n",
    "# source: https://datatofish.com/confusion-matrix-python/\n",
    "\n",
    "confusion_matrix = pd.crosstab(df['label'], df['tweet_predict'], rownames=['Actual'], colnames=['Predicted'])\n",
    "\n",
    "sns.heatmap(confusion_matrix, annot=True)\n",
    "plt.show()\n",
    "\n",
    "# labels:\n",
    "        #0: indirect complaint\n",
    "        #1: remark\n",
    "        #2: negative remark\n",
    "        #3: direct compliment\n",
    "        #4: direct complaint\n",
    "        #5: none\n",
    "        #6: inquiry"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "telkomathon3",
   "language": "python",
   "name": "telkomathon3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}