end-to-end-disaster-NoDisaster-based-Tweet-classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "efd8dcdd",
   "metadata": {},
   "source": [
    "## End-to-End Sentiment Classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a221d5fa",
   "metadata": {},
   "source": [
    "`Things that we would cover`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "5188220d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 'Load the data',\n",
       " 2: 'Shuffle the data',\n",
       " 3: 'Split the data into traing and test sets',\n",
       " 4: 'Visualize the sentences',\n",
       " 5: 'Vectorize the sentences',\n",
       " 6: 'Embed the sentences',\n",
       " 7: 'Build our RNN',\n",
       " 8: 'Compile the RNN',\n",
       " 9: 'Fit the model',\n",
       " 10: 'Evaluate the predictions'}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "things_to_cover={1:'Load the data',\n",
    "                 2:'Shuffle the data',\n",
    "                 3:'Split the data into traing and test sets',\n",
    "                 4:'Visualize the sentences',\n",
    "                 5:'Vectorize the sentences',\n",
    "                 6:'Embed the sentences',\n",
    "                 7:'Build our RNN',\n",
    "                 8:'Compile the RNN',\n",
    "                 9:'Fit the model',\n",
    "                 10:'Evaluate the predictions'}\n",
    "things_to_cover"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c80534a3",
   "metadata": {},
   "source": [
    "### 1. Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "50e2517b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Turn .csv files into pandas DataFrame's\n",
    "import pandas as pd\n",
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e8df91e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text\n",
       "0   0     NaN      NaN                 Just happened a terrible car crash\n",
       "1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n",
       "2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n",
       "3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n",
       "4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f531a16",
   "metadata": {},
   "source": [
    "### 2. Shuffle the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "59d0e772",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2644</th>\n",
       "      <td>3796</td>\n",
       "      <td>destruction</td>\n",
       "      <td>NaN</td>\n",
       "      <td>So you have a new weapon that can cause un-ima...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2227</th>\n",
       "      <td>3185</td>\n",
       "      <td>deluge</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5448</th>\n",
       "      <td>7769</td>\n",
       "      <td>police</td>\n",
       "      <td>UK</td>\n",
       "      <td>DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>191</td>\n",
       "      <td>aftershock</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Aftershock back to school kick off was great. ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6845</th>\n",
       "      <td>9810</td>\n",
       "      <td>trauma</td>\n",
       "      <td>Montgomery County, MD</td>\n",
       "      <td>in response to trauma Children of Addicts deve...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id      keyword               location  \\\n",
       "2644  3796  destruction                    NaN   \n",
       "2227  3185       deluge                    NaN   \n",
       "5448  7769       police                     UK   \n",
       "132    191   aftershock                    NaN   \n",
       "6845  9810       trauma  Montgomery County, MD   \n",
       "\n",
       "                                                   text  target  \n",
       "2644  So you have a new weapon that can cause un-ima...       1  \n",
       "2227  The f$&amp;@ing things I do for #GISHWHES Just...       0  \n",
       "5448  DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...       1  \n",
       "132   Aftershock back to school kick off was great. ...       0  \n",
       "6845  in response to trauma Children of Addicts deve...       0  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Shuffle training dataframe\n",
    "train_df_shuffled = train_df.sample(frac=1, random_state=42) # shuffle with random_state=42 for reproducibility\n",
    "train_df_shuffled.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "8c99bfe5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    4342\n",
       "1    3271\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# How many examples of each class?\n",
    "train_df.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "31ad9c00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training samples: 7613\n",
      "Total test samples: 3263\n",
      "Total samples: 10876\n"
     ]
    }
   ],
   "source": [
    "# How many samples total?\n",
    "print(f\"Total training samples: {len(train_df)}\")\n",
    "print(f\"Total test samples: {len(test_df)}\")\n",
    "print(f\"Total samples: {len(train_df) + len(test_df)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c4b202b",
   "metadata": {},
   "source": [
    "### 3. Split the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "cca8a263",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Use train_test_split to split training data into training and validation sets\n",
    "train_sentences, val_sentences, train_labels, val_labels = train_test_split(train_df_shuffled[\"text\"].to_numpy(),\n",
    "                                                                            train_df_shuffled[\"target\"].to_numpy(),\n",
    "                                                                            test_size=0.1, # dedicate 10% of samples to validation set\n",
    "                                                                            random_state=42) # random state for reproducibility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "5bf771a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6851, 6851, 762, 762)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the lengths\n",
    "len(train_sentences), len(train_labels), len(val_sentences), len(val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "a24a2ea0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['@mogacola @zamtriossu i screamed after hitting tweet',\n",
       "        'Imagine getting flattened by Kurt Zouma',\n",
       "        '@Gurmeetramrahim #MSGDoing111WelfareWorks Green S welfare force ke appx 65000 members har time disaster victim ki help ke liye tyar hai....',\n",
       "        \"@shakjn @C7 @Magnums im shaking in fear he's gonna hack the planet\",\n",
       "        'Somehow find you and I collide http://t.co/Ee8RpOahPk',\n",
       "        '@EvaHanderek @MarleyKnysh great times until the bus driver held us hostage in the mall parking lot lmfao',\n",
       "        'destroy the free fandom honestly',\n",
       "        'Weapons stolen from National Guard Armory in New Albany still missing #Gunsense http://t.co/lKNU8902JE',\n",
       "        '@wfaaweather Pete when will the heat wave pass? Is it really going to be mid month? Frisco Boy Scouts have a canoe trip in Okla.',\n",
       "        'Patient-reported outcomes in long-term survivors of metastatic colorectal cancer - British Journal of Surgery http://t.co/5Yl4DC1Tqt'],\n",
       "       dtype=object),\n",
       " array([0, 0, 1, 0, 0, 1, 1, 0, 1, 1]))"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View the first 10 training sentences and their labels\n",
    "train_sentences[:10], train_labels[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcbf40c2",
   "metadata": {},
   "source": [
    "### 4. Visaulize the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "ae2fdcdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target: 1 (real disaster)\n",
      "Text:\n",
      "A Look at State Actions a Year after FergusonÛªs Upheaval https://t.co/M4tuI0P9nT MD is mentioned in the last group for a 'reporting' bill\n",
      "\n",
      "---\n",
      "\n",
      "Target: 0 (not real disaster)\n",
      "Text:\n",
      "Bad day\n",
      "\n",
      "---\n",
      "\n",
      "Target: 0 (not real disaster)\n",
      "Text:\n",
      "@TheDailyShow Mahalo nui loa for making my 20s. My generation could not have survived the (W.) Bush years without you. #JonVoyage #holomua\n",
      "\n",
      "---\n",
      "\n",
      "Target: 0 (not real disaster)\n",
      "Text:\n",
      "Only SHADOWMAN can save the Big Easy from a deluge of supernatural monstrosities... Read free: http://t.co/TBpQlYfYoT http://t.co/hDcKePosqc\n",
      "\n",
      "---\n",
      "\n",
      "Target: 0 (not real disaster)\n",
      "Text:\n",
      "UPDATE: Va. firefighter on administrative leave after Facebook post calls for people to put officers in 'body bags.' http://t.co/Mt029QJ4Ig\n",
      "\n",
      "---\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Let's visualize some random training examples\n",
    "import random\n",
    "random_index = random.randint(0, len(train_df)-5) # create random indexes not higher than the total number of samples\n",
    "for row in train_df_shuffled[[\"text\", \"target\"]][random_index:random_index+5].itertuples():\n",
    "  _, text, target = row\n",
    "  print(f\"Target: {target}\", \"(real disaster)\" if target > 0 else \"(not real disaster)\")\n",
    "  print(f\"Text:\\n{text}\\n\")\n",
    "  print(\"---\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "392ef5fe",
   "metadata": {},
   "source": [
    "### 5. Vectorize the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "3498507f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
    "# Note: in TensorFlow 2.6+, you no longer need \"layers.experimental.preprocessing\"\n",
    "# you can use: \"tf.keras.layers.TextVectorization\", see https://github.com/tensorflow/tensorflow/releases/tag/v2.6.0 for more\n",
    "\n",
    "# Use the default TextVectorization variables\n",
    "text_vectorizer = TextVectorization(max_tokens=None, # how many words in the vocabulary (all of the different words in your text)\n",
    "                                    standardize=\"lower_and_strip_punctuation\", # how to process text\n",
    "                                    split=\"whitespace\", # how to split tokens\n",
    "                                    ngrams=None, # create groups of n-words?\n",
    "                                    output_mode=\"int\", # how to map tokens to numbers\n",
    "                                    output_sequence_length=None) # how long should the output sequence of tokens be?\n",
    "                                    # pad_to_max_tokens=True) # Not valid if using max_tokens=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "23e9c51d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Find average number of tokens (words) in training Tweets\n",
    "round(sum([len(i.split()) for i in train_sentences])/len(train_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "a9771ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup text vectorization with custom variables\n",
    "max_vocab_length = 10000 # max number of words to have in our vocabulary\n",
    "max_length = 15 # max length our sequences will be (e.g. how many words from a Tweet does our model see?)\n",
    "\n",
    "text_vectorizer = TextVectorization(max_tokens=max_vocab_length,\n",
    "                                    output_mode=\"int\",\n",
    "                                    output_sequence_length=max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "348d0a2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-25 14:14:10.918367: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    }
   ],
   "source": [
    "# Fit the text vectorizer to the training text\n",
    "text_vectorizer.adapt(train_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "28f55015",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
       "array([[264,   3, 232,   4,  13, 698,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0]])>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create sample sentence and tokenize it\n",
    "sample_sentence = \"There's a flood in my street!\"\n",
    "text_vectorizer([sample_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "51b83a8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text:\n",
      "å¤} New Ladies Shoulder Tote #Handbag Faux Leather Hobo Purse Cross Body Bag #Womens http://t.co/UooZXauS26 http://t.co/6MGBizjfgd RT enÛ_      \n",
      "\n",
      "Vectorized version:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\n",
       "array([[6387,   50,  814,  530,  980, 1014, 1139,  703, 1399, 1183,  248,\n",
       "          83,  322,  753, 5423]])>"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choose a random sentence from the training dataset and tokenize it\n",
    "random_sentence = random.choice(train_sentences)\n",
    "print(f\"Original text:\\n{random_sentence}\\\n",
    "      \\n\\nVectorized version:\")\n",
    "text_vectorizer([random_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "9edf23f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in vocab: 10000\n",
      "Top 5 most common words: ['', '[UNK]', 'the', 'a', 'in']\n",
      "Bottom 5 least common words: ['pages', 'paeds', 'pads', 'padres', 'paddytomlinson1']\n"
     ]
    }
   ],
   "source": [
    "# Get the unique words in the vocabulary\n",
    "words_in_vocab = text_vectorizer.get_vocabulary()\n",
    "top_5_words = words_in_vocab[:5] # most common tokens (notice the [UNK] token for \"unknown\" words)\n",
    "bottom_5_words = words_in_vocab[-5:] # least common tokens\n",
    "print(f\"Number of words in vocab: {len(words_in_vocab)}\")\n",
    "print(f\"Top 5 most common words: {top_5_words}\") \n",
    "print(f\"Bottom 5 least common words: {bottom_5_words}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c5d078e",
   "metadata": {},
   "source": [
    "### 6. Embedding the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "75d41092",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<keras.layers.core.embedding.Embedding at 0x2c6ac25e0>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.random.set_seed(42)\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "embedding = layers.Embedding(input_dim=max_vocab_length, # set input shape\n",
    "                             output_dim=128, # set size of embedding vector\n",
    "                             embeddings_initializer=\"uniform\", # default, intialize randomly\n",
    "                             input_length=max_length, # how long is each input\n",
    "                             name=\"embedding_1\") \n",
    "\n",
    "embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "d91bae2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text:\n",
      "Due to the rainstorm last night cupcake decorating is happening NOW at the Rec Hall! $2 - proceeds to #IWK! http://t.co/EaRONLwIFh      \n",
      "\n",
      "Embedded version:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 15, 128), dtype=float32, numpy=\n",
       "array([[[-0.01929183, -0.00627624,  0.0359194 , ..., -0.03173238,\n",
       "          0.04895258, -0.01275152],\n",
       "        [-0.03059796, -0.01728977,  0.0235858 , ...,  0.01896751,\n",
       "          0.03643342, -0.01393596],\n",
       "        [ 0.0377439 ,  0.03986558,  0.03102725, ...,  0.00220744,\n",
       "          0.03912038,  0.0065724 ],\n",
       "        ...,\n",
       "        [ 0.0377439 ,  0.03986558,  0.03102725, ...,  0.00220744,\n",
       "          0.03912038,  0.0065724 ],\n",
       "        [ 0.01304138,  0.0141281 ,  0.00136916, ..., -0.03578488,\n",
       "         -0.03272915,  0.04211178],\n",
       "        [-0.04398369, -0.0233928 , -0.01553049, ..., -0.04928381,\n",
       "          0.01249143,  0.0036379 ]]], dtype=float32)>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a random sentence from training set\n",
    "random_sentence = random.choice(train_sentences)\n",
    "print(f\"Original text:\\n{random_sentence}\\\n",
    "      \\n\\nEmbedded version:\")\n",
    "\n",
    "# Embed the random sentence (turn it into numerical representation)\n",
    "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
    "sample_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "715109d9",
   "metadata": {},
   "source": [
    "### 7. Building our RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "066a6084",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 15, 128)\n",
      "(None, 64)\n"
     ]
    }
   ],
   "source": [
    "# Set random seed and create embedding layer (new embedding layer for each model)\n",
    "tf.random.set_seed(42)\n",
    "from tensorflow.keras import layers\n",
    "model_2_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
    "                                     output_dim=128,\n",
    "                                     embeddings_initializer=\"uniform\",\n",
    "                                     input_length=max_length,\n",
    "                                     name=\"embedding_2\")\n",
    "\n",
    "\n",
    "# Create LSTM model\n",
    "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = model_2_embedding(x)\n",
    "print(x.shape)\n",
    "# x = layers.LSTM(64, return_sequences=True)(x) # return vector for each word in the Tweet (you can stack RNN cells as long as return_sequences=True)\n",
    "x = layers.LSTM(64)(x) # return vector for whole sequence\n",
    "print(x.shape)\n",
    "# x = layers.Dense(64, activation=\"relu\")(x) # optional dense layer on top of output of LSTM cell\n",
    "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "model_2 = tf.keras.Model(inputs, outputs, name=\"model_2_LSTM\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aff5b18",
   "metadata": {},
   "source": [
    "### 8. Compiling the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "fc7d09f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile model\n",
    "model_2.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82fffe85",
   "metadata": {},
   "source": [
    "### 9. fit the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "b2ead2a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-25 14:14:32.187952: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2023-03-25 14:14:32.278710: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  3/215 [..............................] - ETA: 8s - loss: 0.6923 - accuracy: 0.5208  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-25 14:14:32.399097: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 [==============================] - ETA: 0s - loss: 0.5101 - accuracy: 0.7437"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-03-25 14:14:36.341912: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n",
      "2023-03-25 14:14:36.382670: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:114] Plugin optimizer for device_type GPU is enabled.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 [==============================] - 5s 20ms/step - loss: 0.5101 - accuracy: 0.7437 - val_loss: 0.4565 - val_accuracy: 0.7808\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 3s 15ms/step - loss: 0.3155 - accuracy: 0.8726 - val_loss: 0.5164 - val_accuracy: 0.7822\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 3s 15ms/step - loss: 0.2179 - accuracy: 0.9162 - val_loss: 0.5951 - val_accuracy: 0.7717\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 3s 15ms/step - loss: 0.1495 - accuracy: 0.9483 - val_loss: 0.6525 - val_accuracy: 0.7795\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 3s 15ms/step - loss: 0.1044 - accuracy: 0.9602 - val_loss: 0.8309 - val_accuracy: 0.7533\n"
     ]
    }
   ],
   "source": [
    "# Fit model\n",
    "model_2_history = model_2.fit(train_sentences,\n",
    "                              train_labels,\n",
    "                              epochs=5,\n",
    "                              validation_data=(val_sentences, val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "87c0e9ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['DFR EP016 Monthly Meltdown - On Dnbheaven 2015.08.06 http://t.co/EjKRf8N8A8 #Drum and Bass #heavy #nasty http://t.co/SPHWE6wFI5',\n",
       "       'FedEx no longer to transport bioterror germs in wake of anthrax lab mishaps http://t.co/qZQc8WWwcN via @usatoday'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_sentences[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c38fada",
   "metadata": {},
   "source": [
    "### 10. Check out the predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "9c756927",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 34ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.02363209]], dtype=float32)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2_preds = model_2.predict([\"I am going for a party!\"])\n",
    "model_2_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "45cd6bda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No flood!\n"
     ]
    }
   ],
   "source": [
    "if model_2_preds>0.5:\n",
    "    print(\"Flood Statement!\")\n",
    "else:\n",
    "    print(\"No flood!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fad08b3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}