BERT-Sentiment-Analysis-IMDbReviews.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d7kKTpgGM6Bp"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #ffa900, #afcd95, #3eb8a5); padding: 20px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 32px;\"> BERT Mediated Sentiment Analysis of IMDB Movie Reviews</h1>  \n",
    "    \n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zaMg7uN51h_j"
   },
   "source": [
    "Table Of Contents:\n",
    "1. Coding Environment Prepararation  \n",
    "2. Data Download  \n",
    "    2.1 Tokenize Dataset  \n",
    "    2.2 Dataset Processing   \n",
    "3. Pre-Trained BERT Model  \n",
    "    3.1. Dataset Processing for BERT Model\n",
    "4. BERT Model Training   \n",
    "    4.1. Trained Model Save & Load   \n",
    "    4.2. Trained BERT Model Evaluation  \n",
    "5. Mixed Sentiments Identification by trained BERT Model\n",
    "6. Modules & Packages Used In The Jupyter Notebook   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "stsr7fOF2-29"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Coding Environment Prepararation</h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "I739AlrDNkrB",
    "outputId": "faea09da-fcaa-47ed-eed3-7f46f6e45e12"
   },
   "outputs": [],
   "source": [
    "# Install datasets library of HuggingFace\n",
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "eudIPUn9NDlt"
   },
   "outputs": [],
   "source": [
    "# Import required packages\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import random as rn\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from transformers import BertModel\n",
    "from transformers import BertTokenizer\n",
    "from transformers import BertConfig\n",
    "from transformers import BertForSequenceClassification\n",
    "\n",
    "from transformers import AdamW, get_scheduler\n",
    "from datasets import load_dataset\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torch.utils.data import RandomSampler, SequentialSampler\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from IPython.display import clear_output\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "CvjRMb7kNDwx"
   },
   "outputs": [],
   "source": [
    "# Setting random seeds for reproducible results\n",
    "rn.seed(321)\n",
    "np.random.seed(321)\n",
    "torch.manual_seed(321)\n",
    "torch.cuda.manual_seed(321)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BxkZ2MVq3nSx"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Data Download </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B8Pqoaw_ND4p",
    "outputId": "b813f18e-a8d3-48b8-e664-c6a2f995628a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
      "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
      "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
      "You will be able to reuse this secret in all of your notebooks.\n",
      "Please note that authentication is recommended but still optional to access public models or datasets.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 25000\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 25000\n",
       "    })\n",
       "    unsupervised: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 50000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load_dataset is function of datasets library of HuggingFace\n",
    "dataset = load_dataset('imdb')\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FTu2O3vO3eGe"
   },
   "source": [
    "**Remarks:**\n",
    "- Dataset contains train_dataset, test_dataset, and unsupervised_dataset dictionaries with the respective dataset splits.\n",
    "    - Individual data samples can be accessed using these splits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 363
    },
    "id": "mMhSkiwFORli",
    "outputId": "3b7e17eb-d76e-41a8-d7ad-68b27c4ccc93"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"              'label':dataset['train'][:10]['label']})\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"Who are these \\\"They\\\"- the actors? the filmmakers? Certainly couldn't be the audience- this is among the most air-puffed productions in existence. It's the kind of movie that looks like it was a lot of fun to shoot\\u0097 TOO much fun, nobody is getting any actual work done, and that almost always makes for a movie that's no fun to watch.<br /><br />Ritter dons glasses so as to hammer home his character's status as a sort of doppleganger of the bespectacled Bogdanovich; the scenes with the breezy Ms. Stratten are sweet, but have an embarrassing, look-guys-I'm-dating-the-prom-queen feel to them. Ben Gazzara sports his usual cat's-got-canary grin in a futile attempt to elevate the meager plot, which requires him to pursue Audrey Hepburn with all the interest of a narcoleptic at an insomnia clinic. In the meantime, the budding couple's respective children (nepotism alert: Bogdanovich's daughters) spew cute and pick up some fairly disturbing pointers on 'love' while observing their parents. (Ms. Hepburn, drawing on her dignity, manages to rise above the proceedings- but she has the monumental challenge of playing herself, ostensibly.) Everybody looks great, but so what? It's a movie and we can expect that much, if that's what you're looking for you'd be better off picking up a copy of Vogue.<br /><br />Oh- and it has to be mentioned that Colleen Camp thoroughly annoys, even apart from her singing, which, while competent, is wholly unconvincing... the country and western numbers are woefully mismatched with the standards on the soundtrack. Surely this is NOT what Gershwin (who wrote the song from which the movie's title is derived) had in mind; his stage musicals of the 20's may have been slight, but at least they were long on charm. \\\"They All Laughed\\\" tries to coast on its good intentions, but nobody- least of all Peter Bogdanovich - has the good sense to put on the brakes.<br /><br />Due in no small part to the tragic death of Dorothy Stratten, this movie has a special place in the heart of Mr. Bogdanovich- he even bought it back from its producers, then distributed it on his own and went bankrupt when it didn't prove popular. His rise and fall is among the more sympathetic and tragic of Hollywood stories, so there's no joy in criticizing the film... there _is_ real emotional investment in Ms. Stratten's scenes. But \\\"Laughed\\\" is a faint echo of \\\"The Last Picture Show\\\", \\\"Paper Moon\\\" or \\\"What's Up, Doc\\\"- following \\\"Daisy Miller\\\" and \\\"At Long Last Love\\\", it was a thundering confirmation of the phase from which P.B. has never emerged.<br /><br />All in all, though, the movie is harmless, only a waste of rental. I want to watch people having a good time, I'll go to the park on a sunny day. For filmic expressions of joy and love, I'll stick to Ernest Lubitsch and Jaques Demy...\",\n          \"\\\"I Am Curious: Yellow\\\" is a risible and pretentious steaming pile. It doesn't matter what one's political views are because this film can hardly be taken seriously on any level. As for the claim that frontal male nudity is an automatic NC-17, that isn't true. I've seen R-rated films with male nudity. Granted, they only offer some fleeting views, but where are the R-rated films with gaping vulvas and flapping labia? Nowhere, because they don't exist. The same goes for those crappy cable shows: schlongs swinging in the breeze but not a clitoris in sight. And those pretentious indie movies like The Brown Bunny, in which we're treated to the site of Vincent Gallo's throbbing johnson, but not a trace of pink visible on Chloe Sevigny. Before crying (or implying) \\\"double-standard\\\" in matters of nudity, the mentally obtuse should take into account one unavoidably obvious anatomical difference between men and women: there are no genitals on display when actresses appears nude, and the same cannot be said for a man. In fact, you generally won't see female genitals in an American film in anything short of porn or explicit erotica. This alleged double-standard is less a double standard than an admittedly depressing ability to come to terms culturally with the insides of women's bodies.\",\n          \"I would put this at the top of my list of films in the category of unwatchable trash! There are films that are bad, but the worst kind are the ones that are unwatchable but you are suppose to like them because they are supposed to be good for you! The sex sequences, so shocking in its day, couldn't even arouse a rabbit. The so called controversial politics is strictly high school sophomore amateur night Marxism. The film is self-consciously arty in the worst sense of the term. The photography is in a harsh grainy black and white. Some scenes are out of focus or taken from the wrong angle. Even the sound is bad! And some people call this art?<br /><br />\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"label\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 0,\n        \"max\": 0,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-4aa497b2-b2f9-4c00-ac01-0ba24a6d8a3c\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I rented I AM CURIOUS-YELLOW from my video sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"I Am Curious: Yellow\" is a risible and preten...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If only to avoid making this type of film in t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>This film was probably inspired by Godard's Ma...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Oh, brother...after hearing about this ridicul...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>I would put this at the top of my list of film...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Whoever wrote the screenplay for this movie ob...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>When I first saw a glimpse of this movie, I qu...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Who are these \"They\"- the actors? the filmmake...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>This is said to be a personal film for Peter B...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4aa497b2-b2f9-4c00-ac01-0ba24a6d8a3c')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-4aa497b2-b2f9-4c00-ac01-0ba24a6d8a3c button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-4aa497b2-b2f9-4c00-ac01-0ba24a6d8a3c');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-674ad87f-6017-4fd4-ab12-3d0d3ad3214f\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-674ad87f-6017-4fd4-ab12-3d0d3ad3214f')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-674ad87f-6017-4fd4-ab12-3d0d3ad3214f button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  I rented I AM CURIOUS-YELLOW from my video sto...      0\n",
       "1  \"I Am Curious: Yellow\" is a risible and preten...      0\n",
       "2  If only to avoid making this type of film in t...      0\n",
       "3  This film was probably inspired by Godard's Ma...      0\n",
       "4  Oh, brother...after hearing about this ridicul...      0\n",
       "5  I would put this at the top of my list of film...      0\n",
       "6  Whoever wrote the screenplay for this movie ob...      0\n",
       "7  When I first saw a glimpse of this movie, I qu...      0\n",
       "8  Who are these \"They\"- the actors? the filmmake...      0\n",
       "9  This is said to be a personal film for Peter B...      0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check data content as pandas DF\n",
    "pd.DataFrame({'text':dataset['train'][:10]['text'],\n",
    "              'label':dataset['train'][:10]['label']})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fpns-95D5PAi"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Tokenize Dataset </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QvfiKXXvOSld",
    "outputId": "d3474d47-2134-4edd-cc7c-3245baaae155"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Load BERT Tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gHdh6URbOZvQ",
    "outputId": "66ecccaa-7ede-4527-d5cc-0b4d519dfaab"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'love', 'korean', 'food', '!']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check how “tokenizer” object work\n",
    "input_1 = 'I love korean food!'\n",
    "tokenizer.tokenize(input_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xVFjeDT87HCJ"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Pre-Trained BERT Model </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9nN18ryJPNTS",
    "outputId": "5268ed00-5927-44b3-bae1-caad2e9a77e2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load BERT for sequence classification with binary label output\n",
    "# Output classes: positive and negative, thus num_labels = 2\n",
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased',\n",
    "                                                      num_labels = 2)\n",
    "# Move model to the correct device\n",
    "# Define the device (GPU/CPU)\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QvbqmAV25X4P"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Dataset Processing for BERT Model </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "S2LjKYrmqSb5"
   },
   "outputs": [],
   "source": [
    "# Split dataset into train/test\n",
    "train_ds = dataset['train']\n",
    "test_ds = dataset['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6qU1a0yBs67a",
    "outputId": "339a7818-ff69-472b-bf6e-daef9d315d23"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['text', 'label'],\n",
       "    num_rows: 25000\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "PI-bEoGtO98u"
   },
   "outputs": [],
   "source": [
    "# Function to preprocess the text into tokens\n",
    "def preprocess_function(examples):\n",
    "    return tokenizer(examples['text'],\n",
    "                     padding = True,\n",
    "                     truncation = True,\n",
    "                     return_tensors = 'pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "RNfIrxkAPIfl"
   },
   "outputs": [],
   "source": [
    "# Tokenize dataset\n",
    "tokenized_train_ds = train_ds.map(preprocess_function, batched = True)\n",
    "tokenized_test_ds = test_ds.map(preprocess_function, batched = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 81,
     "referenced_widgets": [
      "3fdb00a399c640aba3e3b83ae4ba5846",
      "eadca395412b4b83a165e432e9af0a65",
      "5aeeb3b223c84297b72ff7a233a8cf19",
      "df272f2389604ae3a4100c7519954220",
      "956285db7c564394bd37678539736514",
      "765842c4db114276ba1a60f8756b7376",
      "28fe3a412dd8477bb0a8db40a1d886eb",
      "2676f08e4c8a49c7ba6b18a34496ce6f",
      "38003314725246d0ab4b261e99529525",
      "49f787fbccba48dc979f9b476bc6e1bd",
      "f0b780a2cb3d4bafb6f6c1e7de55fd5c",
      "c6077da86a9e431aacf61b015a3039d6",
      "17005aa581f44a899c5e22a33cd9f9fc",
      "6294f9369c6e425495f6eaceb81491e7",
      "4829c4f88cd24775a1390246afd80f6d",
      "5a24f56ebf2a4c9cb19fc3ee1404d550",
      "a39ab825d0c4416dbff3418421577865",
      "db1979d9173d45ab8f6f6a5560f0cb81",
      "b3c8bbeeeb504a9b8548ee0411d2194b",
      "cac524efd7b948fdb3a414b3ea45f8c3",
      "ccc2c339d7f64086b03ab860c143a925",
      "9a0fbaaf656b41ab8da55b8e5c7d464e"
     ]
    },
    "id": "u3HkOUcpZSkt",
    "outputId": "25aec707-0055-412f-bc6f-144ba8a72678"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fdb00a399c640aba3e3b83ae4ba5846",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/5000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6077da86a9e431aacf61b015a3039d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1500 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Subsample datasets by select() for faster training\n",
    "train_dataset = tokenized_train_ds.shuffle(seed = 42).select(range(5000))\n",
    "test_dataset = tokenized_test_ds.shuffle(seed = 42).select(range(1500))\n",
    "\n",
    "def format_dataset(batch):\n",
    "    # Convert the labels and the inputs to PyTorch tensors\n",
    "    batch['label'] = torch.tensor(batch['label'])\n",
    "    batch['input_ids'] = torch.tensor(batch['input_ids'])\n",
    "    batch['attention_mask'] = torch.tensor(batch['attention_mask'])\n",
    "    return batch\n",
    "\n",
    "# Apply the formatting to the dataset\n",
    "train_dataset = train_dataset.map(format_dataset)\n",
    "test_dataset = test_dataset.map(format_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "z0hLKduicxQr"
   },
   "outputs": [],
   "source": [
    "# Used collate_fn to ensure batch data is handled correctly\n",
    "def collate_fn(batch):\n",
    "    # Stack the input_ids, attention_mask, and labels into tensors\n",
    "    input_ids = torch.stack([torch.tensor(item['input_ids']) for item in batch])\n",
    "    attention_mask = torch.stack([torch.tensor(item['attention_mask']) for item in batch])\n",
    "    labels = torch.tensor([item['label'] for item in batch])\n",
    "\n",
    "    return {'input_ids': input_ids,\n",
    "            'attention_mask': attention_mask,\n",
    "            'labels': labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "BZKmw6oQZgIF"
   },
   "outputs": [],
   "source": [
    "# Pass collate_fn to the DataLoader to ensure proper batching of data\n",
    "train_dataloader = DataLoader(train_dataset, batch_size = 8,\n",
    "                              shuffle = True,\n",
    "                              collate_fn = collate_fn)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size = 8,\n",
    "                             collate_fn = collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XxFSz2As7Zif"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> BERT Model Training </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "updzjx7IbyUU",
    "outputId": "bfba8770-7f6c-4cf2-8a2c-94adeae30db6"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Define the optimizer\n",
    "optimizer = AdamW(model.parameters(), lr = 5e-5)\n",
    "\n",
    "# Define the learning rate scheduler\n",
    "num_training_steps = len(train_dataloader) * 3 # 3 epochs\n",
    "lr_scheduler = get_scheduler('linear',\n",
    "                             optimizer = optimizer,\n",
    "                             num_warmup_steps = 0,\n",
    "                             num_training_steps = num_training_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "iP6EfHaTb6cG",
    "outputId": "2d16df71-39b5-4464-cca3-60860cd40fd2"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Epoch 1: 100%|██████████| 625/625 [07:41<00:00,  1.35it/s, loss=0.127]\n",
      "Training Epoch 2: 100%|██████████| 625/625 [07:45<00:00,  1.34it/s, loss=0.172]\n",
      "Training Epoch 3: 100%|██████████| 625/625 [07:45<00:00,  1.34it/s, loss=0.0158]\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "model.train()\n",
    "\n",
    "for epoch in range(3): # Train for 3 epochs\n",
    "    progress_bar = tqdm(train_dataloader, desc = f\"Training Epoch {epoch + 1}\")\n",
    "\n",
    "    for batch in progress_bar:\n",
    "        # Move inputs and labels to appropriate device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "\n",
    "        # Forward pass\n",
    "        outputs = model(input_ids = input_ids,\n",
    "                    attention_mask = attention_mask,\n",
    "                    labels = labels)\n",
    "        loss = outputs.loss\n",
    "\n",
    "        # Backward pass and optimization\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        progress_bar.set_postfix(loss = loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "a7npexCW7kAr"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Trained Model Save & Load </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "XY7YMzsxkQtv"
   },
   "outputs": [],
   "source": [
    "# Save the trained model\n",
    "model.save_pretrained('imdb-trained-bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pxCtb1Z3lBW1",
    "outputId": "ac4af401-26ca-4560-f84f-9c226a59ea15"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('imdb-trained-bert/tokenizer_config.json',\n",
       " 'imdb-trained-bert/special_tokens_map.json',\n",
       " 'imdb-trained-bert/vocab.txt',\n",
       " 'imdb-trained-bert/added_tokens.json')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the tokenizer\n",
    "tokenizer.save_pretrained('imdb-trained-bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "29N294HGlsJt"
   },
   "outputs": [],
   "source": [
    "# Load saved tokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('imdb-trained-bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "71w1brGxlcTH"
   },
   "outputs": [],
   "source": [
    "# Load saved model\n",
    "model = BertForSequenceClassification.from_pretrained('imdb-trained-bert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eC8xLe9IlrNO",
    "outputId": "333202d6-55bd-478d-df26-725cdf8e9d48"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BertConfig {\n",
      "  \"_name_or_path\": \"imdb-trained-bert\",\n",
      "  \"architectures\": [\n",
      "    \"BertForSequenceClassification\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"problem_type\": \"single_label_classification\",\n",
      "  \"torch_dtype\": \"float32\",\n",
      "  \"transformers_version\": \"4.44.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Check model configuration\n",
    "print(model.config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HGt-cxSooCzA",
    "outputId": "f0a275c9-c7b3-4515-8dd7-98fd8302b502"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Move model to the correct device after loading the model\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tdwxCsxo8C5T"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Trained BERT Model Evaluation </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "9FFC9tcwnDwK"
   },
   "outputs": [],
   "source": [
    "# Set the model to evaluation mode\n",
    "model.eval()\n",
    "\n",
    "all_predictions = []\n",
    "all_labels = []\n",
    "\n",
    "# Disable gradient calculation for evaluation\n",
    "with torch.no_grad():\n",
    "    for batch in test_dataloader:\n",
    "        # Move inputs to appropriate device\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        labels = batch['labels'].to(device)\n",
    "\n",
    "        # Get predictions\n",
    "        outputs = model(input_ids = input_ids,\n",
    "                        attention_mask = attention_mask)\n",
    "\n",
    "        # Get logits\n",
    "        logits = outputs.logits\n",
    "        # Get predicted class (argmax)\n",
    "        predictions = torch.argmax(logits, dim = -1)\n",
    "\n",
    "        # Store predictions and true labels\n",
    "        all_predictions.extend(predictions.cpu().numpy())\n",
    "        all_labels.extend(labels.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xgBhwDgTnk4S",
    "outputId": "7659c6ac-6ea1-4e2b-d752-3978bcb97054"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    Negative       0.93      0.91      0.92       762\n",
      "    Positive       0.91      0.92      0.91       738\n",
      "\n",
      "    accuracy                           0.92      1500\n",
      "   macro avg       0.92      0.92      0.92      1500\n",
      "weighted avg       0.92      0.92      0.92      1500\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generate classification report\n",
    "report = classification_report(all_labels,\n",
    "                               all_predictions,\n",
    "                               target_names = ['Negative', 'Positive'])\n",
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 487
    },
    "id": "77axi9w4obhi",
    "outputId": "2a3932c5-96b6-49b0-9715-fd73dd059e4f"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgEAAAHWCAYAAADuNVprAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTfElEQVR4nO3dd1gU1/4/8PfSliZLCdUgohgURWM0UazRkGCNXklMFBVjiwQsWKLcJEZNIsbEYDAq0RhLxFhjw4q9YS/x2guKBcQGCEjd8/vDH/N1BRV0l12Y9+s+8zzsmTNnPrPuzX72lBmFEEKAiIiIZMdI3wEQERGRfjAJICIikikmAURERDLFJICIiEimmAQQERHJFJMAIiIimWISQEREJFNMAoiIiGSKSQAREZFMMQkgKsHixYvRoEEDWFpaQqFQwNbWVmfn2rlzJxQKBRQKBfr27auz81RG7777rvTeXb16Vd/hEFU4TALIYGRlZSEqKgqtWrWCg4MDzM3N4enpiU6dOmHRokXIy8srlzgSEhLQq1cv/Pvvv3j06FG5nNNQjB8/XvpSVSgU+OCDD4rVOXr0qEYdhUKBnJyclzrf6tWrMX78eIwfP55f4kR6YKLvAIgA4MyZM+jcuTOuXLmiUX716lVcvXoV69evR7169fDmm2/qPJb169ej6JEan3/+OYKCgmBqaqqz8zVs2BB79uwBADg7O+vsPC9j27ZtuHbtGjw8PKSyOXPmaK391atXY8GCBQAe/6qvXr16mY6fPn060tPTAQCurq5ai4tILpgEkN7dv38f7du3R1JSEgDAzc0No0ePhq+vLx4+fIhdu3Zh3rx55RbPrVu3pL8/+eQTtGzZUqfnU6lUaNGihU7P8bLUajXmzp2LiRMnAnjcW7N48WI9R/U4DisrK/j6+uo7FKKKTRDpWUREhAAgAAiVSiVu3LhRrM7t27fFvXv3pNe5ubli8uTJokGDBsLS0lJYWFiI+vXri8jISJGbm6txrIeHh9R+cnKy6NWrl7C1tRXW1taie/fuUruJiYlSvae31q1bCyGE9NrDw0PjHK1bt5b2JSYmSuUrVqwQzZs3FzY2NsLU1FQ4OzuL5s2biy+//FKo1WohhBA7duyQjg0ODtZoNzk5WQwZMkTUqFFDmJmZCZVKJVq3bi2WLVumUe/J2Fu3bi0OHTok3n33XWFhYSGcnZ3FV199JQoLC1/4b/Htt99K7VSpUkUAEK+//rp07Ny5czX2FW2PHj2S2hgxYoTw8/MTLi4uwszMTFhZWYmGDRuKn376SeTn57/wvQYgduzYUez9/vfff4W/v7+wsrKS/j2eft/VarVo27atVBYXFyfFFRISIpVPmjTphe8FkRwwCSC9q1GjhvQf5/Hjx7+wfk5OjmjVqtUzv0BatWqlkQg8mQQ8ea6iLSgoSAih/SRg586dwsjI6JltFn0hPisJuHLlinBxcXnm8WPGjJHqPhm7q6ursLCwKFZ/zpw5L3xvn0wC+vbtK0xNTQUAsX79eiGEEE2aNBEAxKBBg56ZBCiVymfG/Nlnn73wvS4pCVCpVMLBwaHYv0dJ73tiYqKwtraW/p0yMzPF3r17hUKhEADEO++8IwoKCl74XhDJAScGkl5lZmZqzAMoTdf7tGnTsHv3bgCAu7s7Fi9ejL///hvVqlUDAOzevRtRUVElHvvo0SMsWrQIM2fOhJmZGQBgyZIlSE9Ph6urK/bs2YP27dtL9aOjo7Fnzx5Mnz69zNe2bt06qNVqAMCkSZOwbds2LFmyBF9//TV8fHygUCiee/wXX3yBlJQUAI/Hy9euXYtffvkF5ubmAIAff/wRBw8eLHZccnIy3nrrLaxZswZDhw6Vyn///fcyxe/s7IxOnToBAP744w+cOnVKOt+AAQOeedxXX32Fv//+G5s2bcLOnTvxzz//oEmTJgCA+fPn48aNG899r/fs2YOGDRtqtJmeng5jY2PMnj0bmzdvfu75q1evjp9++gkAcO3aNURERGDQoEEQQsDc3BwLFiyAsbFxmd4LokpL31kIyduNGzc0fgGePXv2hcfUr19fqr9u3TqpfN26dVJ5gwYNpPInewJWrVollbdr104qP3HihFQeHBxc7BdpkaLy0vQEjB07Vipbvny5uHv3bonXU1JPwL1796RfrkqlUuPYkSNHSvWHDRsmhND8ZW1mZiZSUlKEEEIUFhYKS0tLAUDY2tq+8L19sidgzJgxYv369QKAMDU1Fd27dxcARP369TXeCzzVE7B3717RpUsX4eLiIkxMTIr9yl+zZk2p3uunz7Fly5Zi+581DCOEEP7+/sXOPXXq1Be+B0Rywp4A0iuVSqXx+slJec9y4cIF6e+iX5gA8M4775RY50mtW7eW/nZwcJD+TktLe+F5yyooKAhKpRIA8PHHH+O1116Ds7MzunXrhq1btz732IsXL0orFGrWrKkR64uus3bt2tIqAyMjI9jZ2QF4uWts164d3N3dkZ+fj2XLlgEABg4c+Mz6hw4dQps2bbBmzRqkpKSgoKCgWJ2XicPc3Bzvv/9+mY6ZO3curKyspNdNmzbF8OHDy3xuosqMSQDplbW1NWrUqCG93rdv30u39aLudQDSFyIAmJj83+KYoi/c0iosLNR4fffu3WJ16tWrh6NHj2Lo0KFo0qQJVCoVUlNTsWrVKgQEBGD//v1lOmeRF13nk9cIaF5nWRkZGeGzzz6TXpubm6NXr17PrB8TE4P8/HwAQKdOnbBhwwbs2bMHffr0keoUDZGUhZOTU5mPuXbtGrKzs6XXSUlJyMjIKHM7RJUZkwDSu08++UT6+5dffimxNyA1NRX3798HALzxxhtS+aFDh6S/nxwff7KONhX1XNy7d0/6srt69SrOnTtXrK4QAnXr1sWvv/6KAwcOIC0tDStWrADw+Itw9erVzzyPl5eX9GV/+fJl3Lt3T9pXHtf5pH79+sHI6PF/KgIDA59798SbN29Kf0dGRqJ9+/Zo0aIFbt++XWL9onaB5ycHpUnwnpSdnY3PPvsMQghp/P/WrVvsCSB6Cu8TQHo3atQoxMbGIikpCWlpaWjSpAlGjRol3Sdg586dmDdvHnbu3Al7e3v07NkT//77LwAgNDQUDx8+hEKhwNixY6U2e/TooZNYvby8cPToUTx69Ag9e/ZEq1atMHPmzGI9AwAwZcoU7Ny5Ex07dkS1atVgZWWFzZs3S/tzc3OfeR4HBwcEBARg06ZNyM3NRffu3REeHo7Lly9j5syZUj1dXeeTPDw8MGPGDKSkpOCjjz56Yd0ikZGRCA4OxsaNGzWu+0lP9losWrQIxsbGMDY2fuX7JowZMwaXL18G8Hgi6Zo1a7B161YsWLAAH3/8MTp27PhK7RNVGvqdkkD02OnTp0tcvvfkdvz4cSHE4yWCLVu2fGa95y0RfNKzJqU9b7La77//Xux81tbW4vXXXy82Qe277757ZoxGRkZi7969QohnLxG8fPnySy0RLFo+96LrL8nTEwOf58lYiiYGHjx4UJrQWLQpFArh5+cnvZ43b57UxpOTOZ/cnj7H0xMxi5Q0MXDHjh1SDH5+fqKwsFBcuXJFWFlZCQDCzc1NPHjw4IXvBZEccDiADIKPjw/+/fdf/PLLL2jRogXs7e1hZmYGd3d3BAQEYMGCBfDx8QEAKJVKxMfHY/Lkyahfvz4sLCxgbm4OX19fREZGYsuWLdLyP20bMGAAIiIi4OTkBAsLC7Rt2xZ79uxBzZo1i9Xt0KEDPv/8c9SrVw92dnYwNjaGvb09PvjgA2zevBnNmzd/7rlq1KiBY8eOISwsDJ6enjA1NYWNjQ1atWqFpUuXYvLkyTq5xlfxzjvvYNWqVfD19YW5uTnq1q2L5cuXl/gMAuDxvIGff/4ZNWvWfKW5C0WysrLQr18/CCFgamqKOXPmwMjICJ6envjhhx8APB4WeHLpJJGcKYQo44woIiIiqhTYE0BERCRTTAKIiIhkikkAERGRTDEJICIikikmAURERDLFJICIiEimmAQQERHJVKW8bbBFwzB9h0Ckc6kHovUdApHOVVHq9reqNr8vHh3/TWttlZdKmQQQERGVikLeHeLyvnoiIiIZY08AERHJVxkfU13ZMAkgIiL54nAAERERyRF7AoiISL44HEBERCRTHA4gIiIiOWJPABERyReHA4iIiGSKwwFEREQkR+wJICIi+eJwABERkUxxOICIiIjkiD0BREQkXxwOICIikikOBxAREZEcsSeAiIjki8MBREREMsXhACIiIpIj9gQQEZF8ybwngEkAERHJl5G85wTIOwUiIiKSMfYEEBGRfHE4gIiISKZkvkRQ3ikQERGRjLEngIiI5IvDAURERDLF4QAiIiKSI/YEEBGRfHE4gIiISKY4HEBERERyxJ4AIiKSLw4HEBERyRSHA4iIiEiO2BNARETyxeEAIiIimeJwABEREckRewKIiEi+OBxAREQkUzJPAuR99URERDLGngAiIpIvmU8MZBJARETyxeEAIiIikiP2BBARkXxxOICIiEimOBxAREREcsSeACIiki8OBxAREcmTQuZJAIcDiIiIZIpJABERyZZCodDaVlY3b95Er1694ODgAAsLC/j6+uLIkSPSfiEExo0bB1dXV1hYWMDf3x8XL17UaOP+/fsICgqCjY0NbG1t0b9/f2RmZpY6BiYBREQkXwotbmXw4MEDNG/eHKampti4cSPOnDmDqVOnws7OTqozZcoUREdHIyYmBgcPHoSVlRUCAgKQk5Mj1QkKCsLp06cRHx+PuLg47N69G4MGDSr95QshRNlCN3wWDcP0HQKRzqUeiNZ3CEQ6V0Wp29+qVh/P01pbWcs/K3XdsWPHYt++fdizZ0+J+4UQcHNzw8iRIzFq1CgAQHp6OpydnTF//nx8+umnOHv2LHx8fHD48GE0btwYALBp0yZ06NABN27cgJub2wvjYE8AERHJljaHA3Jzc5GRkaGx5ebmlnjetWvXonHjxvj444/h5OSEhg0bYs6cOdL+xMREpKSkwN/fXypTqVRo0qQJEhISAAAJCQmwtbWVEgAA8Pf3h5GREQ4ePFiq62cSQEREsqXNJCAyMhIqlUpji4yMLPG8V65cwaxZs1CrVi1s3rwZISEhGDp0KBYsWAAASElJAQA4OztrHOfs7CztS0lJgZOTk8Z+ExMT2NvbS3VehEsEiYiItCAiIgIjRozQKFMqlSXWVavVaNy4MSZNmgQAaNiwIf73v/8hJiYGwcHBOo+1CHsCiIhItrTZE6BUKmFjY6OxPSsJcHV1hY+Pj0ZZnTp1kJSUBABwcXEBANy+fVujzu3bt6V9Li4uSE1N1dhfUFCA+/fvS3VehEkAERHJlr6WCDZv3hznz5/XKLtw4QI8PDwAAJ6ennBxccG2bduk/RkZGTh48CD8/PwAAH5+fkhLS8PRo0elOtu3b4darUaTJk1KFQeHA4iIiMpZeHg4mjVrhkmTJqF79+44dOgQZs+ejdmzZwN4nJwMHz4c33//PWrVqgVPT0988803cHNzQ9euXQE87jlo164dBg4ciJiYGOTn5yMsLAyffvppqVYGAEwCiIhIzvR01+C3334bq1atQkREBCZOnAhPT09MmzYNQUFBUp0vv/wSWVlZGDRoENLS0tCiRQts2rQJ5ubmUp3Y2FiEhYXhvffeg5GREQIDAxEdXfrlw7xPAFEFxfsEkBzo+j4BtkGLtNZWWmwvrbVVXjgngIiISKY4HEBERLIl96cIMgkgIiLZknsSwOEAIiIimWJPABERyZbcewKYBBARkXzJOwfgcAAREZFcGUwSsGfPHvTq1Qt+fn64efMmAOCvv/7C3r179RwZERFVVvq6bbChMIgkYOXKlQgICICFhQWOHz8uPX85PT1desISERGRtjEJMADff/89YmJiMGfOHJiamkrlzZs3x7Fjx/QYGRERUeVlEBMDz58/j1atWhUrV6lUSEtLK/+AiIhIFirqL3htMYieABcXF1y6dKlY+d69e1GjRg09RERERLKg0OJWARlEEjBw4EAMGzYMBw8ehEKhwK1btxAbG4tRo0YhJCRE3+ERERFVSgYxHDB27Fio1Wq89957yM7ORqtWraBUKjFq1CgMGTJE3+EREVElJffhAINIAhQKBb766iuMHj0aly5dQmZmJnx8fGBtba3v0IiIqBKTexJgEMMBixYtQnZ2NszMzODj44N33nmHCQAREZGOGUQSEB4eDicnJ/Ts2RMbNmxAYWGhvkMiIiIZ4H0CDEBycjKWLFkChUKB7t27w9XVFaGhodi/f7++QyMiokqMSYABMDExQadOnRAbG4vU1FRERUXh6tWraNOmDWrWrKnv8IiIiColg5gY+CRLS0sEBATgwYMHuHbtGs6ePavvkIiIqLKqmD/gtcZgkoDs7GysWrUKsbGx2LZtG9zd3dGjRw+sWLFC36EREVElVVG78bXFIJKATz/9FHFxcbC0tET37t3xzTffwM/PT99hERERVWoGkQQYGxtj2bJlCAgIgLGxsb7DISIimWBPgAGIjY3VdwhERCRDTAL0JDo6GoMGDYK5uTmio6OfW3fo0KHlFBUREZF86C0JiIqKQlBQEMzNzREVFfXMegqFgkkAERHphrw7AvSXBCQmJpb4NxERUXmR+3CAQdwsaOLEicjOzi5W/ujRI0ycOFEPEREREVV+BpEETJgwAZmZmcXKs7OzMWHCBD1EREREciD32wYbxOoAIUSJb+DJkydhb2+vh4gIANwcVfh+WBd80LwuLM1Ncfn6XXw+fhGOnUkCADjZV8H3w7rA368OVNYW2HvsEkZMWY7LSXekNvp1a45P2jfGm7Vfh421BVxajkZ65iN9XRLRC3Vu9x6Sb90qVv7xJz0w5qtx+GfFMmzaEIfzZ88gKysLO/YeRBUbGz1EStpQUb+8tUWvSYCdnZ2UQb3xxhsa/xiFhYXIzMzE4MGD9RihfNlWscD2+SOw6/BFdA2biTsPMuFVzREPMv5v2GZZ1CDkFxTi4+G/IyMrB0N7tcWGmCFo2O17ZOfkAQAszU0Rv/8M4vefwXdDu+jrcohKbeHi5ShU/9+TTC9fuojQQf3x3gftAAA5jx6hWfOWaNa8JX779Rd9hUmkFXpNAqZNmwYhBPr164cJEyZApVJJ+8zMzFC9enXeOVBPRn72Pm6kPMDn4xdJZddu3ZP+9qrmhCb1PfFW4Pc4eyUFADB00lJc3ToJ3ds3wvxVCQCA3xbvBAC0bFSr/IInegV2T/U+Lpg7B6+7V0Ojxm8DAHr2DgYAHDl8qNxjI+1jT4AeBQc//j+Tp6cnmjVrBlNTU32GQ0/o2NoXW/efReyUfmjRqBZupaZh9rI9mLfq8eOdlWaPPzo5eQXSMUII5OUVoNmbNaUkgKgiy8/Pw4b16xDUu6/svywqLZn/sxrExMDWrVtLCUBOTg4yMjI0tufJzc0tVl880ZVHL8ez6msY+HFLXEq6gw+/mIE5y/di6pcfIahzEwDA+aspSEq+j++GfAjbKhYwNTHGyL7+eN3FDi6vqV7QOlHFsHP7NmQ+fIjOXf6j71CIdMIgkoDs7GyEhYXByckJVlZWsLOz09ieJzIyEiqVSmMruH20nCKvvIyMFDhx7jq+/W0dTp6/gT//2Yd5q/Zj4EctAAAFBWp8OnIOvDyckLz7J9xP+AWtGr+BTXtPQy3Ueo6eSDvWrFqJZs1bwtHJSd+hkI7IfXWAQSQBo0ePxvbt2zFr1iwolUr88ccfmDBhAtzc3LBw4cLnHhsREYH09HSNzcS5UTlFXnml3M2QxvqLnEtMgbvL/yVlx89eR9NPJ8O55Sh4fvAVuoTNhIPKCok37j3dHFGFk3zrJg4dSECXwI/0HQrpkNyTAINYIrhu3TosXLgQ7777Lj777DO0bNkSXl5e8PDwQGxsLIKCgp55rFKphFKp1ChTGPFJhK8q4cQVvOGh+eunVjUnJCXfL1Y3IzMHAFCzmiPe8qmGCTPjyiVGIl1au3oV7Ozt0aJla32HQqQzBtETcP/+fdSoUQMAYGNjg/v3H3/RtGjRArt379ZnaLI1fdF2vOPridH9PkAN99fwSbvG6BfYHL8v/b9/j27+DdGyUS1Ur+qATu/6Yv2sMKzb+S+2HTgn1XF2qIL6b1RFzWqvAQDq1XJD/Teqws7Gstyviai01Go11q35B50+7AoTE83fSnfv3sH5c2dxI+kaAODSxQs4f+4s0tPT9BApvSqFQntbRWQQPQE1atRAYmIiqlWrhtq1a2PZsmV45513sG7dOtja2uo7PFk6eiYJn4ycg4lDPsR/B7XH1Zv3MPqnlViy8YhUx8XRBj+O7AYnhypIuZuB2LiDiJy9SaOdAR+1xNeDO0ivt/4ZDgAYOO4vLFp3sHwuhqiMDh1IQEpyMj7s2q3YvpXLlmJOzAzp9cDPegMAvv1uEicQVkAVtRtfWxRCCKHvIKKiomBsbIyhQ4di69at6Ny5M4QQyM/Pxy+//IJhw4aVqT2LhmE6ipTIcKQeeP4juIkqgypK3XZY1xq96cWVSuniT+201lZ5MYiegPDwcOlvf39/nDt3DkePHoWXlxfq16+vx8iIiKgyk3lHgGEkAU/z8PCAh4eHvsMgIqJKTu7DAQaRBERHl9ytqVAoYG5uDi8vL7Rq1QrGxpz1T0REpC0GkQRERUXhzp07yM7Olm4O9ODBA1haWsLa2hqpqamoUaMGduzYAXd3dz1HS0RElYXMOwIMY4ngpEmT8Pbbb+PixYu4d+8e7t27hwsXLqBJkyb49ddfkZSUBBcXF425A0RERK/KyEihta0iMoiegK+//horV65EzZo1pTIvLy/8/PPPCAwMxJUrVzBlyhQEBgbqMUoiIqLKxSCSgOTkZBQUFBQrLygoQErK41vXurm54eHDh+UdGhERVWIcDjAAbdq0weeff47jx49LZcePH0dISAjatm0LADh16hQ8PT31FSIREVGlYxBJwNy5c2Fvb49GjRpJzwJo3Lgx7O3tMXfuXACAtbU1pk6dqudIiYioMuEDhAyAi4sL4uPjce7cOVy4cAEA4O3tDW9vb6lOmzZt9BUeERFVUhX0u1trDKInoEiNGjXg7e2NDh06aCQARERElcn48eOL9STUrl1b2p+Tk4PQ0FA4ODjA2toagYGBuH37tkYbSUlJ6NixIywtLeHk5ITRo0eXOL/ueQwiCcjOzkb//v1haWmJunXrIikpCQAwZMgQTJ48Wc/RERFRZaXP4YC6desiOTlZ2vbu3SvtCw8Px7p167B8+XLs2rULt27dQrdu//dAq8LCQnTs2BF5eXnYv38/FixYgPnz52PcuHFlisEgkoCIiAicPHkSO3fuhLm5uVTu7++PpUuX6jEyIiKqzPSZBJiYmMDFxUXaXnvt8SPX09PTMXfuXPzyyy9o27YtGjVqhHnz5mH//v04cOAAAGDLli04c+YMFi1ahDfffBPt27fHd999hxkzZiAvL6/UMRhEErB69Wr89ttvaNGihcYbWbduXVy+fFmPkREREZVObm4uMjIyNLbc3Nxn1r948SLc3NxQo0YNBAUFSb3gR48eRX5+Pvz9/aW6tWvXRrVq1ZCQkAAASEhIgK+vL5ydnaU6AQEByMjIwOnTp0sds0EkAXfu3IGTk1Ox8qysrAo745KIiAyfQqG9LTIyEiqVSmOLjIws8bxNmjTB/PnzsWnTJsyaNQuJiYlo2bIlHj58iJSUFJiZmcHW1lbjGGdnZ+neOSkpKRoJQNH+on2lZRCrAxo3boz169djyJAhAP7vqU5//PEH/Pz89BkaERFVYtr8oRkxNgIjRozQKFMqlSXWbd++vfR3/fr10aRJE3h4eGDZsmWwsLDQWkwvYhBJwKRJk9C+fXucOXMGBQUF+PXXX3HmzBns378fu3bt0nd4REREL1R0n5uXYWtrizfeeAOXLl3C+++/j7y8PKSlpWn0Bty+fRsuLi4AHi+tP3TokEYbRasHiuqUhkEMB7Ro0QInTpxAQUEBfH19sWXLFjg5OSEhIQGNGjXSd3hERFRJaXM44FVkZmbi8uXLcHV1RaNGjWBqaopt27ZJ+8+fP4+kpCSpd9zPzw+nTp1CamqqVCc+Ph42Njbw8fEp9XkNoicAAGrWrIk5c+boOwwiIpIRfc07GzVqFDp37gwPDw/cunUL3377LYyNjdGjRw+oVCr0798fI0aMgL29PWxsbDBkyBD4+fmhadOmAIAPPvgAPj4+6N27N6ZMmYKUlBR8/fXXCA0NLVNvhF6TACMjoxf+AygUijLf/ICIiMiQ3bhxAz169MC9e/fg6OiIFi1a4MCBA3B0dAQAREVFwcjICIGBgcjNzUVAQABmzpwpHW9sbIy4uDiEhITAz88PVlZWCA4OxsSJE8sUh0IIIbR6ZWWwZs2aZ+5LSEhAdHQ01Go1cnJyytSuRcOwVw2NyOClHojWdwhEOldFqdtR68bf79BaW0e+rni3t9drT0CXLl2KlZ0/fx5jx47FunXrEBQUVOashoiIqLTkvgzdICYGAsCtW7cwcOBA+Pr6oqCgACdOnMCCBQvg4eGh79CIiIgqJb0nAenp6RgzZgy8vLxw+vRpbNu2DevWrUO9evX0HRoREVVyhrI6QF/0OhwwZcoU/Pjjj3BxccHff/9d4vAAERGRrsh9OECvScDYsWNhYWEBLy8vLFiwAAsWLCix3j///FPOkREREVV+ek0C+vTpI/ssjIiI9EfuX0F6TQLmz5+vz9MTEZHMyf2HqN4nBhIREZF+GMxtg4mIiMqbzDsCmAQQEZF8cTiAiIiIZIk9AUREJFsy7whgEkBERPLF4QAiIiKSJfYEEBGRbMm9J4BJABERyZbMcwAOBxAREckVewKIiEi2OBxAREQkUzLPATgcQEREJFfsCSAiItnicAAREZFMyTwH4HAAERGRXLEngIiIZMtI5l0BWukJSEtL00YzRERE5Uqh0N5WEZU5Cfjxxx+xdOlS6XX37t3h4OCAqlWr4uTJk1oNjoiIiHSnzElATEwM3N3dAQDx8fGIj4/Hxo0b0b59e4wePVrrARIREemKQqHQ2lYRlXlOQEpKipQExMXFoXv37vjggw9QvXp1NGnSROsBEhER6YpRxfzu1poy9wTY2dnh+vXrAIBNmzbB398fACCEQGFhoXajIyIiIp0pc09At27d0LNnT9SqVQv37t1D+/btAQDHjx+Hl5eX1gMkIiLSlYraja8tZU4CoqKiUL16dVy/fh1TpkyBtbU1ACA5ORlffPGF1gMkIiLSFZnnAGVPAkxNTTFq1Khi5eHh4VoJiIiIiMpHqZKAtWvXlrrBDz/88KWDISIiKk8KyLsroFRJQNeuXUvVmEKh4ORAIiKqMOS+OqBUSYBardZ1HERERFTOXunZATk5OTA3N9dWLEREROVK7qsDynyfgMLCQnz33XeoWrUqrK2tceXKFQDAN998g7lz52o9QCIiIl3hswPK6IcffsD8+fMxZcoUmJmZSeX16tXDH3/8odXgiIiISHfKnAQsXLgQs2fPRlBQEIyNjaXyBg0a4Ny5c1oNjoiISJeMFAqtbRVRmecE3Lx5s8Q7A6rVauTn52slKCIiovJQQb+7tabMPQE+Pj7Ys2dPsfIVK1agYcOGWgmKiIiIdK/MPQHjxo1DcHAwbt68CbVajX/++Qfnz5/HwoULERcXp4sYiYiIdIKrA8qoS5cuWLduHbZu3QorKyuMGzcOZ8+exbp16/D+++/rIkYiIiKdkPvqgJe6T0DLli0RHx+v7ViIiIioHL30zYKOHDmCs2fPAng8T6BRo0ZaC4qIiKg8VNRZ/dpS5iTgxo0b6NGjB/bt2wdbW1sAQFpaGpo1a4YlS5bg9ddf13aMREREOiHvFOAl5gQMGDAA+fn5OHv2LO7fv4/79+/j7NmzUKvVGDBggC5iJCIiIh0oc0/Arl27sH//fnh7e0tl3t7emD59Olq2bKnV4IiIiHRJ7qsDypwEuLu7l3hToMLCQri5uWklKCIiovIg90cJl3k44KeffsKQIUNw5MgRqezIkSMYNmwYfv75Z60GR0REVNlNnjwZCoUCw4cPl8pycnIQGhoKBwcHWFtbIzAwELdv39Y4LikpCR07doSlpSWcnJwwevRoFBQUlOncpeoJsLOz0+gyycrKQpMmTWBi8vjwgoICmJiYoF+/fujatWuZAiAiItIXfQ8HHD58GL///jvq16+vUR4eHo7169dj+fLlUKlUCAsLQ7du3bBv3z4Aj3vfO3bsCBcXF+zfvx/Jycno06cPTE1NMWnSpFKfv1RJwLRp00p/RURERBWEPnOAzMxMBAUFYc6cOfj++++l8vT0dMydOxeLFy9G27ZtAQDz5s1DnTp1cODAATRt2hRbtmzBmTNnsHXrVjg7O+PNN9/Ed999hzFjxmD8+PEaT/l9nlIlAcHBwS9xeURERPKRm5uL3NxcjTKlUgmlUlli/dDQUHTs2BH+/v4aScDRo0eRn58Pf39/qax27dqoVq0aEhIS0LRpUyQkJMDX1xfOzs5SnYCAAISEhOD06dOlfpZPmecEPCknJwcZGRkaGxERUUWhUCi0tkVGRkKlUmlskZGRJZ53yZIlOHbsWIn7U1JSYGZmJt2Lp4izszNSUlKkOk8mAEX7i/aVVplXB2RlZWHMmDFYtmwZ7t27V2x/YWFhWZskIiLSC22uDoiIiMCIESM0ykrqBbh+/TqGDRuG+Ph4mJubay+Al1DmnoAvv/wS27dvx6xZs6BUKvHHH39gwoQJcHNzw8KFC3URIxERkcFTKpWwsbHR2EpKAo4ePYrU1FS89dZbMDExgYmJCXbt2oXo6GiYmJjA2dkZeXl5SEtL0zju9u3bcHFxAQC4uLgUWy1Q9LqoTmmUOQlYt24dZs6cicDAQJiYmKBly5b4+uuvMWnSJMTGxpa1OSIiIr3R5nBAab333ns4deoUTpw4IW2NGzdGUFCQ9LepqSm2bdsmHXP+/HkkJSXBz88PAODn54dTp04hNTVVqhMfHw8bGxv4+PiUOpYyDwfcv38fNWrUAADY2Njg/v37AIAWLVogJCSkrM0RERHpjT4WB1SpUgX16tXTKLOysoKDg4NU3r9/f4wYMQL29vawsbHBkCFD4Ofnh6ZNmwIAPvjgA/j4+KB3796YMmUKUlJS8PXXXyM0NPSZExFLUuaegBo1aiAxMRHA49mKy5YtA/C4h+DpSQxERERUdlFRUejUqRMCAwPRqlUruLi44J9//pH2GxsbIy4uDsbGxvDz80OvXr3Qp08fTJw4sUznUQghRFkDMzY2xtChQ7F161Z07twZQgjk5+fjl19+wbBhw8oUgC5YNAzTdwhEOpd6IFrfIRDpXBXlKy1ie6EBS/+ntbb++KTeiysZmDIPB4SHh0t/+/v749y5czh69Ci8vLyK3fGIiIjIkMn8+UGvdp8AAPDw8EC3bt1gb2+PQYMGaSMmIiIiKgda62e5d+8e5s6dq63miIiIdE4fqwMMSZmHA4iIiCqLCvrdrTW6nXFBREREBos9AUREJFtGMu8KKHUS0K1bt+fuf/r2hkRERIZO5jlA6ZMAlUr1wv19+vR55YCIiIiofJQ6CZg3b54u4yAiIip3FXVWv7ZUyjkBDw7/pu8QiHTOrmn4iysRVXCPjkTptH25z46X+/UTERHJVqXsCSAiIioNDgcQERHJlJG8cwAOBxAREclVqXoC1q5dW+oGP/zww5cOhoiIqDzJvSegVElA165dS9WYQqFAYWHhq8RDRERUbjgnoBTUarWu4yAiIqJyxomBREQkWxwOeAlZWVnYtWsXkpKSkJeXp7Fv6NChWgmMiIhI12Q+GlD2JOD48ePo0KEDsrOzkZWVBXt7e9y9exeWlpZwcnJiEkBERFRBlHmJYHh4ODp37owHDx7AwsICBw4cwLVr19CoUSP8/PPPuoiRiIhIJ4wUCq1tFVGZk4ATJ05g5MiRMDIygrGxMXJzc+Hu7o4pU6bgv//9ry5iJCIi0gkjLW4VUZnjNjU1hZHR48OcnJyQlJQE4PGjhK9fv67d6IiIiEhnyjwnoGHDhjh8+DBq1aqF1q1bY9y4cbh79y7++usv1KtXTxcxEhER6UQF7cXXmjL3BEyaNAmurq4AgB9++AF2dnYICQnBnTt3MHv2bK0HSEREpCtynxNQ5p6Axo0bS387OTlh06ZNWg2IiIiIygdvFkRERLJVQX/Aa02ZkwBPT8/n3mv5ypUrrxQQERFReeEdA8to+PDhGq/z8/Nx/PhxbNq0CaNHj9ZWXERERKRjZU4Chg0bVmL5jBkzcOTIkVcOiIiIqLxU1Al92qK1+xu0b98eK1eu1FZzREREOqdQaG+riLSWBKxYsQL29vbaao6IiIh07KVuFvTkxEAhBFJSUnDnzh3MnDlTq8ERERHpEicGllGXLl00kgAjIyM4Ojri3XffRe3atbUaHBERkS4pIO8soMxJwPjx43UQBhEREZW3Ms8JMDY2RmpqarHye/fuwdjYWCtBERERlQcjhfa2iqjMPQFCiBLLc3NzYWZm9soBERERlZeK+uWtLaVOAqKjowEACoUCf/zxB6ytraV9hYWF2L17N+cEEBERVSClTgKioqIAPO4JiImJ0ej6NzMzQ/Xq1RETE6P9CImIiHTkebfBl4NSJwGJiYkAgDZt2uCff/6BnZ2dzoIiIiIqDxwOKKMdO3boIg4iIiIqZ2VeHRAYGIgff/yxWPmUKVPw8ccfayUoIiKi8sDbBpfR7t270aFDh2Ll7du3x+7du7USFBERUXkwUii0tlVEZU4CMjMzS1wKaGpqioyMDK0ERURERLpX5iTA19cXS5cuLVa+ZMkS+Pj4aCUoIiKi8sCbBZXRN998g27duuHy5cto27YtAGDbtm34+++/sXz5cq0HSEREpCsVtBdfa8qcBHTu3BmrV6/GpEmTsGLFClhYWKB+/frYunUrWrdurYsYiYiISAfKnAQAQMeOHdGxY8di5f/73/9Qr169Vw6KiIioPBjJ/CmCZZ4T8LSHDx9i9uzZeOedd9CgQQNtxERERFQuuETwJe3evRt9+vSBq6srfv75Z7Rt2xYHDhzQZmxERESkQ2VKAlJSUjB58mTUqlULH3/8MVQqFXJzc7F69WpMnjwZb7/9tq7iJCIi0jp9rQ6YNWsW6tevDxsbG9jY2MDPzw8bN26U9ufk5CA0NBQODg6wtrZGYGAgbt++rdFGUlISOnbsCEtLSzg5OWH06NEoKCgo2/WXtmLnzp3h7e2Nf//9F9OmTcOtW7cwffr0Mp2MiIjIkOjrZkGvv/46Jk+ejKNHj+LIkSNo27YtunTpgtOnTwMAwsPDsW7dOixfvhy7du3CrVu30K1bN+n4wsJCdOzYEXl5edi/fz8WLFiA+fPnY9y4cWWKQyGEEKWpaGJigqFDhyIkJAS1atWSyk1NTXHy5EmDukdATtkSIaIKya5puL5DINK5R0eidNr+7APXtNbWoKYer3S8vb09fvrpJ3z00UdwdHTE4sWL8dFHHwEAzp07hzp16iAhIQFNmzbFxo0b0alTJ9y6dQvOzs4AgJiYGIwZMwZ37twp8aZ+JSl1T8DevXvx8OFDNGrUCE2aNMFvv/2Gu3fvvsRlEhERGQZtTgzMzc1FRkaGxpabm/vCGAoLC7FkyRJkZWXBz88PR48eRX5+Pvz9/aU6tWvXRrVq1ZCQkAAASEhIgK+vr5QAAEBAQAAyMjKk3oTSKHUS0LRpU8yZMwfJycn4/PPPsWTJEri5uUGtViM+Ph4PHz4s9UmJiIgMgTaHAyIjI6FSqTS2yMjIZ5771KlTsLa2hlKpxODBg7Fq1Sr4+PggJSUFZmZmsLW11ajv7OyMlJQUAI/n6D2ZABTtL9pX6usvdc3/z8rKCv369cPevXtx6tQpjBw5EpMnT4aTkxM+/PDDsjZHRERUKURERCA9PV1ji4iIeGZ9b29vnDhxAgcPHkRISAiCg4Nx5syZcoz4Fe8T4O3tjSlTpuDGjRv4+++/tRUTERFRudDmcIBSqZRm+xdtSqXymec2MzODl5cXGjVqhMjISDRo0AC//vorXFxckJeXh7S0NI36t2/fhouLCwDAxcWl2GqBotdFdUrjlW8WBADGxsbo2rUr1q5dq43miIiIyoWRFrdXpVarkZubi0aNGsHU1BTbtm2T9p0/fx5JSUnw8/MDAPj5+eHUqVNITU2V6sTHx8PGxqZME/Vf6rbBRERE9PIiIiLQvn17VKtWDQ8fPsTixYuxc+dObN68GSqVCv3798eIESNgb28PGxsbDBkyBH5+fmjatCkA4IMPPoCPjw969+6NKVOmICUlBV9//TVCQ0Of2/vwNCYBREQkWwo93e83NTUVffr0QXJyMlQqFerXr4/Nmzfj/fffBwBERUXByMgIgYGByM3NRUBAAGbOnCkdb2xsjLi4OISEhMDPzw9WVlYIDg7GxIkTyxRHqe8TUJHwPgEkB7xPAMmBru8TsPDIda211aexu9baKi9amRNAREREFQ+HA4iISLbKervfyoZJABERyZa8UwAOBxAREckWewKIiEi2ZD4awCSAiIjkS19LBA0FhwOIiIhkij0BREQkW3L/JcwkgIiIZIvDAURERCRL7AkgIiLZknc/AJMAIiKSMQ4HEBERkSyxJ4CIiGRL7r+EmQQQEZFscTiAiIiIZIk9AUREJFvy7gdgEkBERDIm89EADgcQERHJFXsCiIhItoxkPiDAJICIiGSLwwFEREQkSwaTBOzZswe9evWCn58fbt68CQD466+/sHfvXj1HRkRElZVCi/+riAwiCVi5ciUCAgJgYWGB48ePIzc3FwCQnp6OSZMm6Tk6IiKqrBQK7W0VkUEkAd9//z1iYmIwZ84cmJqaSuXNmzfHsWPH9BgZERFR5WUQEwPPnz+PVq1aFStXqVRIS0sr/4CIiEgW5L46wCB6AlxcXHDp0qVi5Xv37kWNGjX0EBEREckBhwMMwMCBAzFs2DAcPHgQCoUCt27dQmxsLEaNGoWQkBB9h0dERFQpGcRwwNixY6FWq/Hee+8hOzsbrVq1glKpxKhRozBkyBB9h0dERJVURf0Fry0KIYTQdxBF8vLycOnSJWRmZsLHxwfW1tYv1U5OgZYDIzJAdk3D9R0Ckc49OhKl0/bjz97VWlvv13lNa22VF4MYDli0aBGys7NhZmYGHx8fvPPOOy+dABAREVHpGEQSEB4eDicnJ/Ts2RMbNmxAYWGhvkMiIiIZMFJob6uIDCIJSE5OxpIlS6BQKNC9e3e4uroiNDQU+/fv13doRERUifGOgQbAxMQEnTp1QmxsLFJTUxEVFYWrV6+iTZs2qFmzpr7DIyIiqpQMYnXAkywtLREQEIAHDx7g2rVrOHv2rL5DIiKiSkruqwMMoicAALKzsxEbG4sOHTqgatWqmDZtGv7zn//g9OnT+g6NiIgqKbkPBxhET8Cnn36KuLg4WFpaonv37vjmm2/g5+en77CIiIgqNYNIAoyNjbFs2TIEBATA2NhY3+EQEZFMVNRZ/dpiEElAbGysvkMgIiIZqqjd+NqityQgOjoagwYNgrm5OaKjo59bd+jQoeUUFT3PrBnTETPzN42y6p6eWBO3SXp98sRxTP81CqdO/QtjIyN4166DWbPnwtzcvLzDJSoVN0cVvh/SCR80qwNLc1NcvnEXn09YgmNnrwMArCzM8P2QTujc2hf2KktcvXUfM5fuwR8rHy9htrOxxDeft8N7Tb3h7myLu2lZWLfzFCbM2oiMrBx9XhrRC+ktCYiKikJQUBDMzc0RFfXs20IqFAomAQakplctzP5jnvTa2OT/hm9OnjiOLz4fgH4DPsfYr76BibExzp8/ByMjg5l/SqTBtooFts8dil1HLqLrsNm48yATXu6OeJCRLdX5Mbwr3n3bC5+NW4Rrt+7Dv2lt/DomEMl30rF+92m4OtrA1dEGEdPW4uyVFFRztcP0iI/h6qhCzzHz9XdxVCpyXx2gtyQgMTGxxL/JsJkYG+M1R8cS9/30YyR6BPVG/4GDpLLqnnwUNBmukcHv4cbtNHw+cYlUdu3WfY06TRtUx6K4w9hz9DIA4M9VCejfzQ+N61bD+t2nceZyCnp8OV+qn3jzHsbP3IA/v+sFY2MjFBaqy+Va6OXIPAcwjCWCEydORHZ2drHyR48eYeLEiXqIiJ7lWtI1+L/bAh0C3kPElyORfOsWAODevXs49e9J2Ds4oE/Qp2jTqhn6BffCsaNH9Bwx0bN1bFUXx85eR+zkYFzbMhEJsSPxWdemGnUOnLyKTq3qwc1RBQBo1cgLtao5YuuB889s18baHBlZOUwAyOAZxFMEjY2NkZycDCcnJ43ye/fuwcnJ6bnPEsjNzUVubq5GmTBWQqlU6iRWOdu7Zxeys7NRvbon7ty5g99nzUDq7dtYuWYdLl+6hN49P4FKZYsRo7+Ed+06iFuzGkuXLMbKNXHw8Kiu7/ArHT5F8NU92DcFABAduxP/bDuJRj7V8PPIrhgSuQKx6w8DAMxMjTHjq0/Qq9PbyC8ohFot8MUPS7F4fckJroPKCvsXjcDfG49i/MwN5XYtlZWunyKYcClNa235edlqra3yYhCrA4QQUJQwMHPy5EnY29s/99jIyEhMmDBBo+yrb77F1+PGazNEAtCiZWvp7ze8a8O3fgO0f78NNm/aiBo1Ht/e+aPun6DrfwIBAHXq+ODgwQSs/mclhoWP1EvMRM9jZKTAsTPX8e3//7I+ef4m6tZ0wcDAZlIS8MUnLfGOrwcCw/9AUvJ9tHirJqZ9GYjkOxnYceiCRntVrJRY9etAnL1yG9//vqnY+cjwyH04QK9JgJ2dHRQKBRQKBd544w2NRKCwsBCZmZkYPHjwc9uIiIjAiBEjNMqEMXsByoONjQ08PKrjelIS3mnyuAu1xlPPevCsURMpybf0ER7RC6XczcDZxNsaZecSb6Nr2/oAAHOlKSaEdsQno+Zh074zAID/XUpG/TeqYnivdzWSAGtLJdZGf46HWbn4ZPSfKOBQAFUAek0Cpk2bBiEE+vXrhwkTJkClUkn7zMzMUL169RfeOVCpLN71n1Ogk3DpKdlZWbh+/To6fuiIqlVfh6OTE64+Ncnz2tWraNGylZ4iJHq+hJOJeMNDcxiylocTkpIfAABMTYxgZmoCtdD8Qi9UqzVWvVSxUmLd9MHIzS/ARyP+QG4e/yNUYci8K0CvSUBwcDAAwNPTE82aNYOpqak+w6EXmPrTj2j9bhu4urnhTmoqZs2YDmNjI7Tv0AkKhQJ9P+uPWTOmw9u7Nrxr18HaNatwNfEKpkY9/z4QRPoyffEu7PhzGEZ/5o+V8Sfwdt1q6Pefpgj7YRkA4GFWLnYfvYRJwz7Eo9x8JCU/QMu3aiKoQ2OMiVoD4HECEPfbYFiYm+GzbxbBxtocNtaP74tx50Em1Gq9T7ui55D7zYL0NjEwIyMDNjY20t/PU1SvtNgToBtfjgrHsSOHkZaWBjt7ezR8qxGGDA2He7VqUp25c2Zj6ZJYpKenw9u7NoaPGIW3GjXWY9SVFycGakf7Fj6YGNYRXu6OuHrrPqJjd2Le6gPSfmeHKpgY2hH+Tb1hZ2OJpJQH+HNVAqJjdwEAWjaqiS2/h5XYtnfniVKvAr0cXU8MPHg5XWttNampenElA6O3JODJFQFGRkYlTgwsmjD4vNUBJWESQHLAJIDkQNdJwKEr2ksC3qlR8ZIAvd0nYPv27dLM/x07dmD79u3FtqJyIiIiXVBocSuLyMhIvP3226hSpQqcnJzQtWtXnD+vee+JnJwchIaGwsHBAdbW1ggMDMTt25oTWZOSktCxY0dYWlrCyckJo0ePRkFB6X8J621OQOvWrUv8m4iIqLLbtWsXQkND8fbbb6OgoAD//e9/8cEHH+DMmTOwsrICAISHh2P9+vVYvnw5VCoVwsLC0K1bN+zbtw/A41V0HTt2hIuLC/bv34/k5GT06dMHpqammDRpUqniMIibBW3atAnW1tZo0aIFAGDGjBmYM2cOfHx8MGPGDNjZ2ZWpPQ4HkBxwOIDkQNfDAYcTtTccUN/NvNjN60pawVaSO3fuwMnJCbt27UKrVq2Qnp4OR0dHLF68GB999BEA4Ny5c6hTpw4SEhLQtGlTbNy4EZ06dcKtW7fg7OwMAIiJicGYMWNw584dmJmZvfC8BnHb4NGjR0uTA0+dOoURI0agQ4cOSExMLHYPACIiIm1RaPF/kZGRUKlUGltkZGSp4khPf5yMFA2THz16FPn5+fD395fq1K5dG9WqVUNCQgIAICEhAb6+vlICAAABAQHIyMjA6dOnS3Veg7hjYGJiInx8fAAAK1euROfOnTFp0iQcO3YMHTp00HN0REREL1bSzetK0wugVqsxfPhwNG/eHPXq1QMApKSkwMzMDLa2thp1nZ2dkZKSItV5MgEo2l+0rzQMIgkwMzOTHiC0detW9OnTB8DjjOhFyweJiIheljYfJVzarv+nhYaG4n//+x/27t2rvWBKySCSgBYtWmDEiBFo3rw5Dh06hKVLlwIALly4gNdff13P0REREelGWFgY4uLisHv3bo3vOxcXF+Tl5SEtLU2jN+D27dtwcXGR6hw6dEijvaLVA0V1XsQg5gT89ttvMDExwYoVKzBr1ixUrVoVALBx40a0a9dOz9EREVFlpa8lgkIIhIWFYdWqVdi+fTs8PT019jdq1AimpqbYtm2bVHb+/HkkJSVJt9P38/PDqVOnkJqaKtWJj4+HjY2NNMT+wus3hNUB2sbVASQHXB1AcqDr1QHHrmlvyPktj9Lf3faLL77A4sWLsWbNGnh7e0vlKpUKFhYWAICQkBBs2LAB8+fPh42NDYYMGQIA2L9/P4DHSwTffPNNuLm5YcqUKUhJSUHv3r0xYMCAUi8RNIjhAODxxaxevRpnz54FANStWxcffvghjI2N9RwZERGRds2aNQsA8O6772qUz5s3D3379gUAREVFwcjICIGBgcjNzUVAQABmzpwp1TU2NkZcXBxCQkLg5+cHKysrBAcHY+LEiaWOwyB6Ai5duoQOHTrg5s2bUkZ0/vx5uLu7Y/369aj51ONpX4Q9ASQH7AkgOdB1T8Dxaw+11lZDjypaa6u8GMScgKFDh6JmzZq4fv06jh07hmPHjiEpKQmenp4YOnSovsMjIqJKSqHQ3lYRGcRwwK5du3DgwAHpJgkA4ODggMmTJ6N58+Z6jIyIiKjyMogkQKlU4uHD4l0ymZmZpbrtIRER0cuooD/gtcYghgM6deqEQYMG4eDBgxBCQAiBAwcOYPDgwfjwww/1HR4REVVW+lojaCAMIgmIjo6Gl5cXmjVrBnNzc5ibm6N58+bw8vLCr7/+qu/wiIiIKiW9Dgeo1Wr89NNPWLt2LfLy8tC1a1cEBwdDoVCgTp068PLy0md4RERUySkq6k94LdFrEvDDDz9g/Pjx8Pf3h4WFBTZs2ACVSoU///xTn2EREZFMVNRZ/dqi1+GAhQsXYubMmdi8eTNWr16NdevWITY2Fmq1Wp9hERERyYJek4CkpCSNRwX7+/tDoVDg1q1beoyKiIjkQubzAvU7HFBQUABzc3ONMlNTU+Tn5+spIiIikpWK+u2tJXpNAoQQ6Nu3r8bzl3NycjB48GBYWVlJZf/8848+wiMiIqrU9JoEBAcHFyvr1auXHiIhIiI54uoAPZo3b54+T09ERDLH1QFEREQkSwbx7AAiIiJ9kHlHAJMAIiKSMZlnARwOICIikin2BBARkWxxdQAREZFMcXUAERERyRJ7AoiISLZk3hHAJICIiGRM5lkAhwOIiIhkij0BREQkW1wdQEREJFNcHUBERESyxJ4AIiKSLZl3BDAJICIiGZN5FsDhACIiIpliTwAREckWVwcQERHJFFcHEBERkSyxJ4CIiGRL5h0BTAKIiEjGZJ4FcDiAiIhIptgTQEREssXVAURERDLF1QFEREQkS+wJICIi2ZJ5RwCTACIiki8OBxAREZEssSeAiIhkTN5dAUwCiIhItjgcQERERLLEngAiIpItmXcEMAkgIiL54nAAERERyRJ7AoiISLb47AAiIiK5kncOwOEAIiKi8rZ792507twZbm5uUCgUWL16tcZ+IQTGjRsHV1dXWFhYwN/fHxcvXtSoc//+fQQFBcHGxga2trbo378/MjMzyxQHkwAiIpIthRa3ssjKykKDBg0wY8aMEvdPmTIF0dHRiImJwcGDB2FlZYWAgADk5ORIdYKCgnD69GnEx8cjLi4Ou3fvxqBBg8oUh0IIIcoYu8HLKdB3BES6Z9c0XN8hEOncoyNROm0/9WG+1tpyqmL6UscpFAqsWrUKXbt2BfC4F8DNzQ0jR47EqFGjAADp6elwdnbG/Pnz8emnn+Ls2bPw8fHB4cOH0bhxYwDApk2b0KFDB9y4cQNubm6lOjd7AoiIiLQgNzcXGRkZGltubm6Z20lMTERKSgr8/f2lMpVKhSZNmiAhIQEAkJCQAFtbWykBAAB/f38YGRnh4MGDpT4XkwAiIpIthRb/FxkZCZVKpbFFRkaWOaaUlBQAgLOzs0a5s7OztC8lJQVOTk4a+01MTGBvby/VKQ2uDiAiIvnS4uqAiIgIjBgxQqNMqVRq7wQ6wCSAiIhIC5RKpVa+9F1cXAAAt2/fhqurq1R++/ZtvPnmm1Kd1NRUjeMKCgpw//596fjS4HAAERHJlr5WBzyPp6cnXFxcsG3bNqksIyMDBw8ehJ+fHwDAz88PaWlpOHr0qFRn+/btUKvVaNKkSanPxZ4AIiKSLX09OyAzMxOXLl2SXicmJuLEiROwt7dHtWrVMHz4cHz//feoVasWPD098c0338DNzU1aQVCnTh20a9cOAwcORExMDPLz8xEWFoZPP/201CsDACYBRERE5e7IkSNo06aN9LpoLkFwcDDmz5+PL7/8EllZWRg0aBDS0tLQokULbNq0Cebm5tIxsbGxCAsLw3vvvQcjIyMEBgYiOjq6THHwPgFEFRTvE0ByoOv7BNzPKtRaW/ZWxlprq7ywJ4CIiGSLjxImIiIiWWISQEREJFMcDiAiItnicAARERHJEnsCiIhIthRavc1PxcMkgIiIZIvDAURERCRL7AkgIiLZknlHAJMAIiKSMZlnARwOICIikin2BBARkWxxdQAREZFMcXUAERERyRJ7AoiISLZk3hHAJICIiGRM5lkAhwOIiIhkij0BREQkW1wdQEREJFNcHUBERESypBBCCH0HQRVbbm4uIiMjERERAaVSqe9wiHSCn3OqjJgE0CvLyMiASqVCeno6bGxs9B0OkU7wc06VEYcDiIiIZIpJABERkUwxCSAiIpIpJgH0ypRKJb799ltOlqJKjZ9zqow4MZCIiEim2BNAREQkU0wCiIiIZIpJABERkUwxCaByV716dUybNk3fYRCVys6dO6FQKJCWlvbcevxcU0XEJKCS6du3LxQKBSZPnqxRvnr1aijK+UkZ8+fPh62tbbHyw4cPY9CgQeUaC1V+RZ99hUIBMzMzeHl5YeLEiSgoKHildps1a4bk5GSoVCoA/FxT5cIkoBIyNzfHjz/+iAcPHug7lBI5OjrC0tJS32FQJdSuXTskJyfj4sWLGDlyJMaPH4+ffvrpldo0MzODi4vLC5Nofq6pImISUAn5+/vDxcUFkZGRz6yzd+9etGzZEhYWFnB3d8fQoUORlZUl7U9OTkbHjh1hYWEBT09PLF68uFh35y+//AJfX19YWVnB3d0dX3zxBTIzMwE87kL97LPPkJ6eLv06Gz9+PADNbtOePXvik08+0YgtPz8fr732GhYuXAgAUKvViIyMhKenJywsLNCgQQOsWLFCC+8UVTZKpRIuLi7w8PBASEgI/P39sXbtWjx48AB9+vSBnZ0dLC0t0b59e1y8eFE67tq1a+jcuTPs7OxgZWWFunXrYsOGDQA0hwP4uabKhklAJWRsbIxJkyZh+vTpuHHjRrH9ly9fRrt27RAYGIh///0XS5cuxd69exEWFibV6dOnD27duoWdO3di5cqVmD17NlJTUzXaMTIyQnR0NE6fPo0FCxZg+/bt+PLLLwE87kKdNm0abGxskJycjOTkZIwaNapYLEFBQVi3bp2UPADA5s2bkZ2djf/85z8AgMjISCxcuBAxMTE4ffo0wsPD0atXL+zatUsr7xdVXhYWFsjLy0Pfvn1x5MgRrF27FgkJCRBCoEOHDsjPzwcAhIaGIjc3F7t378apU6fw448/wtraulh7/FxTpSOoUgkODhZdunQRQgjRtGlT0a9fPyGEEKtWrRJF/9z9+/cXgwYN0jhuz549wsjISDx69EicPXtWABCHDx+W9l+8eFEAEFFRUc889/Lly4WDg4P0et68eUKlUhWr5+HhIbWTn58vXnvtNbFw4UJpf48ePcQnn3wihBAiJydHWFpaiv3792u00b9/f9GjR4/nvxkkK09+9tVqtYiPjxdKpVJ07dpVABD79u2T6t69e1dYWFiIZcuWCSGE8PX1FePHjy+x3R07dggA4sGDB0IIfq6pcjHRawZCOvXjjz+ibdu2xX6pnDx5Ev/++y9iY2OlMiEE1Go1EhMTceHCBZiYmOCtt96S9nt5ecHOzk6jna1btyIyMhLnzp1DRkYGCgoKkJOTg+zs7FKPjZqYmKB79+6IjY1F7969kZWVhTVr1mDJkiUAgEuXLiE7Oxvvv/++xnF5eXlo2LBhmd4Pqvzi4uJgbW2N/Px8qNVq9OzZE926dUNcXByaNGki1XNwcIC3tzfOnj0LABg6dChCQkKwZcsW+Pv7IzAwEPXr13/pOPi5poqCSUAl1qpVKwQEBCAiIgJ9+/aVyjMzM/H5559j6NChxY6pVq0aLly48MK2r169ik6dOiEkJAQ//PAD7O3tsXfvXvTv3x95eXllmiAVFBSE1q1bIzU1FfHx8bCwsEC7du2kWAFg/fr1qFq1qsZxvIc7Pa1NmzaYNWsWzMzM4ObmBhMTE6xdu/aFxw0YMAABAQFYv349tmzZgsjISEydOhVDhgx56Vj4uaaKgElAJTd58mS8+eab8Pb2lsreeustnDlzBl5eXiUe4+3tjYKCAhw/fhyNGjUC8PiXy5OrDY4ePQq1Wo2pU6fCyOjx1JJly5ZptGNmZobCwsIXxtisWTO4u7tj6dKl2LhxIz7++GOYmpoCAHx8fKBUKpGUlITWrVuX7eJJdqysrIp9ruvUqYOCggIcPHgQzZo1AwDcu3cP58+fh4+Pj1TP3d0dgwcPxuDBgxEREYE5c+aUmATwc02VCZOASs7X1xdBQUGIjo6WysaMGYOmTZsiLCwMAwYMgJWVFc6cOYP4+Hj89ttvqF27Nvz9/TFo0CDMmjULpqamGDlyJCwsLKRlUl5eXsjPz8f06dPRuXNn7Nu3DzExMRrnrl69OjIzM7Ft2zY0aNAAlpaWz+wh6NmzJ2JiYnDhwgXs2LFDKq9SpQpGjRqF8PBwqNVqtGjRAunp6di3bx9sbGwQHBysg3eNKpNatWqhS5cuGDhwIH7//XdUqVIFY8eORdWqVdGlSxcAwPDhw9G+fXu88cYbePDgAXbs2IE6deqU2B4/11Sp6HtSAmnXk5OjiiQmJgozMzPx5D/3oUOHxPvvvy+sra2FlZWVqF+/vvjhhx+k/bdu3RLt27cXSqVSeHh4iMWLFwsnJycRExMj1fnll1+Eq6ursLCwEAEBAWLhwoUaE6iEEGLw4MHCwcFBABDffvutEEJzAlWRM2fOCADCw8NDqNVqjX1qtVpMmzZNeHt7C1NTU+Ho6CgCAgLErl27Xu3NokqlpM9+kfv374vevXsLlUolfV4vXLgg7Q8LCxM1a9YUSqVSODo6it69e4u7d+8KIYpPDBSCn2uqPPgoYSqVGzduwN3dHVu3bsV7772n73CIiEgLmARQibZv347MzEz4+voiOTkZX375JW7evIkLFy5I45pERFSxcU4AlSg/Px///e9/ceXKFVSpUgXNmjVDbGwsEwAiokqEPQFEREQyxdsGExERyRSTACIiIpliEkBERCRTTAKIiIhkikkAERGRTDEJINKCvn37omvXrtLrd999F8OHDy/3OHbu3AmFQoG0tDSdnePpa30Z5REnEb0YkwCqtPr27QuFQgGFQgEzMzN4eXlh4sSJKCgo0Pm5//nnH3z33XelqlveX4jVq1fHtGnTyuVcRGTYeLMgqtTatWuHefPmITc3Fxs2bEBoaChMTU0RERFRrG5eXh7MzMy0cl57e3uttENEpEvsCaBKTalUwsXFBR4eHggJCYG/v7/0fPmibu0ffvgBbm5u0uOWr1+/ju7du8PW1hb29vbo0qULrl69KrVZWFiIESNGwNbWFg4ODvjyyy/x9D23nh4OyM3NxZgxY+Du7g6lUgkvLy/MnTsXV69eRZs2bQAAdnZ2UCgU6Nu3LwBArVYjMjISnp6esLCwQIMGDbBixQqN82zYsAFvvPEGLCws0KZNG404X0ZhYSH69+8vndPb2xu//vpriXUnTJgAR0dH2NjYYPDgwcjLy5P2lSb2J127dg2dO3eGnZ0drKysULduXWzYsOGVroWIXow9ASQrFhYWuHfvnvR627ZtsLGxQXx8PIDHt0sOCAiAn58f9uzZAxMTE3z//fdo164d/v33X5iZmWHq1KmYP38+/vzzT9SpUwdTp07FqlWr0LZt22eet0+fPkhISEB0dDQaNGiAxMRE3L17F+7u7li5ciUCAwNx/vx52NjYwMLCAgAQGRmJRYsWISYmBrVq1cLu3bvRq1cvODo6onXr1rh+/Tq6deuG0NBQDBo0CEeOHMHIkSNf6f1Rq9V4/fXXsXz5cjg4OGD//v0YNGgQXF1d0b17d433zdzcHDt37sTVq1fx2WefwcHBAT/88EOpYn9aaGgo8vLysHv3bunR1tbW1q90LURUCnp8giGRTj35aFm1Wi3i4+OFUqkUo0aNkvY7OzuL3Nxc6Zi//vpLeHt7azz2NTc3V1hYWIjNmzcLIYRwdXUVU6ZMkfbn5+eL119/XeMxtq1btxbDhg0TQghx/vx5AUDEx8eXGGdJj6rNyckRlpaWYv/+/Rp1+/fvL3r06CGEECIiIkL4+Pho7B8zZkyxtp5W0iNvnyc0NFQEBgZKr4ODg4W9vb3IysqSymbNmiWsra1FYWFhqWJ/+pp9fX3F+PHjSx0TEWkHewKoUouLi4O1tTXy8/OhVqvRs2dPjB8/Xtrv6+urMQ/g5MmTuHTpEqpUqaLRTk5ODi5fvoz09HQkJyejSZMm0j4TExM0bty42JBAkRMnTsDY2LjEX8DPcunSJWRnZ+P999/XKM/Ly0PDhg0BAGfPntWIAwD8/PxKfY5nmTFjBv78808kJSXh0aNHyMvLw5tvvqlRp0GDBrC0tNQ4b2ZmJq5fv47MzMwXxv60oUOHIiQkBFu2bIG/vz8CAwNRv379V74WIno+JgFUqbVp0wazZs2CmZkZ3NzcYGKi+ZG3srLSeJ2ZmYlGjRohNja2WFuOjo4vFUNR935ZZGZmAgDWr1+PqlWrauxTKpUvFUdpLFmyBKNGjcLUqVPh5+eHKlWq4KeffsLBgwdL3cbLxD5gwAAEBARg/fr12LJlCyIjIzF16lQMGTLk5S+GiF6ISQBValZWVvDy8ip1/bfeegtLly6Fk5MTbGxsSqzj6uqKgwcPolWrVgCAgoICHD16FG+99VaJ9X19faFWq7Fr1y74+/sX21/UE1FYWCiV+fj4QKlUIikp6Zk9CHXq1JEmORY5cODAiy/yOfbt24dmzZrhiy++kMouX75crN7Jkyfx6NEjKcE5cOAArK2t4e7uDnt7+xfGXhJ3d3cMHjwYgwcPRkREBObMmcMkgEjHuDqA6AlBQUF47bXX0KVLF+zZsweJiYnYuXMnhg4dihs3bgAAhg0bhsmTJ2P16tU4d+4cvvjii+eu8a9evTqCg4PRr18/rF69Wmpz2bJlAAAPDw8oFArExcXhzp07yMzMRJUqVTBq1CiEh4djwYIFuHz5Mo4dO4bp06djwYIFAIDBgwfj4sWLGD16NM6fP4/Fixdj/vz5pbrOmzdv4sSJExrbgwcPUKtWLRw5cgSbN2/GhQsX8M033+Dw4cPFjs/Ly0P//v1x5swZbNiwAd9++y3CwsJgZGRUqtifNnz4cGzevBmJiYk4duwYduzYgTp16pTqWojoFeh7UgKRrjw5MbAs+5OTk0WfPn3Ea6+9JpRKpahRo4YYOHCgSE9PF0I8ngg4bNgwYWNjI2xtbcWIESNEnz59njkxUAghHj16JMLDw4Wrq6swMzMTXl5e4s8//5T2T5w4Ubi4uAiFQiGCg4OFEI8nM06bNk14e3sLU1NT4ejoKAICAsSuXbuk49atWye8vLyEUqkULVu2FH/++WepJgYCKLb99ddfIicnR/Tt21eoVCpha2srQkJCxNixY0WDBg2KvW/jxo0TDg4OwtraWgwcOFDk5ORIdV4U+9MTA8PCwkTNmjWFUqkUjo6Oonfv3uLu3bvPvAYi0g6FEM+YzURERESVGocDiIiIZIpJABERkUwxCSAiIpIpJgFEREQyxSSAiIhIppgEEBERyRSTACIiIpliEkBERCRTTAKIiIhkikkAERGRTDEJICIikqn/B9YMHqy5AAw2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 600x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Compute confusion matrix\n",
    "cm = confusion_matrix(all_labels, all_predictions)\n",
    "\n",
    "# Visualize confusion matrix\n",
    "plt.figure(figsize = (6, 5))\n",
    "sns.heatmap(cm, annot = True, fmt = 'd', cmap = 'Blues',\n",
    "            xticklabels = ['Negative', 'Positive'],\n",
    "            yticklabels = ['Negative', 'Positive'])\n",
    "plt.ylabel('Actual Labels')\n",
    "plt.xlabel('Predicted Labels')\n",
    "plt.title('Confusion Matrix', fontweight = 'bold')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O37ngmv_9B_z"
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #3eb8a5, #afcd95); padding: 15px; border-radius: 10px; text-align: center; background-color: black; color: black;\"> <h1 style=\"font-size: 24px;\"> Mixed Sentiments Identification by trained BERT Model </h1>  \n",
    "</div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X9YYaNyM9DwF"
   },
   "source": [
    "## Approach: Rule-Based Detection Using Sentence Splitting\n",
    "1. Sentencize the review, which means splitting the review into individual sentences.\n",
    "2. Perform sentiment analysis on each sentence.\n",
    "3. The review is classified as mixed when there are both positive and negative sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ty47ql1opUKy",
    "outputId": "2185d06e-a840-496a-fb22-e9a870cfdb01"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To download the necessary tokenizer data\n",
    "import nltk\n",
    "nltk.download('punkt') # To download the necessary tokenizer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "id": "TcnTRPKypfjM"
   },
   "outputs": [],
   "source": [
    "# Define function to classify reviews into 3 classes based on logits\n",
    "def classify_review_mixed(review):\n",
    "    # Tokenize the review into sentences\n",
    "    sentences = nltk.sent_tokenize(review)\n",
    "    pos_count = 0\n",
    "    neg_count = 0\n",
    "\n",
    "    # Classify each sentence\n",
    "    for sentence in sentences:\n",
    "        inputs = tokenizer(sentence, return_tensors = 'pt', \n",
    "                           padding = True, truncation = True, \n",
    "                           max_length = 512)\n",
    "        inputs = {key: val.to(device) for key, val in inputs.items()}\n",
    "\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**inputs)\n",
    "            logits = outputs.logits\n",
    "            \n",
    "            # Get the prediction (0: Negative, 1: Positive)\n",
    "            # Convert tensor to scalar\n",
    "            prediction = torch.argmax(logits, dim = -1).item()\n",
    "\n",
    "        # Increment counts based on prediction\n",
    "        if prediction == 1:\n",
    "            pos_count += 1\n",
    "\n",
    "        else:\n",
    "            neg_count += 1\n",
    "\n",
    "    # Threshold logic to determine 'Mixed'\n",
    "    if pos_count > 0 and neg_count > 0:\n",
    "        return 'Mixed'\n",
    "    elif pos_count > 0:\n",
    "        return 'Positive'\n",
    "    else:\n",
    "        return 'Negative'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ky3I2kYEpmhs",
    "outputId": "9dea13ee-b16c-4389-8d3e-cbb3afd40ea3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review: The movie was great. But the ending was disappointing.\n",
      "Overall Sentiment: Mixed\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: I loved the acting. But I hated the plot.\n",
      "Overall Sentiment: Mixed\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: The plot was dull, and the acting was bad.\n",
      "Overall Sentiment: Negative\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: An excellent movie with great performances.\n",
      "Overall Sentiment: Positive\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: This movie was fantastic! I loved it. However, the ending was quite confusing.\n",
      "Overall Sentiment: Mixed\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: I did not enjoy this film. It was boring and poorly executed.\n",
      "Overall Sentiment: Negative\n",
      "------------------------------------------------------------------------------------------\n",
      "Review: An average experience, nothing special. But not completely bad.\n",
      "Overall Sentiment: Positive\n",
      "------------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Example reviews\n",
    "reviews = ['The movie was great. But the ending was disappointing.', # Should be Mixed\n",
    "           'I loved the acting. But I hated the plot.', # Should be Mixed\n",
    "           'The plot was dull, and the acting was bad.', # Should be Negative\n",
    "           'An excellent movie with great performances.', # Should be Positive\n",
    "           'This movie was fantastic! I loved it. However, the ending was quite confusing.',\n",
    "           'I did not enjoy this film. It was boring and poorly executed.',\n",
    "           'An average experience, nothing special. But not completely bad.']\n",
    "\n",
    "for review in reviews:\n",
    "    sentiment = classify_review_mixed(review)\n",
    "    print(f'Review: {review}')\n",
    "    print(f'Overall Sentiment: {sentiment}')\n",
    "    print('-' * 90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}