kaggle_short_answer_scoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bf732cdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2021-11-26 11:28:51--  https://gitlab.com/brainekt_ai/kaggle-short-ans/-/raw/main/train.tsv\n",
      "Resolving gitlab.com (gitlab.com)... 172.65.251.78, 2606:4700:90:0:f22e:fbec:5bed:a9b9\n",
      "Connecting to gitlab.com (gitlab.com)|172.65.251.78|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 4291891 (4.1M) [text/plain]\n",
      "Saving to: ‘train.tsv’\n",
      "\n",
      "train.tsv           100%[===================>]   4.09M  19.3MB/s    in 0.2s    \n",
      "\n",
      "2021-11-26 11:28:52 (19.3 MB/s) - ‘train.tsv’ saved [4291891/4291891]\n",
      "\n",
      "--2021-11-26 11:28:52--  https://gitlab.com/brainekt_ai/kaggle-short-ans/-/raw/main/test.csv\n",
      "Resolving gitlab.com (gitlab.com)... 172.65.251.78, 2606:4700:90:0:f22e:fbec:5bed:a9b9\n",
      "Connecting to gitlab.com (gitlab.com)|172.65.251.78|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 50272 (49K) [text/plain]\n",
      "Saving to: ‘test.csv’\n",
      "\n",
      "test.csv            100%[===================>]  49.09K  --.-KB/s    in 0.001s  \n",
      "\n",
      "2021-11-26 11:28:53 (84.5 MB/s) - ‘test.csv’ saved [50272/50272]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# download the train and test data\n",
    "! wget https://gitlab.com/brainekt_ai/kaggle-short-ans/-/raw/main/train.tsv\n",
    "! wget https://gitlab.com/brainekt_ai/kaggle-short-ans/-/raw/main/test.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02ba58dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Nov 26 13:00:49 2021       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 418.67       Driver Version: 418.67       CUDA Version: 10.2     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-SXM2...  Off  | 00000000:61:00.0 Off |                    0 |\n",
      "| N/A   30C    P0    57W / 300W |   1150MiB / 32480MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  Tesla V100-SXM2...  Off  | 00000000:62:00.0 Off |                    0 |\n",
      "| N/A   30C    P0    43W / 300W |     11MiB / 32480MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   2  Tesla V100-SXM2...  Off  | 00000000:89:00.0 Off |                    0 |\n",
      "| N/A   31C    P0    43W / 300W |     11MiB / 32480MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   3  Tesla V100-SXM2...  Off  | 00000000:8A:00.0 Off |                    0 |\n",
      "| N/A   50C    P0   202W / 300W |  32088MiB / 32480MiB |     55%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "! nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0aa05fc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert_sa.pt  kaggle_short_answer_scoring.ipynb  test.csv  train.tsv\r\n"
     ]
    }
   ],
   "source": [
    "! ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cdc494f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train = pd.read_csv('train.tsv', sep='\\t')\n",
    "test = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "feb98742",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>EssaySet</th>\n",
       "      <th>Score1</th>\n",
       "      <th>Score2</th>\n",
       "      <th>EssayText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Some additional information that we would need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>After reading the expirement, I realized that ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>What you need is more trials, a control set up...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>The student should list what rock is better an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>For the students to be able to make a replicat...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  EssaySet  Score1  Score2  \\\n",
       "0   1         1       1       1   \n",
       "1   2         1       1       1   \n",
       "2   3         1       1       1   \n",
       "3   4         1       0       0   \n",
       "4   5         1       2       2   \n",
       "\n",
       "                                           EssayText  \n",
       "0  Some additional information that we would need...  \n",
       "1  After reading the expirement, I realized that ...  \n",
       "2  What you need is more trials, a control set up...  \n",
       "3  The student should list what rock is better an...  \n",
       "4  For the students to be able to make a replicat...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d707652",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>essay_set</th>\n",
       "      <th>essay_weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2230</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2231</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2232</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2233</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2234</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id  essay_set  essay_weight\n",
       "0  2230          1             1\n",
       "1  2231          1             1\n",
       "2  2232          1             1\n",
       "3  2233          1             1\n",
       "4  2234          1             1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c5775653",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from sklearn.utils import shuffle\n",
    "random.seed(19)\n",
    "\n",
    "train_sa = []\n",
    "train_scores = []\n",
    "test_sa = []\n",
    "test_scores = []\n",
    "\n",
    "# shuffle\n",
    "train = shuffle(train)\n",
    "train.reset_index(inplace=True, drop=True)\n",
    "\n",
    "for i in range(len(train)):\n",
    "  if i <= len(train) * 0.8:\n",
    "    train_sa.append(train[\"EssayText\"][i])\n",
    "    train_scores.append(train[\"Score1\"][i])\n",
    "  else:\n",
    "    test_sa.append(train[\"EssayText\"][i])\n",
    "    test_scores.append(train[\"Score1\"][i])   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "087f907b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13766\n",
      "13766\n",
      "3441\n",
      "3441\n"
     ]
    }
   ],
   "source": [
    "print(len(train_sa))\n",
    "print(len(train_scores))\n",
    "print(len(test_sa))\n",
    "print(len(test_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06da810a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0, 1, 2, 3}\n"
     ]
    }
   ],
   "source": [
    "print(set(train_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "30cafc4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0, 1, 2, 3}\n"
     ]
    }
   ],
   "source": [
    "print(set(test_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9fd7d713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1819"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(len(a) for a in train_sa)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "96b8b270",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-cased', max_length = 512, padding_side = 'right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ce399ab1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertForSequenceClassification\n",
    "\n",
    "num_class = 4 # we have 4 score classes\n",
    "\n",
    "bertclassifier = BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "56e71f92",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "# create our dataset\n",
    "\n",
    "class ShortAnsDataset(Dataset):\n",
    "    def __init__(self, texts, class_labels, tokenizer):\n",
    "        self.texts = texts\n",
    "        self.class_labels = class_labels\n",
    "        self.tokenizer = tokenizer\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.class_labels)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "            \n",
    "        encoded = tokenizer.encode_plus(self.texts[idx], add_special_tokens=True, max_length = 512, pad_to_max_length = True,\n",
    "                                return_token_type_ids = False,\n",
    "                                return_attention_mask = True)  # Batch size 1\n",
    "            \n",
    "        # text to tokens\n",
    "        token = torch.tensor(encoded['input_ids']) #.unsqueeze(0)\n",
    "        mask = torch.tensor(encoded['attention_mask'])\n",
    "        # ohe label\n",
    "        ohe = torch.tensor(self.class_labels[idx])\n",
    "\n",
    "        return token, mask, ohe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c382ea19",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train = ShortAnsDataset(train_sa, train_scores, tokenizer)\n",
    "dataset_test = ShortAnsDataset(test_sa, test_scores, tokenizer)\n",
    "\n",
    "# in colab, we can use high batch size\n",
    "\n",
    "sa_dataloader_train = DataLoader(dataset_train, batch_size=32, shuffle=True) \n",
    "sa_dataloader_test = DataLoader(dataset_test, batch_size=4, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "39f7e6d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import transformers\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # cuda for gpu acceleration\n",
    "\n",
    "# optimizer\n",
    "\n",
    "optimizer = transformers.AdamW(bertclassifier.parameters(), lr = 1e-5)\n",
    "\n",
    "\n",
    "bertclassifier.to(device) # taking the model to gpu if possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f4c0cef3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:19: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d051b65aaaad486885237092099ed191",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=5.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98927735efc24ce8bf56350675db5e5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train loss: 0.8935734247248577\n",
      "acc: 0.6014597834493427\n",
      "f1: 0.6014597834493427\n",
      "test acc: 0.6608594657375145\n",
      "test f1: 0.6608594657375145\n",
      "epoch: 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2b593398fc544aea366011d6368924d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train loss: 0.6725761873816116\n",
      "acc: 0.7055056071152359\n",
      "f1: 0.7055056071152359\n",
      "test acc: 0.7052845528455285\n",
      "test f1: 0.7052845528455285\n",
      "epoch: 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77dc4de03a6c4c8386f9be1d2a8a836a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train loss: 0.5577772878922885\n",
      "acc: 0.7654920726991492\n",
      "f1: 0.7654920726991492\n",
      "test acc: 0.6977351916376306\n",
      "test f1: 0.6977351916376306\n",
      "epoch: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62b63bcbc0844e9aa8ce29360fa353d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train loss: 0.46487432904973663\n",
      "acc: 0.8120407965970611\n",
      "f1: 0.8120407965970611\n",
      "test acc: 0.7105110336817654\n",
      "test f1: 0.7105110336817654\n",
      "epoch: 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2bdd5d2473141ffaaba9ebbe4faa18b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train loss: 0.3633637853191513\n",
      "acc: 0.8609097061098221\n",
      "f1: 0.8609097061098221\n",
      "test acc: 0.7157375145180023\n",
      "test f1: 0.7157375145180023\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# training epochs\n",
    "\n",
    "epochs = 5\n",
    "\n",
    "# metrics\n",
    "\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, cohen_kappa_score\n",
    "import numpy as np\n",
    "\n",
    "train_losses = []\n",
    "\n",
    "train_metrics = {'acc': [], 'f1': [], 'kappa': []}\n",
    "test_metrics = {'acc': [], 'f1': [], 'kappa': []}\n",
    "\n",
    "# progress bar\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "for e in tqdm_notebook(range(epochs)):\n",
    "    train_loss = 0.0\n",
    "    train_acc = 0.0\n",
    "    train_f1 = 0.0\n",
    "    train_kappa = 0.0\n",
    "    batch_cnt = 0\n",
    "    \n",
    "    bertclassifier.train()\n",
    "    \n",
    "    print(f'epoch: {e+1}')\n",
    "    \n",
    "    for i_batch, (X, X_mask, y) in tqdm_notebook(enumerate(sa_dataloader_train)):\n",
    "        X = X.to(device)\n",
    "        X_mask = X_mask.to(device)\n",
    "        y = y.to(device)\n",
    "        \n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        loss, y_pred = bertclassifier(X, attention_mask = X_mask, labels = y)\n",
    "\n",
    "        train_loss += loss.item()\n",
    "        loss.backward()\n",
    "        \n",
    "        torch.nn.utils.clip_grad_norm_(bertclassifier.parameters(), 1.0)\n",
    "\n",
    "        optimizer.step()\n",
    "        \n",
    "        y_pred = torch.argmax(y_pred, dim = -1)\n",
    "        \n",
    "        # update metrics\n",
    "        train_acc += accuracy_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy())\n",
    "        train_f1 += f1_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy(), average = 'micro')\n",
    "        if np.array_equal(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy()):\n",
    "            train_kappa += 1.\n",
    "        else:\n",
    "            train_kappa += cohen_kappa_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy(), weights = 'quadratic')\n",
    "        batch_cnt += 1\n",
    "    \n",
    "    print(f'train loss: {train_loss/batch_cnt}')\n",
    "    print(f'acc: {train_acc/batch_cnt}')\n",
    "    print(f'f1: {train_f1/batch_cnt}')\n",
    "    train_losses.append(train_loss/batch_cnt)\n",
    "    train_metrics['acc'].append(train_acc/batch_cnt)\n",
    "    train_metrics['f1'].append(train_f1/batch_cnt)\n",
    "    train_metrics['kappa'].append(train_kappa/batch_cnt)\n",
    "        \n",
    "        \n",
    "    test_loss = 0.0\n",
    "    test_acc = 0.0\n",
    "    test_f1 = 0.0\n",
    "    test_kappa = 0.0\n",
    "    batch_cnt = 0\n",
    "    \n",
    "    bertclassifier.eval()\n",
    "    with torch.no_grad():\n",
    "        for i_batch, (X, X_mask, y) in enumerate(sa_dataloader_test):\n",
    "            X = X.to(device)\n",
    "            X_mask = X_mask.to(device)\n",
    "            y = y.to(device)\n",
    "\n",
    "            y_pred = bertclassifier(X, attention_mask = X_mask, labels = y)[1] # in eval model we get the softmax output so, don't need to index\n",
    "\n",
    "            \n",
    "            y_pred = torch.argmax(y_pred, dim = -1)\n",
    "\n",
    "            # update metrics\n",
    "            test_acc += accuracy_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy())\n",
    "            test_f1 += f1_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy(), average = 'micro')\n",
    "            if np.array_equal(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy()):\n",
    "                test_kappa += 1.\n",
    "            else:\n",
    "                test_kappa += cohen_kappa_score(y.cpu().detach().numpy(), y_pred.cpu().detach().numpy(), weights = 'quadratic')\n",
    "            batch_cnt += 1\n",
    "            \n",
    "    test_metrics['acc'].append(test_acc/batch_cnt)\n",
    "    test_metrics['f1'].append(test_f1/batch_cnt)\n",
    "    test_metrics['kappa'].append(test_kappa/batch_cnt)\n",
    "    print(f'test acc: {test_acc/batch_cnt}')\n",
    "    print(f'test f1: {test_f1/batch_cnt}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "69ff45f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(bertclassifier, \"bert_sa.pt\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2a6a4892",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:6: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0e24226b7904ca293f1eceefd431722",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "y_trues = []\n",
    "y_preds = []\n",
    "\n",
    "bertclassifier.eval()\n",
    "with torch.no_grad():\n",
    "    for i_batch, (X, X_mask, y) in tqdm_notebook(enumerate(sa_dataloader_test)):\n",
    "        X = X.to(device)\n",
    "        X_mask = X_mask.to(device)\n",
    "        y = y.to(device)\n",
    "\n",
    "        y_pred = bertclassifier(X, attention_mask = X_mask, labels = y)[1] # in eval model we get the softmax output so, don't need to index\n",
    "\n",
    "        y_pred = torch.argmax(y_pred, dim = -1)\n",
    "        \n",
    "        y_trues.extend(y.cpu().detach().numpy().tolist())\n",
    "        y_preds.extend(y_pred.cpu().detach().numpy().tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "91c76601",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 1, 0, 0]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_trues[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "10839ab2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 3, 1, 1, 0]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_preds[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7609d25f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7595809748343627"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohen_kappa_score(y_trues, y_preds, weights = 'quadratic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7e9d629",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}