Notebook_1.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "**Jonathan Ng**\n",
        "\n",
        "**Vinty Dong**\n",
        "\n",
        "**CSE 354 Final Project Notebook 1**\n",
        "\n",
        "The code in this notebook is heavily based off the code from HW assignment 3\n",
        "\n",
        "This notebook was used to finetune DistilBert-base-uncased using the MLM task on the math_qa dataset.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86,
          "referenced_widgets": [
            "b84a5c91252f4431b71d460d51222914",
            "6d1bf779d9e8442a92a58dd61d312617",
            "ae4e7279e92945619c093927aeccd67b",
            "236935fcc42647e3bfbf29ff163da0d9",
            "e7938316d405473294df45a86d2739be",
            "0afa14b889ab4c32b9a807adbc7c8a07",
            "6b65d2e7537d4e6da3edcb57ff1b4082",
            "f725a190a97a4ab095f83a0203021e32",
            "643edae209b94e8cbdd62967e20920f0",
            "99ae6a823ae3459091c3d8c462a64cc3",
            "f17791ceee9147ae8fe492422b3fe951"
          ]
        },
        "id": "y_L4OF34efal",
        "outputId": "c8f1da5f-56a6-4728-d30b-7d4d49543b0e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Found cached dataset math_qa (C:/Users/waibong/.cache/huggingface/datasets/math_qa/default/0.1.0/67fc1cc5d22b185002c6fd16e19e4d5215eae01fb04d656bed83204ba6ee55ff)\n",
            "100%|██████████| 3/3 [00:00<00:00, 374.20it/s]\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "# import pandas as pd\n",
        "# from sklearn.model_selection import train_test_split\n",
        "\n",
        "from torch.utils.data import Dataset, TensorDataset, DataLoader\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from tqdm import tqdm\n",
        "#from transformers import AdamW, pipeline\n",
        "#from sklearn.metrics import precision_score, recall_score, f1_score\n",
        "from datasets import load_dataset\n",
        "torch.manual_seed(42)\n",
        "np.random.seed(42)\n",
        "dataset = load_dataset(\"math_qa\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "8evwN3isefap"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForMaskedLM\n",
        "class DistillBERT():\n",
        "  def __init__(self):\n",
        "        self.tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "        self.model = AutoModelForMaskedLM.from_pretrained('distilbert-base-uncased')\n",
        "\n",
        "  def get_tokenizer_and_model(self):\n",
        "    return self.model, self.tokenizer\n",
        "  \n",
        "class DatasetLoader(Dataset):\n",
        "\n",
        "  def __init__(self, data, tokenizer):\n",
        "    self.data = data\n",
        "    self.tokenizer = tokenizer\n",
        "\n",
        "  def tokenize_data(self):\n",
        "    print(\"Processing data..\")\n",
        "    tokens = []\n",
        "    labels = []\n",
        "    # label_dict = {'a':0, 'b':1, 'c':2, 'd':3, 'e':4}\n",
        "\n",
        "\n",
        "    def mask_input(problem, rationale, mask_prob=0.15,mask_token='[MASK]'):\n",
        "\n",
        "      masked_output = []\n",
        "\n",
        "      if np.random.random() > 0.5:\n",
        "        # problem gets masked\n",
        "        for token in problem:\n",
        "          if np.random.random() <= mask_prob:\n",
        "            if np.random.random() <= 0.85:\n",
        "              masked_output.append(mask_token)\n",
        "            else:\n",
        "              masked_output.append(token)\n",
        "          else:\n",
        "            masked_output.append(token)\n",
        "\n",
        "        masked_output += rationale\n",
        "      else:\n",
        "        for token in rationale:\n",
        "          if np.random.random() <= mask_prob:\n",
        "            if np.random.random() <= 0.85:\n",
        "              masked_output.append(mask_token)\n",
        "            else:\n",
        "              masked_output.append(token)\n",
        "          else:\n",
        "            masked_output.append(token)\n",
        "        masked_output += problem\n",
        "      return \" \".join(masked_output)\n",
        "\n",
        "\n",
        "    c = 0\n",
        "    for training_instance in self.data:\n",
        "      if(c == 30000):\n",
        "        break\n",
        "      c = c + 1\n",
        "      problem = training_instance['Problem'].split()\n",
        "      rationale = training_instance['Rationale'].split()\n",
        "\n",
        "\n",
        "      expected_output = \"\".join((problem + rationale))\n",
        "\n",
        "      input = mask_input(problem, rationale)\n",
        "      masked_encoding = self.tokenizer.encode_plus(input, max_length=512, truncation=True, pad_to_max_length=True, add_special_tokens=True, return_tensors='pt')['input_ids']#.to(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "      unmasked_encoding = self.tokenizer.encode_plus(expected_output, max_length=512, truncation=True, pad_to_max_length=True, add_special_tokens=True, return_tensors='pt')['input_ids']#.to(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "      tokens.append(masked_encoding)\n",
        "      labels.append(unmasked_encoding)\n",
        "    \n",
        "    tokens = pad_sequence(tokens, batch_first=True)\n",
        "    labels = pad_sequence(labels, batch_first=True)\n",
        "    dataset = TensorDataset(tokens, labels)\n",
        "    return dataset\n",
        "\n",
        "  def get_data_loaders(self, batch_size=32, shuffle=True):\n",
        "    processed_dataset = self.tokenize_data()\n",
        "\n",
        "    data_loader = DataLoader(\n",
        "        processed_dataset,\n",
        "        shuffle=shuffle,\n",
        "        batch_size=batch_size\n",
        "    )\n",
        "\n",
        "    return data_loader\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "yPKMbY0Hefaq"
      },
      "outputs": [],
      "source": [
        "class Trainer():\n",
        "\n",
        "  def __init__(self, options):\n",
        "    self.device = options['device']\n",
        "    self.train_data = options['train_data']\n",
        "    self.val_data = options['val_data']\n",
        "    self.batch_size = options['batch_size']\n",
        "    self.epochs = options['epochs']\n",
        "    self.save_path = options['save_path']\n",
        "    self.training_type = options['training_type']\n",
        "    transformer = DistillBERT()\n",
        "    self.model, self.tokenizer = transformer.get_tokenizer_and_model()\n",
        "    self.model.to(self.device)\n",
        "\n",
        "  # def get_performance_metrics(self, preds, labels):\n",
        "  #   pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "  #   labels_flat = labels.flatten()\n",
        "  #   precision = precision_score(labels_flat, pred_flat, average='micro', zero_division=0)\n",
        "  #   recall = recall_score(labels_flat, pred_flat, average='micro', zero_division=0)\n",
        "  #   f1 = f1_score(labels_flat, pred_flat, average='micro', zero_division=0)\n",
        "  #   return precision, recall, f1\n",
        "  \n",
        "  def set_training_parameters(self):\n",
        "    # TODO(students): start\n",
        "    t = self.training_type\n",
        "    if t == 'frozen_embeddings':\n",
        "      # will not turn on require_grad = True for any layer\n",
        "      for name, layer in self.model.named_parameters():\n",
        "        if 'classifier' in name:\n",
        "          continue\n",
        "        layer.require_grad = False\n",
        "    elif t == 'top_2_training':\n",
        "      # require_grad = True for layers 4,5\n",
        "      for name, layer in self.model.named_parameters():\n",
        "        if 'classifier' in name:\n",
        "          continue\n",
        "        if 'layer.4' in name or 'layer.5' in name:\n",
        "          layer.require_grad = True\n",
        "        else:\n",
        "          layer.require_grad = False  \n",
        "    elif t == 'top_4_training':\n",
        "      #require_grad = True for layers 2,3,4,5\n",
        "      for name, layer in self.model.named_parameters():\n",
        "        if 'classifier' in name:\n",
        "          continue\n",
        "        if 'layer.2' in name or 'layer.3' in name or 'layer.4' in name or 'layer.5' in name:\n",
        "          layer.require_grad = True\n",
        "        else:\n",
        "          layer.require_grad = False \n",
        "    elif t == 'all_training':\n",
        "      # require_grad = True for layers 0,1,2,3,4,5\n",
        "      for name, layer in self.model.named_parameters():\n",
        "        layer.require_grad = True\n",
        "    else:\n",
        "      raise KeyError(f\"training_type={t} not found\")\n",
        "    # TODO(students): end\n",
        "\n",
        "  def train(self, data_loader, optimizer):\n",
        "    self.model.train()\n",
        "    total_recall = 0\n",
        "    total_precision = 0\n",
        "    total_f1 = 0\n",
        "    total_loss = 0\n",
        "\n",
        "    for (problem, labels) in tqdm(data_loader):\n",
        "      self.model.zero_grad()\n",
        "\n",
        "      problem = problem.squeeze(1).to(self.device)\n",
        "      labels = labels.squeeze(1).to(self.device)\n",
        "      output = self.model(problem, labels=labels)\n",
        "\n",
        "      logits = output.logits\n",
        "      loss = output.loss\n",
        "\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "      total_loss += loss\n",
        "      # precision, recall, f1 = self.get_performance_metrics(preds=logits.detach().cpu().numpy(), labels=labels.detach().cpu().numpy())\n",
        "\n",
        "    #   total_recall += recall\n",
        "    #   total_precision += precision\n",
        "    #   total_f1 += f1\n",
        "\n",
        "    precision = total_precision/len(data_loader)\n",
        "    recall = total_recall/len(data_loader)\n",
        "    f1 = total_f1/len(data_loader)\n",
        "    loss = total_loss/len(data_loader)\n",
        "\n",
        "    return precision, recall, f1, loss\n",
        "\n",
        "  def eval(self, data_loader):\n",
        "    self.model.eval()\n",
        "    total_recall = 0\n",
        "    total_precision = 0\n",
        "    total_f1 = 0\n",
        "    total_loss = 0\n",
        "\n",
        "    with torch.no_grad():\n",
        "      for (problem, labels) in tqdm(data_loader):\n",
        "        problem = problem.squeeze(1).to(self.device)\n",
        "        labels = labels.squeeze(1).to(self.device)\n",
        "        output = self.model(problem, labels=labels)\n",
        "\n",
        "        logits = output.logits\n",
        "        loss = output.loss\n",
        "        \n",
        "        total_loss += loss\n",
        "        # precision, recall, f1 = self.get_performance_metrics(preds=logits.detach().cpu().numpy(), labels=labels.detach().cpu().numpy())\n",
        "\n",
        "        # total_recall += recall\n",
        "        # total_precision += precision\n",
        "        # total_f1 += f1\n",
        "    \n",
        "    precision = total_precision/len(data_loader)\n",
        "    recall = total_recall/len(data_loader)\n",
        "    f1 = total_f1/len(data_loader)\n",
        "    loss = total_loss/len(data_loader)\n",
        "\n",
        "    return precision, recall, f1, loss\n",
        "\n",
        "  def save_transformer(self):\n",
        "    self.model.save_pretrained(self.save_path)\n",
        "    self.tokenizer.save_pretrained(self.save_path)\n",
        "\n",
        "  def execute(self):\n",
        "    last_best = 0\n",
        "    train_dataset = DatasetLoader(self.train_data, self.tokenizer)\n",
        "    train_data_loader = train_dataset.get_data_loaders(self.batch_size)\n",
        "    val_dataset = DatasetLoader(self.val_data, self.tokenizer)\n",
        "    val_data_loader = val_dataset.get_data_loaders(self.batch_size)\n",
        "    optimizer = torch.optim.AdamW(self.model.parameters(), lr = 3e-5, eps = 1e-8)\n",
        "    self.set_training_parameters()\n",
        "\n",
        "    print(\"Done processing data\")\n",
        "\n",
        "    for epoch_i in range(0, self.epochs):\n",
        "      train_precision, train_recall, train_f1, train_loss = self.train(train_data_loader, optimizer)\n",
        "      print(f'Epoch {epoch_i + 1}: train_loss: {train_loss:.4f} train_precision: {train_precision:.4f} train_recall: {train_recall:.4f} train_f1: {train_f1:.4f}')\n",
        "      val_precision, val_recall, val_f1, val_loss = self.eval(val_data_loader)\n",
        "      print(f'Epoch {epoch_i + 1}: val_loss: {val_loss:.4f} val_precision: {val_precision:.4f} val_recall: {val_recall:.4f} val_f1: {val_f1:.4f}')\n",
        "\n",
        "      if val_f1 > last_best:\n",
        "        print(\"Saving model..\")\n",
        "        self.save_transformer()\n",
        "        last_best = val_f1\n",
        "        print(\"Model saved.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i8IhlJE4efar",
        "outputId": "0eb9e8b5-21c1-4423-fc0f-ade24970a0a9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cuda:0\n",
            "Processing data..\n",
            "Processing data..\n",
            "Done processing data\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:20<00:00,  3.08it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1: train_loss: 1.3548 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:56<00:00,  9.60it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1: val_loss: 1.1459 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:57<00:00,  3.04it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2: train_loss: 1.0567 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.71it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2: val_loss: 0.9575 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:13<00:00,  3.02it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3: train_loss: 0.9164 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.68it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3: val_loss: 0.8794 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:00<00:00,  3.03it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4: train_loss: 0.8306 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.66it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4: val_loss: 0.8094 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:03<00:00,  3.03it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5: train_loss: 0.7687 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:54<00:00,  9.74it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5: val_loss: 0.7529 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:57<00:00,  3.04it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6: train_loss: 0.7230 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.69it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6: val_loss: 0.7369 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:55<00:00,  3.04it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7: train_loss: 0.6850 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.71it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7: val_loss: 0.7011 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:21<00:00,  3.08it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8: train_loss: 0.6536 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:53<00:00,  9.86it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8: val_loss: 0.6813 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:19<00:00,  3.08it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9: train_loss: 0.6270 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:53<00:00,  9.85it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9: val_loss: 0.6598 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:18<00:00,  3.08it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10: train_loss: 0.6041 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:53<00:00,  9.84it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10: val_loss: 0.6612 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        }
      ],
      "source": [
        "BATCH_SIZE = 4\n",
        "EPOCHS = 10\n",
        "# TEST_PATH = \"data/test_data.csv\"\n",
        "# TRAIN_PATH = \"data/train_data.csv\"\n",
        "# VAL_PATH = \"data/val_data.csv\"\n",
        "SAVE_PATH = \"models/DistilBERT\"\n",
        "\n",
        "options = {}\n",
        "options['batch_size'] = BATCH_SIZE\n",
        "options['device'] = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "options['train_data'] = dataset['train']\n",
        "options['val_data'] = dataset['validation']\n",
        "options['save_path'] = SAVE_PATH + '_top_2_training'\n",
        "options['epochs'] = EPOCHS\n",
        "options['training_type'] = 'top_2_training'\n",
        "\n",
        "print(options['device'])\n",
        "\n",
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "trainer = Trainer(options)\n",
        "trainer.execute()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "trainer.save_transformer()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing data..\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\ProgramData\\miniconda3\\envs\\cse354\\lib\\site-packages\\transformers\\tokenization_utils_base.py:2304: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Processing data..\n",
            "Done processing data\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [40:55<00:00,  3.04it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1: train_loss: 0.6445 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:57<00:00,  9.56it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1: val_loss: 0.6527 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [45:00<00:00,  2.76it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2: train_loss: 0.6080 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [02:04<00:00,  8.96it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2: val_loss: 0.6263 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [45:49<00:00,  2.71it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3: train_loss: 0.5813 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [02:08<00:00,  8.71it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 3: val_loss: 0.6087 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [46:14<00:00,  2.69it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4: train_loss: 0.5606 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [02:08<00:00,  8.68it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 4: val_loss: 0.6072 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [45:48<00:00,  2.71it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5: train_loss: 0.5408 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:54<00:00,  9.74it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 5: val_loss: 0.5903 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [42:14<00:00,  2.94it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6: train_loss: 0.5237 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.69it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 6: val_loss: 0.5913 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [43:02<00:00,  2.89it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7: train_loss: 0.5104 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:55<00:00,  9.68it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 7: val_loss: 0.5795 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:28<00:00,  3.00it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8: train_loss: 0.4980 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:54<00:00,  9.75it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 8: val_loss: 0.5804 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:36<00:00,  2.99it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9: train_loss: 0.4853 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [02:04<00:00,  8.98it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 9: val_loss: 0.5745 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 7460/7460 [41:01<00:00,  3.03it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10: train_loss: 0.4738 train_precision: 0.0000 train_recall: 0.0000 train_f1: 0.0000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 1119/1119 [01:48<00:00, 10.30it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 10: val_loss: 0.5684 val_precision: 0.0000 val_recall: 0.0000 val_f1: 0.0000\n"
          ]
        }
      ],
      "source": [
        "trainer.execute()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "trainer.save_transformer()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "cse354",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.10"
    },
    "orig_nbformat": 4,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0afa14b889ab4c32b9a807adbc7c8a07": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "236935fcc42647e3bfbf29ff163da0d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_99ae6a823ae3459091c3d8c462a64cc3",
            "placeholder": "​",
            "style": "IPY_MODEL_f17791ceee9147ae8fe492422b3fe951",
            "value": " 3/3 [00:00&lt;00:00, 54.51it/s]"
          }
        },
        "643edae209b94e8cbdd62967e20920f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6b65d2e7537d4e6da3edcb57ff1b4082": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6d1bf779d9e8442a92a58dd61d312617": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0afa14b889ab4c32b9a807adbc7c8a07",
            "placeholder": "​",
            "style": "IPY_MODEL_6b65d2e7537d4e6da3edcb57ff1b4082",
            "value": "100%"
          }
        },
        "99ae6a823ae3459091c3d8c462a64cc3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae4e7279e92945619c093927aeccd67b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f725a190a97a4ab095f83a0203021e32",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_643edae209b94e8cbdd62967e20920f0",
            "value": 3
          }
        },
        "b84a5c91252f4431b71d460d51222914": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6d1bf779d9e8442a92a58dd61d312617",
              "IPY_MODEL_ae4e7279e92945619c093927aeccd67b",
              "IPY_MODEL_236935fcc42647e3bfbf29ff163da0d9"
            ],
            "layout": "IPY_MODEL_e7938316d405473294df45a86d2739be"
          }
        },
        "e7938316d405473294df45a86d2739be": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f17791ceee9147ae8fe492422b3fe951": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f725a190a97a4ab095f83a0203021e32": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}