training_super_fast_tiny_bert.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pranshurastogi29/News-summarization-Topic-Prediction/blob/main/training_super_fast_tiny_bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b3a69fce",
      "metadata": {
        "_kg_hide-input": true,
        "_kg_hide-output": true,
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:48.068626Z",
          "iopub.status.busy": "2022-01-20T21:22:48.067091Z",
          "iopub.status.idle": "2022-01-20T21:22:56.293614Z",
          "shell.execute_reply": "2022-01-20T21:22:56.292983Z",
          "shell.execute_reply.started": "2022-01-20T21:13:15.807629Z"
        },
        "papermill": {
          "duration": 8.247803,
          "end_time": "2022-01-20T21:22:56.293778",
          "exception": false,
          "start_time": "2022-01-20T21:22:48.045975",
          "status": "completed"
        },
        "tags": [],
        "id": "b3a69fce",
        "outputId": "04ea5d84-0dff-4767-fb84-b1635396e030"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /usr/share/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import os\n",
        "from glob import glob\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import Dataset,DataLoader\n",
        "from torch.autograd import Variable\n",
        "from torch.utils.data.sampler import SequentialSampler, RandomSampler\n",
        "import sklearn\n",
        "\n",
        "import time\n",
        "import random\n",
        "from datetime import datetime\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\n",
        "from transformers import AutoTokenizer, AutoModel\n",
        "import transformers\n",
        "import gc\n",
        "import re\n",
        "\n",
        "# !pip install nltk > /dev/null\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "from nltk import sent_tokenize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "90bc1b3a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:56.333735Z",
          "iopub.status.busy": "2022-01-20T21:22:56.333124Z",
          "iopub.status.idle": "2022-01-20T21:22:56.339114Z",
          "shell.execute_reply": "2022-01-20T21:22:56.339531Z",
          "shell.execute_reply.started": "2022-01-20T21:13:18.673474Z"
        },
        "papermill": {
          "duration": 0.028209,
          "end_time": "2022-01-20T21:22:56.339695",
          "exception": false,
          "start_time": "2022-01-20T21:22:56.311486",
          "status": "completed"
        },
        "tags": [],
        "id": "90bc1b3a"
      },
      "outputs": [],
      "source": [
        "SEED = 42\n",
        "\n",
        "MAX_LENGTH = 512\n",
        "BACKBONE_PATH = \"distilroberta-base\"\n",
        "\n",
        "\n",
        "def seed_everything(seed):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = True\n",
        "\n",
        "seed_everything(SEED)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b04ecb44",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:56.379082Z",
          "iopub.status.busy": "2022-01-20T21:22:56.377898Z",
          "iopub.status.idle": "2022-01-20T21:22:57.776419Z",
          "shell.execute_reply": "2022-01-20T21:22:57.775891Z",
          "shell.execute_reply.started": "2022-01-20T21:13:18.684144Z"
        },
        "papermill": {
          "duration": 1.421395,
          "end_time": "2022-01-20T21:22:57.776558",
          "exception": false,
          "start_time": "2022-01-20T21:22:56.355163",
          "status": "completed"
        },
        "tags": [],
        "id": "b04ecb44"
      },
      "outputs": [],
      "source": [
        "from nltk import sent_tokenize\n",
        "from random import shuffle\n",
        "import random\n",
        "import albumentations\n",
        "from albumentations.core.transforms_interface import DualTransform, BasicTransform\n",
        "\n",
        "\n",
        "LANGS = {\n",
        "    'en': 'english',\n",
        "}\n",
        "\n",
        "def get_sentences(text, lang='en'):\n",
        "    return sent_tokenize(text, LANGS.get(lang, 'english'))\n",
        "\n",
        "def exclude_duplicate_sentences(text, lang='en'):\n",
        "    sentences = []\n",
        "    for sentence in get_sentences(text, lang):\n",
        "        sentence = sentence.strip()\n",
        "        if sentence not in sentences:\n",
        "            sentences.append(sentence)\n",
        "    return ' '.join(sentences)\n",
        "\n",
        "def clean_text(text, lang='en'):\n",
        "    text = str(text)\n",
        "    text = re.sub(r' r','',text)\n",
        "    text = exclude_duplicate_sentences(text, lang)\n",
        "    return text.strip()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e78866dd",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:57.813057Z",
          "iopub.status.busy": "2022-01-20T21:22:57.812502Z",
          "iopub.status.idle": "2022-01-20T21:22:57.892962Z",
          "shell.execute_reply": "2022-01-20T21:22:57.892221Z",
          "shell.execute_reply.started": "2022-01-20T21:13:19.250668Z"
        },
        "papermill": {
          "duration": 0.101011,
          "end_time": "2022-01-20T21:22:57.893121",
          "exception": false,
          "start_time": "2022-01-20T21:22:57.792110",
          "status": "completed"
        },
        "tags": [],
        "id": "e78866dd"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "df = pd.read_csv('../input/scraping-news-and-create-dataset/data.csv')\n",
        "df.columns = ['date','topic','news']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c03c3e16",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:57.935338Z",
          "iopub.status.busy": "2022-01-20T21:22:57.934575Z",
          "iopub.status.idle": "2022-01-20T21:22:57.940375Z",
          "shell.execute_reply": "2022-01-20T21:22:57.939929Z",
          "shell.execute_reply.started": "2022-01-20T21:13:19.293501Z"
        },
        "papermill": {
          "duration": 0.031796,
          "end_time": "2022-01-20T21:22:57.940510",
          "exception": false,
          "start_time": "2022-01-20T21:22:57.908714",
          "status": "completed"
        },
        "tags": [],
        "id": "c03c3e16"
      },
      "outputs": [],
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "labelencoder = LabelEncoder()\n",
        "df['labels'] = labelencoder.fit_transform(df['topic'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f9f5171e",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:57.981259Z",
          "iopub.status.busy": "2022-01-20T21:22:57.980431Z",
          "iopub.status.idle": "2022-01-20T21:22:57.983814Z",
          "shell.execute_reply": "2022-01-20T21:22:57.984211Z",
          "shell.execute_reply.started": "2022-01-20T21:13:19.302241Z"
        },
        "papermill": {
          "duration": 0.028333,
          "end_time": "2022-01-20T21:22:57.984376",
          "exception": false,
          "start_time": "2022-01-20T21:22:57.956043",
          "status": "completed"
        },
        "tags": [],
        "id": "f9f5171e",
        "outputId": "1940b7a6-0781-478a-db2a-d79f9b574eaf"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([3, 5, 4, 7, 6, 0, 1, 2])"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df['labels'].unique()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c01e31ed",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:22:58.019755Z",
          "iopub.status.busy": "2022-01-20T21:22:58.018857Z",
          "iopub.status.idle": "2022-01-20T21:23:04.199242Z",
          "shell.execute_reply": "2022-01-20T21:23:04.198733Z",
          "shell.execute_reply.started": "2022-01-20T21:13:19.318569Z"
        },
        "papermill": {
          "duration": 6.199087,
          "end_time": "2022-01-20T21:23:04.199408",
          "exception": false,
          "start_time": "2022-01-20T21:22:58.000321",
          "status": "completed"
        },
        "tags": [],
        "colab": {
          "referenced_widgets": [
            "80055852174449318e68003d350504a2",
            "7516dbc31c0247b889c4ca6e7abb2a59",
            "781b72014e4743598dde4d79b898c536",
            "a523475166f74d22ada4182dd3676194"
          ]
        },
        "id": "c01e31ed",
        "outputId": "797e6229-257b-477d-f486-881df45eff81"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "80055852174449318e68003d350504a2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/480 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7516dbc31c0247b889c4ca6e7abb2a59",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "781b72014e4743598dde4d79b898c536",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a523475166f74d22ada4182dd3676194",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(BACKBONE_PATH)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "00697264",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:04.249723Z",
          "iopub.status.busy": "2022-01-20T21:23:04.248769Z",
          "iopub.status.idle": "2022-01-20T21:23:04.251114Z",
          "shell.execute_reply": "2022-01-20T21:23:04.251600Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.032752Z"
        },
        "papermill": {
          "duration": 0.03255,
          "end_time": "2022-01-20T21:23:04.251771",
          "exception": false,
          "start_time": "2022-01-20T21:23:04.219221",
          "status": "completed"
        },
        "tags": [],
        "id": "00697264"
      },
      "outputs": [],
      "source": [
        "def onehot(size, target):\n",
        "    vec = torch.zeros(size, dtype=torch.float32)\n",
        "    vec[target] = 1.\n",
        "    return vec\n",
        "\n",
        "class DatasetRetriever(Dataset):\n",
        "\n",
        "    def __init__(self, labels_or_ids, comment_texts, lang = 'en', test=False):\n",
        "        self.test = test\n",
        "        self.lang = lang\n",
        "        self.labels_or_ids = labels_or_ids\n",
        "        self.comment_texts = comment_texts\n",
        "        \n",
        "    def get_tokens(self, text):\n",
        "        encoded = tokenizer.encode_plus(\n",
        "            text, \n",
        "            add_special_tokens=True, \n",
        "            max_length=MAX_LENGTH, \n",
        "            pad_to_max_length=True\n",
        "        )\n",
        "        return encoded['input_ids'], encoded['attention_mask']\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.comment_texts.shape[0]\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        text = self.comment_texts[idx]\n",
        "        if self.test is False:\n",
        "            label = self.labels_or_ids[idx]\n",
        "            target = onehot(8, label)\n",
        "\n",
        "        tokens, attention_mask = self.get_tokens(str(text))\n",
        "        tokens, attention_mask = torch.tensor(tokens), torch.tensor(attention_mask)\n",
        "\n",
        "        if self.test is False:\n",
        "            return target, tokens, attention_mask\n",
        "        return self.labels_or_ids[idx], tokens, attention_mask\n",
        "\n",
        "    def get_labels(self):\n",
        "        return list(np.char.add(self.labels_or_ids.astype(str),''))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "259457e2",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:04.295620Z",
          "iopub.status.busy": "2022-01-20T21:23:04.293872Z",
          "iopub.status.idle": "2022-01-20T21:23:04.296230Z",
          "shell.execute_reply": "2022-01-20T21:23:04.296712Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.044896Z"
        },
        "papermill": {
          "duration": 0.026615,
          "end_time": "2022-01-20T21:23:04.296865",
          "exception": false,
          "start_time": "2022-01-20T21:23:04.270250",
          "status": "completed"
        },
        "tags": [],
        "id": "259457e2"
      },
      "outputs": [],
      "source": [
        "train_dataset = DatasetRetriever(\n",
        "    labels_or_ids=df['labels'].values, \n",
        "    comment_texts=df['news'].values, \n",
        "    lang='en')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1a981065",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:04.346188Z",
          "iopub.status.busy": "2022-01-20T21:23:04.345264Z",
          "iopub.status.idle": "2022-01-20T21:23:04.347839Z",
          "shell.execute_reply": "2022-01-20T21:23:04.347317Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.058227Z"
        },
        "papermill": {
          "duration": 0.032652,
          "end_time": "2022-01-20T21:23:04.347967",
          "exception": false,
          "start_time": "2022-01-20T21:23:04.315315",
          "status": "completed"
        },
        "tags": [],
        "id": "1a981065"
      },
      "outputs": [],
      "source": [
        "class AucMeter(object):\n",
        "    def __init__(self):\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.y_true = np.array([0,1,2])\n",
        "        self.y_pred = np.array([0,1,2])\n",
        "        self.score = 0\n",
        "\n",
        "    def update(self, y_true, y_pred):\n",
        "        y_true = y_true.cpu().numpy().argmax(axis=1)\n",
        "        y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy().argmax(axis=1)\n",
        "        self.score = sklearn.metrics.accuracy_score(y_true, y_pred)\n",
        "    \n",
        "    @property\n",
        "    def avg(self):\n",
        "        return self.score\n",
        "\n",
        "class AverageMeter(object):\n",
        "    \"\"\"Computes and stores the average and current value\"\"\"\n",
        "    def __init__(self):\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.val = 0\n",
        "        self.avg = 0\n",
        "        self.sum = 0\n",
        "        self.count = 0\n",
        "\n",
        "    def update(self, val, n=1):\n",
        "        self.val = val\n",
        "        self.sum += val * n\n",
        "        self.count += n\n",
        "        self.avg = self.sum / self.count"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f6ed6a41",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:04.391673Z",
          "iopub.status.busy": "2022-01-20T21:23:04.390757Z",
          "iopub.status.idle": "2022-01-20T21:23:04.392575Z",
          "shell.execute_reply": "2022-01-20T21:23:04.393045Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.069940Z"
        },
        "papermill": {
          "duration": 0.0272,
          "end_time": "2022-01-20T21:23:04.393198",
          "exception": false,
          "start_time": "2022-01-20T21:23:04.365998",
          "status": "completed"
        },
        "tags": [],
        "id": "f6ed6a41"
      },
      "outputs": [],
      "source": [
        "class LabelSmoothing(nn.Module):\n",
        "    def __init__(self, smoothing = 0):\n",
        "        super(LabelSmoothing, self).__init__()\n",
        "        self.confidence = 1.0 - smoothing\n",
        "        self.smoothing = smoothing\n",
        "\n",
        "    def forward(self, x, target):\n",
        "        x = x.float()\n",
        "        target = target.float()\n",
        "        logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n",
        "        nll_loss = -logprobs * target\n",
        "        nll_loss = nll_loss.sum(-1)\n",
        "        smooth_loss = -logprobs.mean(dim=-1)\n",
        "        loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n",
        "        return loss.mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bd7e76f8",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:04.460521Z",
          "iopub.status.busy": "2022-01-20T21:23:04.452998Z",
          "iopub.status.idle": "2022-01-20T21:23:06.130759Z",
          "shell.execute_reply": "2022-01-20T21:23:06.129784Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.083754Z"
        },
        "papermill": {
          "duration": 1.719544,
          "end_time": "2022-01-20T21:23:06.130921",
          "exception": false,
          "start_time": "2022-01-20T21:23:04.411377",
          "status": "completed"
        },
        "tags": [],
        "id": "bd7e76f8"
      },
      "outputs": [],
      "source": [
        "import warnings\n",
        "from tqdm import tqdm\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "from catalyst.data.sampler import DistributedSamplerWrapper, BalanceClassSampler\n",
        "\n",
        "class GPUFitter:\n",
        "    \n",
        "    def __init__(self, model, device, config):\n",
        "        if not os.path.exists('node_submissions'):\n",
        "            os.makedirs('node_submissions')\n",
        "\n",
        "        self.config = config\n",
        "        self.epoch = 0\n",
        "        self.log_path = 'log.txt'\n",
        "\n",
        "        self.model = model\n",
        "        self.device = device\n",
        "\n",
        "        param_optimizer = list(self.model.named_parameters())\n",
        "        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
        "        optimizer_grouped_parameters = [\n",
        "            {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n",
        "            {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
        "        ]\n",
        "\n",
        "        self.optimizer = AdamW(optimizer_grouped_parameters, lr=config.lr)\n",
        "        self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)\n",
        "\n",
        "        self.criterion = config.criterion\n",
        "        print(f'Fitter prepared. Device is {self.device}')\n",
        "\n",
        "    def fit(self, train_loader, validation_loader):\n",
        "        for e in range(self.config.n_epochs):\n",
        "            if self.config.verbose:\n",
        "                lr = self.optimizer.param_groups[0]['lr']\n",
        "                timestamp = datetime.utcnow().isoformat()\n",
        "                print(f'\\n{timestamp}\\nLR: {lr}')\n",
        "\n",
        "            t = time.time()\n",
        "            para_loader = train_loader\n",
        "            losses, final_scores = self.train_one_epoch(para_loader)\n",
        "            \n",
        "            print(f'[RESULT]: Train. Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n",
        "\n",
        "            t = time.time()\n",
        "            para_loader = validation_loader\n",
        "            losses, final_scores = self.validation(para_loader)\n",
        "\n",
        "            print(f'[RESULT]: Validation. Epoch: {self.epoch}, loss: {losses.avg:.5f}, final_score: {final_scores.avg:.5f}, time: {(time.time() - t):.5f}')\n",
        "\n",
        "            if self.config.validation_scheduler:\n",
        "                self.scheduler.step(metrics=final_scores.avg)\n",
        "\n",
        "            self.epoch += 1\n",
        "    \n",
        "    def run_tuning_and_inference(self, test_loader, validation_tune_loader):\n",
        "        for e in range(2):\n",
        "            self.optimizer.param_groups[0]['lr'] = self.config.lr / (e + 1)\n",
        "            para_loader = validation_tune_loader\n",
        "            losses, final_scores = self.train_one_epoch(para_loader)\n",
        "            para_loader = test_loader\n",
        "            self.run_inference(para_loader)\n",
        "\n",
        "    def validation(self, val_loader):\n",
        "        self.model.eval()\n",
        "        losses = AverageMeter()\n",
        "        final_scores = AucMeter()\n",
        "\n",
        "        t = time.time()\n",
        "        for step, (targets, inputs, attention_masks) in tqdm(enumerate(val_loader)):\n",
        "            if self.config.verbose:\n",
        "                if step % self.config.verbose_step == 0:\n",
        "                    print(\n",
        "                        f'Valid Step {step}, loss: ' + \\\n",
        "                        f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \\\n",
        "                        f'time: {(time.time() - t):.5f}'\n",
        "                    )\n",
        "            with torch.no_grad():\n",
        "                inputs = inputs.to(self.device, dtype=torch.long) \n",
        "                attention_masks = attention_masks.to(self.device, dtype=torch.long) \n",
        "                targets = targets.to(self.device, dtype=torch.float) \n",
        "\n",
        "                outputs = self.model(inputs, attention_masks)\n",
        "                loss = self.criterion(outputs, targets)\n",
        "                \n",
        "                batch_size = inputs.size(0)\n",
        "\n",
        "                final_scores.update(targets, outputs)\n",
        "                losses.update(loss.detach().item(), batch_size)\n",
        "                \n",
        "        return losses, final_scores\n",
        "         \n",
        "    def train_one_epoch(self, train_loader):\n",
        "        self.model.train()\n",
        "\n",
        "        losses = AverageMeter()\n",
        "        final_scores = AucMeter()\n",
        "        t = time.time()\n",
        "        for step, (targets, inputs, attention_masks) in tqdm(enumerate(train_loader)):   \n",
        "            if self.config.verbose:\n",
        "                if step % self.config.verbose_step == 0:\n",
        "                    print(\n",
        "                        f'Train Step {step}, loss: ' + \\\n",
        "                        f'{losses.avg:.5f}, final_score: {final_scores.avg:.5f}, ' + \\\n",
        "                        f'time: {(time.time() - t):.5f}'\n",
        "                    )\n",
        "\n",
        "            inputs = inputs.to(self.device, dtype=torch.long)\n",
        "            attention_masks = attention_masks.to(self.device, dtype=torch.long)\n",
        "            targets = targets.to(self.device, dtype=torch.float)\n",
        "\n",
        "            self.optimizer.zero_grad()\n",
        "\n",
        "            outputs = self.model(inputs, attention_masks)\n",
        "            loss = self.criterion(outputs, targets)\n",
        "\n",
        "            batch_size = inputs.size(0)\n",
        "            \n",
        "            final_scores.update(targets, outputs)\n",
        "            \n",
        "            losses.update(loss.detach().item(), batch_size)\n",
        "\n",
        "            loss.backward()\n",
        "            self.optimizer.step()\n",
        "\n",
        "            if self.config.step_scheduler:\n",
        "                self.scheduler.step()\n",
        "        \n",
        "        self.model.eval()\n",
        "        self.save('last-checkpoint.bin')\n",
        "        return losses, final_scores\n",
        "\n",
        "    def run_inference(self, test_loader):\n",
        "        self.model.eval()\n",
        "        result = {'id': [], 'topic': []}\n",
        "        l = []\n",
        "        t = time.time()\n",
        "        for step, (ids, inputs, attention_masks) in enumerate(test_loader):\n",
        "            if self.config.verbose:\n",
        "                if step % self.config.verbose_step == 0:\n",
        "                    print(f'Prediction Step {step}, time: {(time.time() - t):.5f}')\n",
        "\n",
        "            with torch.no_grad():\n",
        "                inputs = inputs.to(self.device, dtype=torch.long) \n",
        "                attention_masks = attention_masks.to(self.device, dtype=torch.long)\n",
        "                outputs = self.model(inputs, attention_masks)\n",
        "                topics = nn.functional.sigmoid(outputs).cpu().numpy()\n",
        "                _ , topics = torch.topk(torch.tensor(topics), dim = 1, k = 3)\n",
        "                topics = np.array(topics)\n",
        "            l.extend(topics)\n",
        "        return np.array(l) \n",
        "    \n",
        "    def save(self, path):        \n",
        "        torch.save(self.model.state_dict(), path)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ffb21a17",
      "metadata": {
        "papermill": {
          "duration": 0.017832,
          "end_time": "2022-01-20T21:23:06.166474",
          "exception": false,
          "start_time": "2022-01-20T21:23:06.148642",
          "status": "completed"
        },
        "tags": [],
        "id": "ffb21a17"
      },
      "source": [
        "### Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c0ed7503",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:06.209791Z",
          "iopub.status.busy": "2022-01-20T21:23:06.208890Z",
          "iopub.status.idle": "2022-01-20T21:23:06.210873Z",
          "shell.execute_reply": "2022-01-20T21:23:06.211293Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.810980Z"
        },
        "papermill": {
          "duration": 0.027556,
          "end_time": "2022-01-20T21:23:06.211457",
          "exception": false,
          "start_time": "2022-01-20T21:23:06.183901",
          "status": "completed"
        },
        "tags": [],
        "id": "c0ed7503"
      },
      "outputs": [],
      "source": [
        "class TopicSimpleNNModel(nn.Module):\n",
        "\n",
        "    def __init__(self):\n",
        "        super(TopicSimpleNNModel, self).__init__()\n",
        "        self.backbone = AutoModel.from_pretrained(BACKBONE_PATH)\n",
        "        self.dropout = nn.Dropout(0.3)\n",
        "        self.linear = nn.Linear(in_features=self.backbone.pooler.dense.out_features*2,out_features=8)\n",
        "        \n",
        "    def forward(self, input_ids, attention_masks):\n",
        "        seq_x, _= self.backbone(input_ids=input_ids, attention_mask=attention_masks, return_dict=False)\n",
        "        apool = torch.mean(seq_x, 1)\n",
        "        mpool, _ = torch.max(seq_x, 1)\n",
        "        x = torch.cat((apool, mpool), 1)\n",
        "        x = self.dropout(x)\n",
        "        return self.linear(x)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "909aa876",
      "metadata": {
        "papermill": {
          "duration": 0.018825,
          "end_time": "2022-01-20T21:23:06.248483",
          "exception": false,
          "start_time": "2022-01-20T21:23:06.229658",
          "status": "completed"
        },
        "tags": [],
        "id": "909aa876"
      },
      "source": [
        "# Custom Config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3f819273",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:06.291527Z",
          "iopub.status.busy": "2022-01-20T21:23:06.290666Z",
          "iopub.status.idle": "2022-01-20T21:23:23.813003Z",
          "shell.execute_reply": "2022-01-20T21:23:23.812567Z",
          "shell.execute_reply.started": "2022-01-20T21:13:24.821106Z"
        },
        "papermill": {
          "duration": 17.545712,
          "end_time": "2022-01-20T21:23:23.813138",
          "exception": false,
          "start_time": "2022-01-20T21:23:06.267426",
          "status": "completed"
        },
        "tags": [],
        "colab": {
          "referenced_widgets": [
            "f708f95126544a3bbb11a5bfc2140bb2"
          ]
        },
        "id": "3f819273",
        "outputId": "98b762f4-f30f-4059-ba04-84de199011e3"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f708f95126544a3bbb11a5bfc2140bb2",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/316M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilroberta-base were not used when initializing RobertaModel: ['lm_head.dense.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.bias']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "net = TopicSimpleNNModel()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7328c92b",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:23.857786Z",
          "iopub.status.busy": "2022-01-20T21:23:23.856873Z",
          "iopub.status.idle": "2022-01-20T21:23:23.859613Z",
          "shell.execute_reply": "2022-01-20T21:23:23.859117Z",
          "shell.execute_reply.started": "2022-01-20T21:13:26.737259Z"
        },
        "papermill": {
          "duration": 0.028525,
          "end_time": "2022-01-20T21:23:23.859738",
          "exception": false,
          "start_time": "2022-01-20T21:23:23.831213",
          "status": "completed"
        },
        "tags": [],
        "id": "7328c92b"
      },
      "outputs": [],
      "source": [
        "class TrainGlobalConfig:\n",
        "    num_workers = 0 \n",
        "    batch_size = 16\n",
        "    n_epochs = 4\n",
        "    lr = 1e-5\n",
        "\n",
        "    # -------------------\n",
        "    verbose = True\n",
        "    verbose_step = 50\n",
        "    # -------------------\n",
        "\n",
        "    # --------------------\n",
        "    step_scheduler = False  # do scheduler.step after optimizer.step\n",
        "    validation_scheduler = True  # do scheduler.step after validation stage loss\n",
        "    SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau\n",
        "    scheduler_params = dict(\n",
        "        mode='max',\n",
        "        factor=0.7,\n",
        "        patience=0,\n",
        "        verbose=False, \n",
        "        threshold=0.0001,\n",
        "        threshold_mode='abs',\n",
        "        cooldown=0, \n",
        "        min_lr=1e-8,\n",
        "        eps=1e-08\n",
        "    )\n",
        "    # --------------------\n",
        "\n",
        "    # -------------------\n",
        "    criterion = LabelSmoothing()\n",
        "    # -------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "39425b19",
      "metadata": {
        "papermill": {
          "duration": 0.018002,
          "end_time": "2022-01-20T21:23:23.896670",
          "exception": false,
          "start_time": "2022-01-20T21:23:23.878668",
          "status": "completed"
        },
        "tags": [],
        "id": "39425b19"
      },
      "source": [
        "### Main method"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "edc2584b",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:23:23.943788Z",
          "iopub.status.busy": "2022-01-20T21:23:23.943089Z",
          "iopub.status.idle": "2022-01-20T21:32:51.190966Z",
          "shell.execute_reply": "2022-01-20T21:32:51.190445Z",
          "shell.execute_reply.started": "2022-01-20T21:13:26.745847Z"
        },
        "papermill": {
          "duration": 567.27585,
          "end_time": "2022-01-20T21:32:51.191189",
          "exception": false,
          "start_time": "2022-01-20T21:23:23.915339",
          "status": "completed"
        },
        "tags": [],
        "id": "edc2584b",
        "outputId": "575e4c98-c1bd-4868-8ccf-e8061c93c44c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitter prepared. Device is cuda:0\n",
            "\n",
            "2022-01-20T21:23:29.032498\n",
            "LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.04076\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:23,  2.24it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 1.24328, final_score: 1.00000, time: 23.29675\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:45,  2.21it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 1.20798, final_score: 0.18750, time: 45.84786\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:08,  2.20it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Train. Epoch: 0, loss: 0.96394, final_score: 1.00000, time: 69.01419\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1it [00:00,  6.11it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02062\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "51it [00:08,  6.02it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 50, loss: 0.36435, final_score: 1.00000, time: 8.34846\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "101it [00:16,  5.84it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 100, loss: 0.31907, final_score: 0.87500, time: 16.82298\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [00:25,  5.94it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Validation. Epoch: 0, loss: 0.29542, final_score: 1.00000, time: 25.26708\n",
            "\n",
            "2022-01-20T21:25:03.314243\n",
            "LR: 1e-05\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02948\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:22,  2.22it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 0.30566, final_score: 1.00000, time: 22.44093\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:44,  2.20it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 0.26425, final_score: 0.87500, time: 45.01461\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:07,  2.22it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Train. Epoch: 1, loss: 0.23031, final_score: 1.00000, time: 68.47971\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1it [00:00,  6.07it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02350\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "51it [00:08,  6.03it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 50, loss: 0.25812, final_score: 1.00000, time: 8.35039\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "101it [00:16,  5.84it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 100, loss: 0.19495, final_score: 0.93750, time: 16.77101\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [00:25,  5.97it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Validation. Epoch: 1, loss: 0.17273, final_score: 1.00000, time: 25.13326\n",
            "\n",
            "2022-01-20T21:26:36.928097\n",
            "LR: 7e-06\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02170\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:22,  2.24it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 0.20220, final_score: 1.00000, time: 22.46948\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:44,  2.21it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 0.17099, final_score: 0.93750, time: 44.98788\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:07,  2.22it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Train. Epoch: 2, loss: 0.15413, final_score: 1.00000, time: 68.68927\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1it [00:00,  5.67it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 0, loss: 0.00000, final_score: 0.00000, time: 0.04031\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "51it [00:08,  6.02it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 50, loss: 0.18103, final_score: 1.00000, time: 8.37000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "101it [00:16,  5.91it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 100, loss: 0.13299, final_score: 0.93750, time: 16.83754\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [00:25,  5.96it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Validation. Epoch: 2, loss: 0.11658, final_score: 1.00000, time: 25.19764\n",
            "\n",
            "2022-01-20T21:28:10.815714\n",
            "LR: 4.9e-06\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02106\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:22,  2.23it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 0.15727, final_score: 1.00000, time: 22.45092\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:44,  2.21it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 0.12673, final_score: 0.93750, time: 44.98556\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:07,  2.22it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Train. Epoch: 3, loss: 0.11331, final_score: 1.00000, time: 68.41001\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "1it [00:00,  6.08it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02791\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "51it [00:08,  5.95it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 50, loss: 0.12938, final_score: 1.00000, time: 8.48131\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "101it [00:17,  5.90it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Valid Step 100, loss: 0.09174, final_score: 0.93750, time: 16.97485\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [00:25,  5.92it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[RESULT]: Validation. Epoch: 3, loss: 0.08144, final_score: 1.00000, time: 25.35253\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.02383\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:22,  2.24it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 0.13401, final_score: 1.00000, time: 22.48259\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:45,  2.21it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 0.10570, final_score: 0.93750, time: 45.06919\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:07,  2.22it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Prediction Step 0, time: 0.01899\n",
            "Prediction Step 50, time: 8.31959\n",
            "Prediction Step 100, time: 16.67188\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 0, loss: 0.00000, final_score: 0.00000, time: 0.01831\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "50it [00:22,  2.23it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 50, loss: 0.08088, final_score: 1.00000, time: 22.46386\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100it [00:44,  2.20it/s]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Step 100, loss: 0.06110, final_score: 0.93750, time: 44.99648\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "150it [01:07,  2.23it/s]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Prediction Step 0, time: 0.01582\n",
            "Prediction Step 50, time: 8.36469\n",
            "Prediction Step 100, time: 16.67850\n"
          ]
        }
      ],
      "source": [
        "device = 'cuda:0'\n",
        "net.to(device)\n",
        "from torch.utils.data.distributed import DistributedSampler\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    train_dataset,\n",
        "    batch_size=TrainGlobalConfig.batch_size,\n",
        "    pin_memory=False,\n",
        "    drop_last=True,\n",
        "    num_workers=TrainGlobalConfig.num_workers,\n",
        ")\n",
        "\n",
        "validation_loader = torch.utils.data.DataLoader(\n",
        "    train_dataset,\n",
        "    batch_size=TrainGlobalConfig.batch_size,\n",
        "    pin_memory=False,\n",
        "    drop_last=False,\n",
        "    num_workers=TrainGlobalConfig.num_workers\n",
        ")\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    train_dataset,\n",
        "    batch_size=TrainGlobalConfig.batch_size,\n",
        "    pin_memory=False,\n",
        "    drop_last=False,\n",
        "    num_workers=TrainGlobalConfig.num_workers\n",
        ")\n",
        "\n",
        "fitter = GPUFitter(model=net, device=device, config=TrainGlobalConfig)\n",
        "fitter.fit(train_loader, train_loader)\n",
        "fitter.run_tuning_and_inference(train_loader, train_loader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b185840e",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:32:52.064166Z",
          "iopub.status.busy": "2022-01-20T21:32:52.063301Z",
          "iopub.status.idle": "2022-01-20T21:33:16.882173Z",
          "shell.execute_reply": "2022-01-20T21:33:16.883179Z",
          "shell.execute_reply.started": "2022-01-20T21:15:02.685649Z"
        },
        "papermill": {
          "duration": 25.274827,
          "end_time": "2022-01-20T21:33:16.883410",
          "exception": false,
          "start_time": "2022-01-20T21:32:51.608583",
          "status": "completed"
        },
        "tags": [],
        "id": "b185840e",
        "outputId": "ebe5d2ac-6a45-425e-bd03-9d74e991e649"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Prediction Step 0, time: 0.01764\n",
            "Prediction Step 50, time: 8.19521\n",
            "Prediction Step 100, time: 16.59983\n"
          ]
        }
      ],
      "source": [
        "result = fitter.run_inference(train_loader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1c775fa1",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-01-20T21:33:17.767423Z",
          "iopub.status.busy": "2022-01-20T21:33:17.766535Z",
          "iopub.status.idle": "2022-01-20T21:33:17.775744Z",
          "shell.execute_reply": "2022-01-20T21:33:17.776165Z",
          "shell.execute_reply.started": "2022-01-20T21:19:09.196617Z"
        },
        "papermill": {
          "duration": 0.446543,
          "end_time": "2022-01-20T21:33:17.776326",
          "exception": false,
          "start_time": "2022-01-20T21:33:17.329783",
          "status": "completed"
        },
        "tags": [],
        "id": "1c775fa1"
      },
      "outputs": [],
      "source": [
        "w = pd.DataFrame(result)\n",
        "w.to_csv('out.csv',index = False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "63f6cef9",
      "metadata": {
        "papermill": {
          "duration": 0.415552,
          "end_time": "2022-01-20T21:33:18.609859",
          "exception": false,
          "start_time": "2022-01-20T21:33:18.194307",
          "status": "completed"
        },
        "tags": [],
        "id": "63f6cef9"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.12"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 642.70975,
      "end_time": "2022-01-20T21:33:22.478749",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2022-01-20T21:22:39.768999",
      "version": "2.3.3"
    },
    "colab": {
      "name": "training-super-fast-tiny-bert.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}