model_train.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:18.715157Z",
          "iopub.status.busy": "2021-12-12T02:32:18.714432Z",
          "iopub.status.idle": "2021-12-12T02:32:21.026899Z",
          "shell.execute_reply": "2021-12-12T02:32:21.026144Z",
          "shell.execute_reply.started": "2021-12-12T02:32:18.715121Z"
        },
        "id": "aC1ZDUFZIcho"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from random import randint\n",
        "\n",
        "from torchvision import datasets, models, transforms\n",
        "import torchvision\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import random\n",
        "import os\n",
        "\n",
        "%matplotlib inline\n",
        "import matplotlib.pyplot as plt\n",
        "import json\n",
        "import cv2\n",
        "import logging\n",
        "\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
        "import imgaug.augmenters as iaa\n",
        "\n",
        "\n",
        "path_local = '../../../DATA/spoof_sets/CelebA_Spoof/'#'../input/celeba-spoof-for-face-antispoofing/CelebA_Spoof_/CelebA_Spoof/'\n",
        "PATH_TO_MODEL = '../../../../DATA/SolovyevDA/models/hub/mobilenet_v2.pt'#'../models/hub/mobilenet_v2.pt' '../models/hub/wide_resnet50_2.pt'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ROrjZ9OHIchq"
      },
      "outputs": [],
      "source": [
        "sometimes = lambda aug: iaa.Sometimes(0.5, aug)\n",
        "seq = iaa.Sequential([\n",
        "    iaa.SomeOf((0, 5),\n",
        "    [\n",
        "        iaa.AdditiveGaussianNoise(scale=(0, 0.2*255)),\n",
        "        sometimes(iaa.AdditiveLaplaceNoise(scale=(0, 0.2*255))),\n",
        "        iaa.CoarseDropout((0.0, 0.05), size_percent=(0.25, 0.35)),\n",
        "        iaa.JpegCompression(compression=(50, 75)),\n",
        "        sometimes(iaa.SaltAndPepper(0.1, per_channel=True)),\n",
        "        iaa.MotionBlur(k=15),\n",
        "        sometimes(iaa.GaussianBlur(sigma=(0.0, 3.0))),\n",
        "        sometimes(iaa.RemoveSaturation(0.25)),\n",
        "        sometimes(iaa.AddToHueAndSaturation((-10, 10), per_channel=True)),\n",
        "        sometimes(iaa.LogContrast(gain=(0.6, 1.4)))\n",
        "    ]),\n",
        "    iaa.size.PadToSquare(),\n",
        "    iaa.Rotate((0, 20)),\n",
        "    iaa.size.Resize((224, 224))\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XmmR6FBnIchr"
      },
      "outputs": [],
      "source": [
        "def get_class(x):\n",
        "    if x == 0:\n",
        "        return 0\n",
        "    return 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h61O9haLIchr"
      },
      "outputs": [],
      "source": [
        "def filter_df(df, lower_n = False, binary = False, n = 6, onlydisplay=False):\n",
        "    if onlydisplay:\n",
        "        df = df[[x in [0,7,8,9] for x in list(df['Class'])]].reset_index()\n",
        "    \n",
        "    if lower_n:\n",
        "        df = df[df['Class'] < n].reset_index()\n",
        "        \n",
        "    if binary:\n",
        "        df['Class'] = df['Class'].apply(lambda x: get_class(x))\n",
        "    \n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9aBqwHORIchs"
      },
      "outputs": [],
      "source": [
        "class FocalLoss(nn.modules.loss._WeightedLoss):\n",
        "    def __init__(self, weight=None, gamma=2, reduction='mean'):\n",
        "        super(FocalLoss, self).__init__(weight, reduction=reduction)\n",
        "        self.gamma = gamma\n",
        "        self.weight = weight\n",
        "        \n",
        "    def forward(self, input, target):\n",
        "        ce_loss = F.cross_entropy(input, target, reduction=self.reduction, weight=self.weight)\n",
        "        pt = torch.exp(-ce_loss)\n",
        "        focal_loss = ((1-pt)**self.gamma*ce_loss).mean()\n",
        "        \n",
        "        return focal_loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.028904Z",
          "iopub.status.busy": "2021-12-12T02:32:21.028645Z",
          "iopub.status.idle": "2021-12-12T02:32:21.03593Z",
          "shell.execute_reply": "2021-12-12T02:32:21.035291Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.028864Z"
        },
        "id": "5wo_5FD6Ichs"
      },
      "outputs": [],
      "source": [
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        seq.augment_image,\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        iaa.size.PadToSquare().augment_image,\n",
        "        transforms.ToPILImage(),\n",
        "        transforms.Resize((224, 224)),   \n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])  \n",
        "    ]),\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.037526Z",
          "iopub.status.busy": "2021-12-12T02:32:21.037127Z",
          "iopub.status.idle": "2021-12-12T02:32:21.107597Z",
          "shell.execute_reply": "2021-12-12T02:32:21.106731Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.037466Z"
        },
        "id": "ArB0ZHT0Ichs",
        "outputId": "be1a5610-6f7a-4dda-889c-72b3598dfaa5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cuda\n"
          ]
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.110376Z",
          "iopub.status.busy": "2021-12-12T02:32:21.110098Z",
          "iopub.status.idle": "2021-12-12T02:32:21.117431Z",
          "shell.execute_reply": "2021-12-12T02:32:21.116518Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.110331Z"
        },
        "id": "4gAZhAMJIchu"
      },
      "outputs": [],
      "source": [
        "def create_model(n_classes):\n",
        "    model = torchvision.models.mobilenet_v2(pretrained=False)\n",
        "    model.load_state_dict(torch.load(PATH_TO_MODEL))\n",
        "    model.classifier[1] = nn.Linear(1280, n_classes)\n",
        "    \n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ok4LsvFtIchu"
      },
      "outputs": [],
      "source": [
        "def random_blur(img, qual):\n",
        "    ksize = (qual, qual)\n",
        "    i = randint(1, 4)\n",
        "    w, h = img.shape[1], img.shape[0]\n",
        "    \n",
        "    if w // 2 == 0 or h // 2 == 0:\n",
        "        return img\n",
        "    \n",
        "    if i == 1:\n",
        "        img[:h//2, :w//2] = cv2.blur(img[:h//2, :w//2], ksize)\n",
        "    elif i == 2:\n",
        "        img[:h//2, w//2:] = cv2.blur(img[:h//2, w//2:], ksize)\n",
        "    elif i == 3:\n",
        "        img[h//2:, :w//2] = cv2.blur(img[h//2:, :w//2], ksize)\n",
        "    else:\n",
        "        img[h//2:, w//2:] = cv2.blur(img[h//2:, w//2:], ksize)\n",
        "    \n",
        "    return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RBIql4vdIchv"
      },
      "outputs": [],
      "source": [
        "def random_crop(img):\n",
        "    \n",
        "    i = randint(1, 4)\n",
        "    w, h = img.shape[1], img.shape[0]\n",
        "    \n",
        "    if i == 1:\n",
        "        return img[:h//2, :w//2]\n",
        "    elif i == 2:\n",
        "        return img[:h//2, w//2:]\n",
        "    elif i == 3:\n",
        "        return img[h//2:, :w//2]\n",
        "    else:\n",
        "        return img[h//2:, w//2:]\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.119606Z",
          "iopub.status.busy": "2021-12-12T02:32:21.119012Z",
          "iopub.status.idle": "2021-12-12T02:32:21.137835Z",
          "shell.execute_reply": "2021-12-12T02:32:21.137085Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.119566Z"
        },
        "id": "d9zvolyFIchw"
      },
      "outputs": [],
      "source": [
        "def pad(img, y1, y2, x1, x2, percent):\n",
        "    h, w = img.shape[0], img.shape[1]\n",
        "    if ((y2 - y1) * (x2 - x1)) / (h * w) >= percent:\n",
        "        return img\n",
        "    else:\n",
        "        newS = ((y2 - y1) * (x2 - x1)) / percent\n",
        "        newD = np.sqrt(newS).astype(int) // 2\n",
        "\n",
        "        center_x = (x1 + x2) / 2.\n",
        "        center_y = (y1 + y2) / 2.\n",
        "        \n",
        "        x1_pad = int(center_x - newD)\n",
        "        x2_pad = int(center_x + newD)\n",
        "        y1_pad = int(center_y - newD)\n",
        "        y2_pad = int(center_y + newD)\n",
        "       \n",
        "        l = x1_pad >= 0\n",
        "        t = y1_pad >= 0\n",
        "        r = x2_pad <= w\n",
        "        b = y2_pad <= h\n",
        "\n",
        "        if not l and r:\n",
        "            x1_pad = 0\n",
        "            x2_pad = newD * 2\n",
        "        elif not l and not r:\n",
        "            x1_pad = 0\n",
        "            x2_pad = w\n",
        "        elif not r and l:\n",
        "            x2_pad = w\n",
        "            x1_pad = w - newD * 2\n",
        "            \n",
        "            if x1_pad < 0:\n",
        "                x1_pad = 0\n",
        "        \n",
        "        if not t and b:\n",
        "            y1_pad = 0\n",
        "            y2_pad = newD * 2\n",
        "        elif not t and not b:\n",
        "            y1_pad = 0\n",
        "            y2_pad = h\n",
        "        elif not b and t:\n",
        "            y2_pad = h\n",
        "            y1_pad = h - newD * 2\n",
        "            \n",
        "            if y1_pad < 0:\n",
        "                y1_pad = 0\n",
        "    \n",
        "\n",
        "            \n",
        "        return img[y1_pad:y2_pad, x1_pad:x2_pad, :]\n",
        "            \n",
        "\n",
        "def read_image_with_padding_adap(image_path, p=0.3):\n",
        "\n",
        "    img = cv2.imread(image_path)\n",
        "    real_h,real_w,c = img.shape\n",
        "    assert os.path.exists(image_path[:-4] + '.txt'),'path not exists' + ' ' + image_path # check ur labeles in ur dataset, mb u need \"+ '_BB.txt'\" or smth else\n",
        "    \n",
        "    with open(image_path[:-4] + '.txt','r') as f:\n",
        "        material = f.readline()\n",
        "        try:\n",
        "            x,y,w,h = material.strip().split(' ')\n",
        "        except:\n",
        "            logging.info('Bounding Box of' + ' ' + image_path + ' ' + 'is wrong')  \n",
        "\n",
        "        try:\n",
        "            w = int(float(w))\n",
        "            h = int(float(h))\n",
        "            x = int(float(x))\n",
        "            y = int(float(y))\n",
        "\n",
        "            \n",
        "            if x < 0:\n",
        "              x = 0\n",
        "            \n",
        "            if y < 0:\n",
        "              y = 0\n",
        "\n",
        "            w = w - x\n",
        "            h = h - y\n",
        "            \n",
        "    \n",
        "            img = pad(img, y, y+h, x, x+w, p)\n",
        "            # insert here needed func for ur pipeline (crop/blur)\n",
        "\n",
        "        except:\n",
        "            logging.info('Cropping Bounding Box of' + ' ' + image_path + ' ' + 'goes wrong')\n",
        "\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "\n",
        "        \n",
        "    return img\n",
        "\n",
        "\n",
        "def read_image_with_padding_default(image_path):\n",
        "\n",
        "    img = cv2.imread(image_path)\n",
        "    real_h,real_w,c = img.shape\n",
        "    pad_scale = 0.35\n",
        "    assert os.path.exists(image_path[:-4] + '.txt'),'path not exists' + ' ' + image_path # check ur labeles in ur dataset, mb u need \"+ '_BB.txt'\" or smth else\n",
        "    \n",
        "    with open(image_path[:-4] + '.txt','r') as f:\n",
        "        material = f.readline()\n",
        "        try:\n",
        "            x,y,w,h = material.strip().split(' ')\n",
        "        except:\n",
        "            logging.info('Bounding Box of' + ' ' + image_path + ' ' + 'is wrong')  \n",
        "\n",
        "        try:\n",
        "            w = int(float(w))\n",
        "            h = int(float(h))\n",
        "            x = int(float(x))\n",
        "            y = int(float(y))\n",
        "\n",
        "            \n",
        "            if x < 0:\n",
        "              x = 0\n",
        "            \n",
        "            if y < 0:\n",
        "              y = 0\n",
        "\n",
        "            w = w - x\n",
        "            h = h - y\n",
        "            \n",
        "    \n",
        "                \n",
        "            # Make face bbox square\n",
        "            x1_sq = int(x - ((h - w) / 2))\n",
        "            x2_sq = int(x + w + ((h - w) / 2))\n",
        "            y1_sq = y\n",
        "            y2_sq = y + h\n",
        "            \n",
        "            h_sq = y2_sq - y1_sq\n",
        "            w_sq = x2_sq - x1_sq\n",
        "            x_sq = x1_sq\n",
        "            y_sq = y1_sq\n",
        "            \n",
        "            y1_sq = 0 if y_sq < 0 else y_sq\n",
        "            x1_sq = 0 if x_sq < 0 else x_sq\n",
        "            y2_sq = real_h if y2_sq > real_h else y2_sq \n",
        "            x2_sq = real_w if x2_sq > real_w else x2_sq\n",
        "            \n",
        "            # create padding coordinats\n",
        "            h_pad = h_sq + int(h_sq * pad_scale)\n",
        "            w_pad = w_sq + int(w_sq * pad_scale)\n",
        "            x_pad = x_sq - int(w_sq * pad_scale / 2)\n",
        "            y_pad = y_sq - int(h_sq * pad_scale / 2)\n",
        "            \n",
        "            y1_pad = 0 if y_pad < 0 else y_pad\n",
        "            x1_pad = 0 if x_pad < 0 else x_pad\n",
        "            y2_pad = real_h if y1_pad + h_pad > real_h else y_pad + h_pad\n",
        "            x2_pad = real_w if x1_pad + w_pad > real_w else x_pad + w_pad\n",
        "\n",
        "            # Crop face based on its bounding box\n",
        "            y1 = 0 if y < 0 else y\n",
        "            x1 = 0 if x < 0 else x \n",
        "            y2 = real_h if y1 + h > real_h else y + h\n",
        "            x2 = real_w if x1 + w > real_w else x + w\n",
        "            \n",
        "            # extract padding region\n",
        "            y1_outborder = (y1 == y1_pad)\n",
        "            x1_outborder = (x1 == x1_pad)\n",
        "            y2_outborder = (y2 == y2_pad)\n",
        "            x2_outborder = (x2 == x2_pad)  \n",
        "                \n",
        "            img = img[y1_pad:y2_pad,x1_pad:x2_pad,:]\n",
        "            # insert here needed func for ur pipeline (crop/blur)\n",
        "        except:\n",
        "            logging.info('Cropping Bounding Box of' + ' ' + image_path + ' ' + 'goes wrong')\n",
        "\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "\n",
        "        \n",
        "    return img\n",
        "\n",
        "\n",
        "def read_image_with_padding_face_only(image_path):\n",
        "\n",
        "    img = cv2.imread(image_path)\n",
        "    real_h,real_w,c = img.shape\n",
        "    assert os.path.exists(image_path[:-4] + '.txt'),'path not exists' + ' ' + image_path # check ur labeles in ur dataset, mb u need \"+ '_BB.txt'\" or smth else\n",
        "    \n",
        "    with open(image_path[:-4] + '.txt','r') as f:\n",
        "        material = f.readline()\n",
        "        try:\n",
        "            x,y,w,h = material.strip().split(' ')\n",
        "        except:\n",
        "            logging.info('Bounding Box of' + ' ' + image_path + ' ' + 'is wrong')  \n",
        "\n",
        "        try:\n",
        "            w = int(float(w))\n",
        "            h = int(float(h))\n",
        "            x = int(float(x))\n",
        "            y = int(float(y))\n",
        "\n",
        "            \n",
        "            if x < 0:\n",
        "              x = 0\n",
        "            \n",
        "            if y < 0:\n",
        "              y = 0\n",
        "\n",
        "            w = w - x\n",
        "            h = h - y\n",
        "            \n",
        "    \n",
        "            img = img[y:y+h, x:x+w, :]\n",
        "            # insert here needed func for ur pipeline (crop/blur)\n",
        "\n",
        "        except:\n",
        "            logging.info('Cropping Bounding Box of' + ' ' + image_path + ' ' + 'goes wrong')\n",
        "\n",
        "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "\n",
        "        \n",
        "    return img"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "XYb91hOOIchy",
        "outputId": "12fb7090-2847-4b37-8026-ca2d64292385"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "237471\n",
            "87349\n",
            "45411\n"
          ]
        }
      ],
      "source": [
        "df_train = pd.read_csv('train_v2.csv')\n",
        "df_train = filter_df(df_train, True, True, 6, False)\n",
        "print(len(df_train))\n",
        "\n",
        "df_val = pd.read_csv('val_v2.csv')\n",
        "df_val = filter_df(df_val, True, True, 6, False)\n",
        "print(len(df_val))\n",
        "\n",
        "df_test = pd.read_csv('test.csv')\n",
        "df_test = filter_df(df_test, True, True, 6, False)\n",
        "print(len(df_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.879142Z",
          "iopub.status.busy": "2021-12-12T02:32:21.87873Z",
          "iopub.status.idle": "2021-12-12T02:32:21.886592Z",
          "shell.execute_reply": "2021-12-12T02:32:21.885589Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.879102Z"
        },
        "id": "olEHD8OkIchz"
      },
      "outputs": [],
      "source": [
        "class CelebAspoofDataset(Dataset):\n",
        "    def __init__(self, root, meta, transform=None):\n",
        "        self.root = root\n",
        "        self.meta = meta # csv file with cols \"Filepath\", \"Class\"\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.meta)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        img_path = os.path.join(self.root, self.meta['Filepath'][idx])\n",
        "        face = read_image_with_padding_adap(img_path) # insert here needed func for reading image (default, adaptive or face only)\n",
        "        \n",
        "        \n",
        "        label = self.meta['Class'][idx]   \n",
        "        \n",
        "        if self.transform:\n",
        "            face = self.transform(face)\n",
        "            \n",
        "        return (face, label)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.888355Z",
          "iopub.status.busy": "2021-12-12T02:32:21.888027Z",
          "iopub.status.idle": "2021-12-12T02:32:21.901109Z",
          "shell.execute_reply": "2021-12-12T02:32:21.90032Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.88831Z"
        },
        "id": "cSdMZQGhIchz"
      },
      "outputs": [],
      "source": [
        "train = CelebAspoofDataset(path_local, df_train, data_transforms['train'])\n",
        "val = CelebAspoofDataset(path_local, df_val, data_transforms['test'])\n",
        "test = CelebAspoofDataset(path_local, df_test, data_transforms['test'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.904356Z",
          "iopub.status.busy": "2021-12-12T02:32:21.903797Z",
          "iopub.status.idle": "2021-12-12T02:32:21.911189Z",
          "shell.execute_reply": "2021-12-12T02:32:21.910507Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.904301Z"
        },
        "id": "UX227e9VIchz"
      },
      "outputs": [],
      "source": [
        "batch_size = 128\n",
        "epochs = 500"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.91554Z",
          "iopub.status.busy": "2021-12-12T02:32:21.915174Z",
          "iopub.status.idle": "2021-12-12T02:32:21.922482Z",
          "shell.execute_reply": "2021-12-12T02:32:21.921681Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.915515Z"
        },
        "id": "73HVexbsIchz"
      },
      "outputs": [],
      "source": [
        "train_loader = DataLoader(train, batch_size=batch_size, shuffle=True, num_workers=1)\n",
        "val_loader = DataLoader(val, batch_size=batch_size, shuffle=False, num_workers=1)\n",
        "test_loader = DataLoader(test, batch_size=batch_size, shuffle=False, num_workers=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:21.92416Z",
          "iopub.status.busy": "2021-12-12T02:32:21.923892Z",
          "iopub.status.idle": "2021-12-12T02:32:28.584973Z",
          "shell.execute_reply": "2021-12-12T02:32:28.58428Z",
          "shell.execute_reply.started": "2021-12-12T02:32:21.924123Z"
        },
        "scrolled": false,
        "id": "H2m5LMy4Ichz"
      },
      "outputs": [],
      "source": [
        "model = create_model(len(np.unique(list(df_train['Class'])))).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hc9sRkPKIch0",
        "outputId": "f86c5957-ab0c-42c7-84f1-8608fa0d01b9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[0.99516813 1.00487902]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "class_wts = compute_class_weight(class_weight='balanced', classes=np.unique(list(df_train['Class'])), y=list(df_train['Class']))\n",
        "\n",
        "print(class_wts)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:28.586908Z",
          "iopub.status.busy": "2021-12-12T02:32:28.586413Z",
          "iopub.status.idle": "2021-12-12T02:32:28.592893Z",
          "shell.execute_reply": "2021-12-12T02:32:28.592074Z",
          "shell.execute_reply.started": "2021-12-12T02:32:28.586871Z"
        },
        "id": "CRngJX6YIch0"
      },
      "outputs": [],
      "source": [
        "weights = torch.tensor(class_wts, dtype=torch.float)\n",
        "weights = weights.to(device)\n",
        "\n",
        "optimizer = optim.AdamW(model.parameters(), lr=0.001)\n",
        "scheduler = ReduceLROnPlateau(optimizer, factor=0.5, patience=2,verbose=True)\n",
        "criterion = FocalLoss(weight=weights)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:28.595931Z",
          "iopub.status.busy": "2021-12-12T02:32:28.595723Z",
          "iopub.status.idle": "2021-12-12T02:32:28.605317Z",
          "shell.execute_reply": "2021-12-12T02:32:28.604524Z",
          "shell.execute_reply.started": "2021-12-12T02:32:28.595898Z"
        },
        "id": "KXt_Vok5Ich0"
      },
      "outputs": [],
      "source": [
        "def train():\n",
        "  \n",
        "    model.train()\n",
        "\n",
        "    total_loss = 0.0\n",
        "  \n",
        "    total_preds=[]\n",
        "  \n",
        "    for step,batch in enumerate(train_loader):\n",
        "        if step % 50 == 0 and not step == 0:\n",
        "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_loader)))\n",
        "        batch = [r.to(device) for r in batch]\n",
        "        \n",
        "        images, labels = batch\n",
        "\n",
        "        model.zero_grad() \n",
        "        \n",
        "        preds = model(images)\n",
        "        \n",
        "        loss = criterion(preds, labels)\n",
        "        total_loss = total_loss + loss.item()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        preds=preds.detach().cpu().numpy()\n",
        "        total_preds.append(preds)\n",
        "\n",
        "    avg_loss = total_loss / len(train_loader)\n",
        "\n",
        "    total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "    return avg_loss, total_preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:28.60779Z",
          "iopub.status.busy": "2021-12-12T02:32:28.607373Z",
          "iopub.status.idle": "2021-12-12T02:32:28.618144Z",
          "shell.execute_reply": "2021-12-12T02:32:28.617426Z",
          "shell.execute_reply.started": "2021-12-12T02:32:28.607755Z"
        },
        "id": "KXgOFzFKIch0"
      },
      "outputs": [],
      "source": [
        "def evaluate(loader):\n",
        "  \n",
        "    print(\"\\nEvaluating...\")\n",
        "  \n",
        "    model.eval()\n",
        "\n",
        "    total_loss = 0.0\n",
        "  \n",
        "    total_preds = []\n",
        "\n",
        "    for step,batch in enumerate(loader):\n",
        "    \n",
        "        if step % 50 == 0 and not step == 0:\n",
        "                  \n",
        "            print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(loader)))\n",
        "\n",
        "        batch = [t.to(device) for t in batch]\n",
        "        \n",
        "        #faces, fulls, labels = batch\n",
        "        images, labels = batch\n",
        "\n",
        "        with torch.no_grad():\n",
        "      \n",
        "            #preds = model(faces, fulls)\n",
        "            preds = model(images)\n",
        "\n",
        "            loss = criterion(preds,labels)\n",
        "\n",
        "            total_loss = total_loss + loss.item()\n",
        "\n",
        "            preds = preds.detach().cpu().numpy()\n",
        "\n",
        "            total_preds.append(preds)\n",
        "\n",
        "    avg_loss = total_loss / len(loader) \n",
        "\n",
        "    total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "    return avg_loss, total_preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2021-12-12T02:32:28.621062Z",
          "iopub.status.busy": "2021-12-12T02:32:28.62057Z"
        },
        "id": "VOQB9C1zIch0",
        "outputId": "6374418f-612e-47b0-e602-ed7c755ba738"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            " Epoch 1 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.002\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.825\n",
            "\n",
            " Epoch 2 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.001\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.916\n",
            "\n",
            " Epoch 3 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.001\n",
            "Validation Loss: 0.004\n",
            "\n",
            "F-score on test-set: 0.716\n",
            "\n",
            " Epoch 4 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.001\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.857\n",
            "\n",
            " Epoch 5 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.862\n",
            "\n",
            " Epoch 6 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.001\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.791\n",
            "\n",
            " Epoch 7 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.803\n",
            "\n",
            " Epoch 8 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "Epoch     8: reducing learning rate of group 0 to 5.0000e-04.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.931\n",
            "\n",
            " Epoch 9 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.862\n",
            "\n",
            " Epoch 10 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.849\n",
            "\n",
            " Epoch 11 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.839\n",
            "\n",
            " Epoch 12 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.831\n",
            "\n",
            " Epoch 13 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.836\n",
            "\n",
            " Epoch 14 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.846\n",
            "\n",
            " Epoch 15 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.870\n",
            "\n",
            " Epoch 16 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.866\n",
            "\n",
            " Epoch 17 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.856\n",
            "\n",
            " Epoch 18 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.890\n",
            "\n",
            " Epoch 19 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "Epoch    19: reducing learning rate of group 0 to 2.5000e-04.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.886\n",
            "\n",
            " Epoch 20 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.902\n",
            "\n",
            " Epoch 21 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n",
            "  Batch 1,600  of  1,856.\n",
            "  Batch 1,650  of  1,856.\n",
            "  Batch 1,700  of  1,856.\n",
            "  Batch 1,750  of  1,856.\n",
            "  Batch 1,800  of  1,856.\n",
            "  Batch 1,850  of  1,856.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    683.\n",
            "  Batch   100  of    683.\n",
            "  Batch   150  of    683.\n",
            "  Batch   200  of    683.\n",
            "  Batch   250  of    683.\n",
            "  Batch   300  of    683.\n",
            "  Batch   350  of    683.\n",
            "  Batch   400  of    683.\n",
            "  Batch   450  of    683.\n",
            "  Batch   500  of    683.\n",
            "  Batch   550  of    683.\n",
            "  Batch   600  of    683.\n",
            "  Batch   650  of    683.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    355.\n",
            "  Batch   100  of    355.\n",
            "  Batch   150  of    355.\n",
            "  Batch   200  of    355.\n",
            "  Batch   250  of    355.\n",
            "  Batch   300  of    355.\n",
            "  Batch   350  of    355.\n",
            "\n",
            "Training Loss: 0.000\n",
            "Validation Loss: 0.000\n",
            "\n",
            "F-score on test-set: 0.858\n",
            "\n",
            " Epoch 22 / 500\n",
            "  Batch    50  of  1,856.\n",
            "  Batch   100  of  1,856.\n",
            "  Batch   150  of  1,856.\n",
            "  Batch   200  of  1,856.\n",
            "  Batch   250  of  1,856.\n",
            "  Batch   300  of  1,856.\n",
            "  Batch   350  of  1,856.\n",
            "  Batch   400  of  1,856.\n",
            "  Batch   450  of  1,856.\n",
            "  Batch   500  of  1,856.\n",
            "  Batch   550  of  1,856.\n",
            "  Batch   600  of  1,856.\n",
            "  Batch   650  of  1,856.\n",
            "  Batch   700  of  1,856.\n",
            "  Batch   750  of  1,856.\n",
            "  Batch   800  of  1,856.\n",
            "  Batch   850  of  1,856.\n",
            "  Batch   900  of  1,856.\n",
            "  Batch   950  of  1,856.\n",
            "  Batch 1,000  of  1,856.\n",
            "  Batch 1,050  of  1,856.\n",
            "  Batch 1,100  of  1,856.\n",
            "  Batch 1,150  of  1,856.\n",
            "  Batch 1,200  of  1,856.\n",
            "  Batch 1,250  of  1,856.\n",
            "  Batch 1,300  of  1,856.\n",
            "  Batch 1,350  of  1,856.\n",
            "  Batch 1,400  of  1,856.\n",
            "  Batch 1,450  of  1,856.\n",
            "  Batch 1,500  of  1,856.\n",
            "  Batch 1,550  of  1,856.\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import f1_score\n",
        "# set initial loss to infinite\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "# empty lists to store training and validation loss of each epoch\n",
        "train_losses=[]\n",
        "valid_losses=[]\n",
        "test_losses=[]\n",
        "\n",
        "ys = list(filter_df(df_test, False, True)['Class'])\n",
        "\n",
        "#for each epoch\n",
        "for epoch in range(epochs):\n",
        "     \n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "    train_loss, _ = train()\n",
        "\n",
        "    valid_loss, _ = evaluate(val_loader)\n",
        "    scheduler.step(valid_loss)\n",
        "\n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        \n",
        "    torch.save(model.state_dict(), 'pad_0.3/mobnet_v2_' + str(epoch) + '.pt')\n",
        "\n",
        "    train_losses.append(train_loss)\n",
        "    valid_losses.append(valid_loss)\n",
        "    \n",
        "    \n",
        "    with open('pad_0.3/train_loss.txt', 'a') as f:\n",
        "        f.write(str(train_loss) + '\\n')\n",
        "        \n",
        "        \n",
        "    with open('pad_0.3/valid_loss.txt', 'a') as f:\n",
        "        f.write(str(valid_loss) + '\\n')\n",
        "\n",
        "        \n",
        "    test_loss, preds = evaluate(test_loader)\n",
        "    test_losses.append(test_loss)\n",
        "    \n",
        "    \n",
        "    with open('pad_0.3/test_loss.txt', 'a') as f:\n",
        "        f.write(str(test_loss) + '\\n')\n",
        "        \n",
        "    preds = np.argmax(preds, axis = 1)\n",
        "    \n",
        "    f_score = f1_score(list(df_test['Class']), preds, average = 'macro')\n",
        "    \n",
        "\n",
        "    print(f'\\nTraining Loss: {train_loss:.3f}')\n",
        "    print(f'Validation Loss: {valid_loss:.3f}')\n",
        "    print(f'\\nF-score on test-set: {f_score:.3f}')\n",
        "\n",
        "print(f'\\nBest valid loss: {best_valid_loss:.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h1j0ek8DIch0"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1MWy0F_1Ich0"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python3 Self-Maintained",
      "language": "python",
      "name": "sm"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}