notebook_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0343a98b",
   "metadata": {},
   "source": [
    "## 시작"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9892bce4",
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# ------ LIBRARY -------#\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import sys\n",
    "import pandas as pd\n",
    "import re\n",
    "import cv2\n",
    "import json\n",
    "# torch\n",
    "import torch\n",
    "import torch.cuda.amp as amp\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.sampler import *\n",
    "from glob import glob\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts, CosineAnnealingLR, ReduceLROnPlateau, MultiStepLR, OneCycleLR\n",
    "\n",
    "import math\n",
    "import torch\n",
    "from torch.optim.optimizer import Optimizer, required\n",
    "import torch_optimizer as optim\n",
    "from collections import defaultdict\n",
    "import itertools as it\n",
    "\n",
    "import tqdm\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "from timeit import default_timer as timer\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_absolute_error, accuracy_score, log_loss\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch\n",
    "import timm\n",
    "from warmup_scheduler import GradualWarmupScheduler\n",
    "from adamp import AdamP, SGDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f87869f9",
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# class args\n",
    "class args:\n",
    "    # ---- factor ---- #\n",
    "    debug=False\n",
    "    amp = True\n",
    "    tta=0\n",
    "    gpu = '0,1'\n",
    "    img_size = [512, 512]\n",
    "    \n",
    "    batch_size=8#32#128\n",
    "    weight_decay=1e-6\n",
    "    n_fold=5\n",
    "    fold=3 # [0, 1, 2, 3, 4] # 원래는 3\n",
    "    \n",
    "    dir_ = f'./saved_models/'\n",
    "    pt = 'resnet50'#'gluon_seresnext50_32x4d'\n",
    "    exp_name = f'{img_size}_{pt}_' + 'trash_practice'\n",
    "    \n",
    "    warmup_factor=10\n",
    "    warmup_epo = 5\n",
    "    cosine_epo = 20\n",
    "    epochs= cosine_epo+warmup_epo\n",
    "    #epochs=freeze_epo + warmup_epo + cosine_epo\n",
    "    \n",
    "    scheduler = 'CosineAnnealingLR'#'warmupv2'\n",
    "    \n",
    "    start_lr = 1e-4 #3e-5\n",
    "    min_lr= 1e-6\n",
    "    \n",
    "    # ---- Dataset ---- #\n",
    "    \n",
    "    # factor\n",
    "    T_max = epochs\n",
    "    \n",
    "\n",
    "    # ---- Else ---- #\n",
    "    num_workers=8\n",
    "    seed=2021\n",
    "\n",
    "\n",
    "data_dir = './'\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = args.gpu\n",
    "device = torch.device(f\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "##----------------\n",
    "def set_seeds(seed=42):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False # for faster training, but not deterministic\n",
    "\n",
    "set_seeds(seed=args.seed)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "299e9265",
   "metadata": {
    "code_folding": [
     1,
     33,
     97,
     99,
     124,
     126,
     132
    ]
   },
   "outputs": [],
   "source": [
    "# - util - #\n",
    "def get_learning_rate(optimizer):\n",
    "    lr=[]\n",
    "    for param_group in optimizer.param_groups:\n",
    "        lr +=[ param_group['lr'] ]\n",
    "\n",
    "    assert(len(lr)==1) #we support only one param_group\n",
    "    lr = lr[0]\n",
    "\n",
    "    return lr\n",
    "\n",
    "def merge_json():\n",
    "    train_path = './data/train'\n",
    "    test_path = './data/test'\n",
    "\n",
    "    hand_gesture = pd.read_csv('./data/hand_gesture_pose.csv')\n",
    "    sample_submission = pd.read_csv('./data/sample_submission.csv')\n",
    "    #\n",
    "    train_folders = sorted(glob(train_path + '/*'), key = lambda x : int(x.split('/')[-1]))\n",
    "    test_folders  = sorted(glob(test_path + '/*'), key = lambda x : int(x.split('/')[-1]))\n",
    "    #\n",
    "    answers = []\n",
    "    for train_folder in train_folders[1:] :\n",
    "        json_path = glob(train_folder + '/*.json')[0]\n",
    "        js = json.load(open(json_path))\n",
    "        cat = js.get('action')[0]\n",
    "        cat_name = js.get('action')[1]\n",
    "        answers.append([train_folder.replace('./data',''),cat, cat_name])\n",
    "\n",
    "    df = pd.DataFrame(answers, columns = ['folder','pose_id', 'answer_name'])\n",
    "    df['folder'] = './data' + df['folder']\n",
    "    \n",
    "    return df\n",
    "def load_data(crop=False):\n",
    "    # train file\n",
    "    try:\n",
    "        print('load dataset')\n",
    "        train_df = pd.read_csv('./data/train.csv')\n",
    "        test_df = pd.read_csv('./data/test.csv')\n",
    "    except:\n",
    "        train_img_path = []\n",
    "        test_img_path = []\n",
    "        for (path, dir, files) in os.walk(\"./data/train\"):\n",
    "            for filename in files:\n",
    "                #ext = os.path.splitext(filename)[-1]\n",
    "                if ('ipynb' not in path)&('json' not in filename):\n",
    "                    train_img_path.append(\"%s/%s\" % (path, filename))\n",
    "\n",
    "        # test file\n",
    "        for (path, dir, files) in os.walk(\"./data/test\"):\n",
    "            for filename in files:\n",
    "                #ext = os.path.splitext(filename)[-1]\n",
    "                if ('ipynb' not in path)&('json' not in filename):\n",
    "                    test_img_path.append(\"%s/%s\" % (path, filename))\n",
    "\n",
    "        # load all json \n",
    "        df = merge_json()\n",
    "\n",
    "        # train & test\n",
    "        train_df = pd.DataFrame()\n",
    "        test_df = pd.DataFrame()\n",
    "\n",
    "        train_df['path'] = train_img_path\n",
    "        test_df['path'] = test_img_path\n",
    "\n",
    "        train_df['folder'] = train_df['path'].apply(lambda x: x.split(x.split('/')[-1])[0][:-1])\n",
    "        test_df['folder'] = test_df['path'].apply(lambda x: x.split(x.split('/')[-1])[0][:-1])\n",
    "\n",
    "        # merge target\n",
    "        train_df = pd.merge(train_df, df[['folder', 'pose_id']], how='left', on='folder')\n",
    "        train_df = train_df.dropna(axis=0).reset_index(drop=True) # drop 0 folder\n",
    "        train_df['pose_id'] = train_df['pose_id'].astype(int)\n",
    "\n",
    "        # encoding label\n",
    "        le =LabelEncoder()\n",
    "        train_df['target'] = le.fit_transform(train_df['pose_id'])\n",
    "\n",
    "        # split fold\n",
    "        kf = KFold(n_splits=5, random_state=42, shuffle=True)\n",
    "        train_df['fold'] = -1\n",
    "        for n_fold, (_,v_idx) in enumerate(kf.split(train_df)):\n",
    "            train_df.loc[v_idx, 'fold']  = n_fold\n",
    "        \n",
    "        # test_df\n",
    "        sub = pd.read_csv('./data/sample_submission.csv')\n",
    "        sub['folder'] = './data/test/'+sub['Image_Path'].apply(lambda x: x.split('test')[-1][1:])\n",
    "\n",
    "        test_df = pd.merge(test_df, sub, how='left', on='folder')\n",
    "        test_df = test_df.groupby('folder').first().reset_index()[['path','folder']]\n",
    "        \n",
    "        train_df.to_csv('./data/train.csv', index=False)\n",
    "        test_df.to_csv('./data/test.csv', index=False)\n",
    "        print('Saved train&test csv file!')\n",
    "    \n",
    "    \n",
    "    return train_df, test_df\n",
    "\n",
    "class Logger(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.terminal = sys.stdout  #stdout\n",
    "        self.file = None\n",
    "\n",
    "    def open(self, file, mode=None):\n",
    "        if mode is None: mode ='w'\n",
    "        self.file = open(file, mode)\n",
    "\n",
    "    def write(self, message, is_terminal=1, is_file=1 ):\n",
    "        if '\\r' in message: is_file=0\n",
    "\n",
    "        if is_terminal == 1:\n",
    "            self.terminal.write(message)\n",
    "            self.terminal.flush()\n",
    "            #time.sleep(1)\n",
    "\n",
    "        if is_file == 1:\n",
    "            self.file.write(message)\n",
    "            self.file.flush()\n",
    "\n",
    "    def flush(self):\n",
    "        # this flush method is needed for python 3 compatibility.\n",
    "        # this handles the flush command by doing nothing.\n",
    "        # you might want to specify some extra behavior here.\n",
    "        pass   \n",
    "def print_args(args, logger=None):\n",
    "    for k, v in vars(args).items():\n",
    "        if logger is not None:\n",
    "            logger.write('{:<16} : {}\\n'.format(k, v))\n",
    "        else:\n",
    "            print('{:<16} : {}'.format(k, v))\n",
    "            \n",
    "\n",
    "class GradualWarmupSchedulerV2(GradualWarmupScheduler):\n",
    "    def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None):\n",
    "        super(GradualWarmupSchedulerV2, self).__init__(optimizer, multiplier, total_epoch, after_scheduler)\n",
    "    def get_lr(self):\n",
    "        if self.last_epoch > self.total_epoch:\n",
    "            if self.after_scheduler:\n",
    "                if not self.finished:\n",
    "                    self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs]\n",
    "                    self.finished = True\n",
    "                return self.after_scheduler.get_lr()\n",
    "            return [base_lr * self.multiplier for base_lr in self.base_lrs]\n",
    "        if self.multiplier == 1.0:\n",
    "            return [base_lr * (float(self.last_epoch) / self.total_epoch) for base_lr in self.base_lrs]\n",
    "        else:\n",
    "            return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7be6f239",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fbff676f",
   "metadata": {},
   "source": [
    "# models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de6900c5",
   "metadata": {
    "code_folding": [
     3
    ],
    "tags": []
   },
   "outputs": [],
   "source": [
    "# ------------------------\n",
    "#  dataset\n",
    "# ------------------------\n",
    "class MotionDataSet(Dataset):\n",
    "    \n",
    "    def __init__(self, data, transform = None, test=False):\n",
    "        self.data = data # dataframe\n",
    "        self.test = test # bool\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    \n",
    "    def __getitem__(self,idx):\n",
    "        \n",
    "        file_name_input = self.data.iloc[idx]['path']\n",
    "        \n",
    "        if self.test: # test\n",
    "            images = cv2.imread(file_name_input)\n",
    "            \n",
    "            if self.transform:\n",
    "                transformed = self.transform(image=images)\n",
    "                images = transformed[\"image\"]/255.\n",
    "            return images\n",
    "            \n",
    "            \n",
    "        else: # train\n",
    "            images = cv2.imread(file_name_input)\n",
    "            targets = self.data.iloc[idx]['target']\n",
    "            if self.transform:\n",
    "                transformed = self.transform(image=images)\n",
    "                images = transformed[\"image\"]/255.\n",
    "        \n",
    "            \n",
    "            return images, targets\n",
    "           \n",
    "# ------------------------\n",
    "# transformers(Augmentation)\n",
    "# ------------------------\n",
    "from albumentations import (\n",
    "    HorizontalFlip, VerticalFlip, IAAPerspective, ShiftScaleRotate, CLAHE, RandomRotate90, ImageCompression,\n",
    "    Transpose, ShiftScaleRotate, Blur, OpticalDistortion, GridDistortion, HueSaturationValue,\n",
    "    IAAAdditiveGaussianNoise, GaussNoise, MotionBlur, MedianBlur, IAAPiecewiseAffine, RandomCrop, RandomResizedCrop,\n",
    "    IAASharpen, IAAEmboss, RandomBrightnessContrast, Flip, OneOf, Compose, Normalize, Cutout, CoarseDropout, ShiftScaleRotate, CenterCrop, Resize,\n",
    "    ChannelShuffle, LongestMaxSize, HueSaturationValue, ISONoise\n",
    ")\n",
    "\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "import albumentations as A\n",
    "def get_transforms(*, data):\n",
    "    if data == 'train':\n",
    "        return A.Compose([\n",
    "            Resize(args.img_size[0],args.img_size[1]),\n",
    "            Blur(),\n",
    "            RandomBrightnessContrast(),\n",
    "            HueSaturationValue(),\n",
    "#             A.OneOf([\n",
    "#                 ChannelShuffle(),\n",
    "#                 CLAHE()  \n",
    "#             ], p=0.6),\n",
    "            \n",
    "            CoarseDropout(max_holes=4, max_height=16, max_width=16, p=0.5),\n",
    "            ShiftScaleRotate(rotate_limit=0, p=0.5),\n",
    "            ImageCompression (quality_lower=40, quality_upper=80, p=0.5),\n",
    "            GaussNoise(p=0.5),\n",
    "            ToTensorV2(transpose_mask=False)\n",
    "        ],p=1.)\n",
    "    elif data == 'valid':\n",
    "        return A.Compose([\n",
    "            Resize(args.img_size[0],args.img_size[1]),\n",
    "            ToTensorV2(transpose_mask=False)\n",
    "        ],p=1.)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b47fb622",
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# ------------------------\n",
    "#  model\n",
    "# ------------------------\n",
    "class SAModels(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.model = timm.create_model(args.pt, pretrained=True)\n",
    "        modules = list(self.model.children())[:-1]\n",
    "        self.model = nn.Sequential(*modules)\n",
    "        if args.pt == 'resnet34':\n",
    "            self.fc = nn.Linear(512, 157)\n",
    "        else:\n",
    "            self.fc = nn.Linear(2048, 157)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        output = self.model(x)\n",
    "        output = self.fc(output) # (bs, 1)\n",
    "        return output\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2fd0f29",
   "metadata": {},
   "source": [
    "# training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89dee00d",
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# ------------------------\n",
    "#  scheduler\n",
    "# ------------------------\n",
    "def get_scheduler(optimizer):\n",
    "    if args.scheduler=='ReduceLROnPlateau':\n",
    "        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=args.factor, patience=args.patience, \n",
    "                                      min_lr = 1e-5, verbose=True, eps=args.eps)\n",
    "    elif args.scheduler=='CosineAnnealingLR':\n",
    "        print('scheduler : Cosineannealinglr')\n",
    "        scheduler = CosineAnnealingLR(optimizer, T_max=args.T_max, eta_min=args.min_lr, last_epoch=-1)\n",
    "    elif args.scheduler=='CosineAnnealingWarmRestarts':\n",
    "        scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=args.T_0, T_mult=1, eta_min=args.min_lr, last_epoch=-1)\n",
    "    elif args.scheduler == 'MultiStepLR':\n",
    "        scheduler = MultiStepLR(optimizer, milestones=args.decay_epoch, gamma= args.factor, verbose=True)\n",
    "    elif args.scheduler == 'OneCycleLR':\n",
    "        scheduler = OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3, \n",
    "                                      max_lr=1e-3, epochs=args.epochs, steps_per_epoch=len(train_loader))\n",
    "        \n",
    "    elif args.scheduler =='warmupv2':\n",
    "        print('gradual warmupv2')\n",
    "        scheduler_cosine=torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, args.cosine_epo)\n",
    "        scheduler_warmup=GradualWarmupSchedulerV2(optimizer, multiplier=args.warmup_factor, total_epoch=args.warmup_epo, after_scheduler=scheduler_cosine)\n",
    "        scheduler=scheduler_warmup  \n",
    "        \n",
    "    else:\n",
    "        scheduler = None\n",
    "        print('scheduler is None')\n",
    "    return scheduler\n",
    "\n",
    "def do_valid(net, valid_loader, tta=args.tta):\n",
    "\n",
    "    val_loss = 0\n",
    "    target_lst = []\n",
    "    pred_lst = []\n",
    "    logit = []\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "    net.eval()\n",
    "    start_timer = timer()\n",
    "    for t, (images, targets) in enumerate(tqdm.tqdm(valid_loader)):\n",
    "        images  = images.to(device)\n",
    "        targets  = targets.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            if args.amp:\n",
    "                with amp.autocast():\n",
    "                    # TTA\n",
    "                    if tta>1:\n",
    "                        output=0\n",
    "                        for t in range(tta):\n",
    "                            output+= net(images)/tta\n",
    "                            \n",
    "                    else:\n",
    "                        output = net(images)#.squeeze(1)\n",
    "\n",
    "                    # loss\n",
    "                    #loss = loss_fn(output, targets)\n",
    "\n",
    "            else:\n",
    "                output = net(images)#.squeeze(1)\n",
    "                loss = loss_fn(output, targets)\n",
    "            \n",
    "            #val_loss += loss\n",
    "            target_lst.append(targets.detach())\n",
    "            #pred_lst.extend(output.argmax(1).tolist())\n",
    "            pred_lst.append(output.detach())\n",
    "    \n",
    "    target_lst = torch.cat(target_lst, 0)\n",
    "    pred_lst = torch.cat(pred_lst, 0)\n",
    "    \n",
    "    val_mean_loss = loss_fn(pred_lst, target_lst)\n",
    "    #log_loss(np.eye(target_lst.shape[0])[target_lst], pred_lst) #val_loss / len(valid_loader)\n",
    "    validation_score = (target_lst== pred_lst.argmax(1)).sum()/target_lst.shape[0]\n",
    "    #accuracy_score(target_lst, pred_lst.argmax(1))\n",
    "\n",
    "    return val_mean_loss, validation_score, pred_lst\n",
    "def do_test(net, test_loader):\n",
    "\n",
    "    val_loss = 0\n",
    "    target_lst = []\n",
    "    pred_lst = []\n",
    "    logit = []\n",
    "    loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "    net.eval()\n",
    "    start_timer = timer()\n",
    "    for t, images in enumerate(tqdm.tqdm(test_loader)):\n",
    "        images  = images.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            if args.amp:\n",
    "                with amp.autocast():\n",
    "                    output = net(images)#.squeeze(1)\n",
    "\n",
    "            else:\n",
    "                output = net(images)#.squeeze(1)\n",
    "                loss = loss_fn(output, targets)\n",
    "            \n",
    "            pred_lst.append(output.detach())\n",
    "    \n",
    "    pred_lst = torch.cat(pred_lst, 0)\n",
    "    \n",
    "    return pred_lst\n",
    "\n",
    "def run_train(folds=3):\n",
    "    out_dir = args.dir_+ f'/fold{args.fold}/{args.exp_name}'\n",
    "    os.makedirs(out_dir, exist_ok=True)\n",
    "    log = Logger()\n",
    "    log.open(out_dir + '/log.train.txt', mode='a')\n",
    "    print_args(args, log)\n",
    "    log.write('\\n')\n",
    "    \n",
    "    # load dataset\n",
    "    train, test = load_data()    \n",
    "        \n",
    "    train_transform = get_transforms(data='train')\n",
    "    val_transform = get_transforms(data='valid')\n",
    "\n",
    "    # split fold\n",
    "    for n_fold in range(5):\n",
    "        if n_fold != folds:\n",
    "            print(f'{n_fold} fold pass'+'\\n')\n",
    "            continue\n",
    "            \n",
    "        if args.debug:\n",
    "            train = train.sample(1000).copy()\n",
    "        \n",
    "        train_data = train[train['fold']!=n_fold].reset_index(drop=True)\n",
    "        val_data = train[train['fold']==n_fold].reset_index(drop=True)\n",
    "\n",
    "        ## dataset ------------------------------------\n",
    "        train_dataset = MotionDataSet(data = train_data, transform=train_transform)\n",
    "        valid_dataset = MotionDataSet(data = val_data, transform= tta_transform if args.tta>1  else val_transform)\n",
    "        trainloader = DataLoader(dataset=train_dataset, batch_size=args.batch_size,\n",
    "                                 num_workers=8, shuffle=True, pin_memory=True)\n",
    "        validloader = DataLoader(dataset=valid_dataset, batch_size=args.batch_size, \n",
    "                                 num_workers=8, shuffle=False, pin_memory=True)\n",
    "        \n",
    "        ## net ----------------------------------------\n",
    "        scaler = amp.GradScaler()\n",
    "        net = SAModels()\n",
    "\n",
    "        net.to(device)\n",
    "        if len(args.gpu)>1:\n",
    "            net = nn.DataParallel(net)\n",
    "\n",
    "        # ------------------------\n",
    "        # loss\n",
    "        # ------------------------\n",
    "        loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "        # ------------------------\n",
    "        #  Optimizer\n",
    "        # ------------------------\n",
    "        optimizer = AdamP(net.parameters(), lr=args.start_lr, weight_decay=args.weight_decay)\n",
    "        scheduler = get_scheduler(optimizer)\n",
    "        \n",
    "        # ----\n",
    "        start_timer = timer()\n",
    "        best_score = 0\n",
    "        best_loss = 10000\n",
    "        best_epoch, best_epoch_loss = 0,0\n",
    "        \n",
    "\n",
    "        for epoch in range(1, args.epochs+1):\n",
    "            train_loss = 0\n",
    "            valid_loss = 0\n",
    "\n",
    "            target_lst = []\n",
    "            pred_lst = []\n",
    "            lr = get_learning_rate(optimizer)\n",
    "            log.write(f'-------------------')\n",
    "            log.write(f'{epoch}epoch start')\n",
    "            log.write(f'-------------------'+'\\n')\n",
    "            log.write(f'learning rate : {lr : .6f}')\n",
    "            for t, (images, targets) in enumerate(tqdm.tqdm(trainloader)):\n",
    "\n",
    "                # one iteration update  -------------\n",
    "                images  = images.to(device)\n",
    "                targets  = targets.to(device)\n",
    "                # ------------\n",
    "                net.train()\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                if args.amp:\n",
    "                    with amp.autocast():\n",
    "                        # output\n",
    "                        output = net(images)#.squeeze(1)\n",
    "\n",
    "                        # loss\n",
    "                        loss = loss_fn(output, targets)\n",
    "                        train_loss += loss\n",
    "\n",
    "\n",
    "                    scaler.scale(loss).backward()\n",
    "                    scaler.step(optimizer)\n",
    "                    scaler.update()\n",
    "\n",
    "                else:\n",
    "                    # output\n",
    "                    output = net(images)#.squeeze(1)\n",
    "\n",
    "                    # loss\n",
    "                    loss = loss_fn(output, targets)\n",
    "                    train_loss += loss\n",
    "\n",
    "                    # update\n",
    "                    loss.backward()\n",
    "                    optimizer.step()\n",
    "\n",
    "\n",
    "                # for calculate f1 score\n",
    "                target_lst.extend(targets.detach().cpu().numpy())\n",
    "                pred_lst.extend(output.argmax(1).tolist())\n",
    "\n",
    "\n",
    "            if scheduler is not None:\n",
    "                scheduler.step() \n",
    "            train_loss = train_loss / len(trainloader)\n",
    "            train_score = accuracy_score(target_lst, pred_lst)\n",
    "\n",
    "            # validation\n",
    "            valid_loss, valid_score, val_preds = do_valid(net, validloader)\n",
    "\n",
    "            if valid_loss < best_loss:\n",
    "                best_val_preds = val_preds\n",
    "                best_loss = valid_loss\n",
    "                best_epoch = epoch\n",
    "                print('best LOSS model saved'+'\\n')\n",
    "\n",
    "                torch.save(net.state_dict(), out_dir + f'/{folds}f_{best_epoch}e_{best_loss:.4f}_loss.pth')\n",
    "                \n",
    "                \n",
    "            log.write(f'train loss : {train_loss:.4f}, train ACC score : {train_score : .4f}'+'\\n')\n",
    "            log.write(f'valid loss : {valid_loss:.4f}, valid ACC score : {valid_score : .4f}'+'\\n')\n",
    "            \n",
    "\n",
    "\n",
    "        log.write(f'best epoch (ACC) : {best_epoch }'+'\\n')\n",
    "        log.write(f'best score : {best_score : .4f}'+'\\n')\n",
    "        log.write(f'best epoch (LOSS) : {best_epoch_loss }'+'\\n')\n",
    "        log.write(f'best score : {best_loss : .4f}'+'\\n')\n",
    "        \n",
    "        return best_val_preds\n",
    "    \n",
    "def run_test(ckpt = ''):\n",
    "    _, test = load_data()    \n",
    "        \n",
    "    val_transform = get_transforms(data='valid')\n",
    "    \n",
    "    test_dataset = MotionDataSet(data = test, transform=val_transform, test=True)\n",
    "\n",
    "    testloader = DataLoader(dataset=test_dataset, batch_size=args.batch_size, \n",
    "                             num_workers=8, shuffle=False, pin_memory=True)\n",
    "    \n",
    "    net = SAModels()\n",
    "    net.to(device)\n",
    "    if len(args.gpu)>1:\n",
    "        net = nn.DataParallel(net)\n",
    "    net.load_state_dict(torch.load(ckpt))\n",
    "    test_preds = do_test(net, testloader)\n",
    "    \n",
    "    return test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6fd9323-76a7-48e1-88ab-f2291d508864",
   "metadata": {
    "code_folding": [],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"Training\"\"\"\n",
    "if __name__ == '__main__':\n",
    "    # model 1\n",
    "    args.pt = 'resnet50'\n",
    "    args.img_size = [384, 768]\n",
    "    preds = run_train(folds=0)\n",
    "    \n",
    "    # model 2\n",
    "    args.img_size = [512, 512]\n",
    "    preds = run_train(folds=0)\n",
    "    \n",
    "    # model 3\n",
    "    preds = run_train(folds=3)\n",
    "    \n",
    "    # model 4\n",
    "    args.img_size = [384, 768]\n",
    "    args.pt = 'seresnet50'\n",
    "    preds = run_train(folds=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "648b767e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72e14298",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Inference\"\"\"\n",
    "if __name__ == '__main__':\n",
    "    test_preds=0\n",
    "    args.pt = 'resnet50'\n",
    "    \n",
    "    # model 1\n",
    "    args.img_size = [384, 768]\n",
    "    test_preds += run_test('./pretrained/384_768_2.pth')\n",
    "    \n",
    "    # model 2\n",
    "    args.img_size = [512, 512]\n",
    "    test_preds += run_test('./pretrained/512_1.pth')\n",
    "    \n",
    "    # model 3\n",
    "    test_preds += run_test('./pretrained/512_2.pth')\n",
    "    \n",
    "    # model 4\n",
    "    args.img_size = [384, 768]\n",
    "    args.pt = 'seresnet50'\n",
    "    test_preds += run_test('./pretrained/384_768_1.pth')\n",
    "    \n",
    "    # ensemble\n",
    "    preds= test_preds/4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7914ecf0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3cba9d8a",
   "metadata": {},
   "source": [
    "# submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94d2e26d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('./data/sample_submission.csv')\n",
    "sub.iloc[:,1:] = torch.softmax(preds,1).cpu().numpy()\n",
    "sub.to_csv('./submission/final_dacon_submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5569d4fe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}