3_semantic_segmentation/3-7_PSPNet_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.5"
    },
    "colab": {
      "name": "3-7_PSPNet_training.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g7fOv9Gc5Dkg"
      },
      "source": [
        "# 3.7 学習と検証の実施\n",
        "\n",
        "- 本ファイルでは、PSPNetの学習と検証の実施を行います。AWSのGPUマシンで計算します。\n",
        "- p2.xlargeで約12時間かかります。\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QYZ3dHjo5Dkh"
      },
      "source": [
        "# 学習目標\n",
        "\n",
        "1.\tPSPNetの学習と検証を実装できるようになる\n",
        "2.\tセマンティックセグメンテーションのファインチューニングを理解する\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2kCZ8pIO5Dki"
      },
      "source": [
        "# 事前準備\n",
        "\n",
        "- 本書に従い学習済みモデルのファイル「pspnet50_ADE20K.pth」をダウンロードし、フォルダ「weights」に用意します。"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aokMFYco5Dki"
      },
      "source": [
        "# パッケージのimport\n",
        "import random\n",
        "import math\n",
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "import torch.utils.data as data\n",
        "import torch.nn as nn\n",
        "import torch.nn.init as init\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wvHo-7O-5Dkl"
      },
      "source": [
        "# 初期設定\n",
        "# Setup seeds\n",
        "torch.manual_seed(1234)\n",
        "np.random.seed(1234)\n",
        "random.seed(1234)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ejh1Rqxm5Dko"
      },
      "source": [
        "# DataLoader作成"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1lKxvOS65Dko"
      },
      "source": [
        "from utils.dataloader import make_datapath_list, DataTransform, VOCDataset\n",
        "\n",
        "# ファイルパスリスト作成\n",
        "rootpath = \"./data/VOCdevkit/VOC2012/\"\n",
        "train_img_list, train_anno_list, val_img_list, val_anno_list = make_datapath_list(\n",
        "    rootpath=rootpath)\n",
        "\n",
        "# Dataset作成\n",
        "# (RGB)の色の平均値と標準偏差\n",
        "color_mean = (0.485, 0.456, 0.406)\n",
        "color_std = (0.229, 0.224, 0.225)\n",
        "\n",
        "train_dataset = VOCDataset(train_img_list, train_anno_list, phase=\"train\", transform=DataTransform(\n",
        "    input_size=475, color_mean=color_mean, color_std=color_std))\n",
        "\n",
        "val_dataset = VOCDataset(val_img_list, val_anno_list, phase=\"val\", transform=DataTransform(\n",
        "    input_size=475, color_mean=color_mean, color_std=color_std))\n",
        "\n",
        "# DataLoader作成\n",
        "batch_size = 8\n",
        "\n",
        "train_dataloader = data.DataLoader(\n",
        "    train_dataset, batch_size=batch_size, shuffle=True)\n",
        "\n",
        "val_dataloader = data.DataLoader(\n",
        "    val_dataset, batch_size=batch_size, shuffle=False)\n",
        "\n",
        "# 辞書型変数にまとめる\n",
        "dataloaders_dict = {\"train\": train_dataloader, \"val\": val_dataloader}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dEAUQA9q5Dkr"
      },
      "source": [
        "# ネットワークモデル作成"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8AkC5NH5Dkr",
        "outputId": "9d661cec-bbc4-455c-ff02-9e6639385f11"
      },
      "source": [
        "from utils.pspnet import PSPNet\n",
        "\n",
        "# ファインチューニングでPSPNetを作成\n",
        "# ADE20Kデータセットの学習済みモデルを使用、ADE20Kはクラス数が150です\n",
        "net = PSPNet(n_classes=150)\n",
        "\n",
        "# ADE20K学習済みパラメータをロード\n",
        "state_dict = torch.load(\"./weights/pspnet50_ADE20K.pth\")\n",
        "net.load_state_dict(state_dict)\n",
        "\n",
        "# 分類用の畳み込み層を、出力数21のものにつけかえる\n",
        "n_classes = 21\n",
        "net.decode_feature.classification = nn.Conv2d(\n",
        "    in_channels=512, out_channels=n_classes, kernel_size=1, stride=1, padding=0)\n",
        "\n",
        "net.aux.classification = nn.Conv2d(\n",
        "    in_channels=256, out_channels=n_classes, kernel_size=1, stride=1, padding=0)\n",
        "\n",
        "# 付け替えた畳み込み層を初期化する。活性化関数がシグモイド関数なのでXavierを使用する。\n",
        "\n",
        "\n",
        "def weights_init(m):\n",
        "    if isinstance(m, nn.Conv2d):\n",
        "        nn.init.xavier_normal_(m.weight.data)\n",
        "        if m.bias is not None:  # バイアス項がある場合\n",
        "            nn.init.constant_(m.bias, 0.0)\n",
        "\n",
        "\n",
        "net.decode_feature.classification.apply(weights_init)\n",
        "net.aux.classification.apply(weights_init)\n",
        "\n",
        "\n",
        "print('ネットワーク設定完了：学習済みの重みをロードしました')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ネットワーク設定完了：学習済みの重みをロードしました\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXuKq8VP5Dku",
        "outputId": "7257e247-3e56-4f49-e72c-8ceb68357433"
      },
      "source": [
        "net"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PSPNet(\n",
              "  (feature_conv): FeatureMap_convolution(\n",
              "    (cbnr_1): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (cbnr_2): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (cbnr_3): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "  )\n",
              "  (feature_res_1): ResidualBlockPSP(\n",
              "    (block1): bottleNeckPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (cb_residual): conv2DBatchNorm(\n",
              "        (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block2): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block3): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "  )\n",
              "  (feature_res_2): ResidualBlockPSP(\n",
              "    (block1): bottleNeckPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (cb_residual): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block2): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block3): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block4): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "  )\n",
              "  (feature_dilated_res_1): ResidualBlockPSP(\n",
              "    (block1): bottleNeckPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (cb_residual): conv2DBatchNorm(\n",
              "        (conv): Conv2d(512, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block2): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block3): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block4): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block5): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block6): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(2, 2), dilation=(2, 2), bias=False)\n",
              "        (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "  )\n",
              "  (feature_dilated_res_2): ResidualBlockPSP(\n",
              "    (block1): bottleNeckPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(4, 4), dilation=(4, 4), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (cb_residual): conv2DBatchNorm(\n",
              "        (conv): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block2): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(4, 4), dilation=(4, 4), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (block3): bottleNeckIdentifyPSP(\n",
              "      (cbr_1): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cbr_2): conv2DBatchNormRelu(\n",
              "        (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(4, 4), dilation=(4, 4), bias=False)\n",
              "        (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace)\n",
              "      )\n",
              "      (cb_3): conv2DBatchNorm(\n",
              "        (conv): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "        (batchnorm): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "  )\n",
              "  (pyramid_pooling): PyramidPooling(\n",
              "    (avpool_1): AdaptiveAvgPool2d(output_size=6)\n",
              "    (cbr_1): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (avpool_2): AdaptiveAvgPool2d(output_size=3)\n",
              "    (cbr_2): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (avpool_3): AdaptiveAvgPool2d(output_size=2)\n",
              "    (cbr_3): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (avpool_4): AdaptiveAvgPool2d(output_size=1)\n",
              "    (cbr_4): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "  )\n",
              "  (decode_feature): DecodePSPFeature(\n",
              "    (cbr): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(4096, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (dropout): Dropout2d(p=0.1)\n",
              "    (classification): Conv2d(512, 21, kernel_size=(1, 1), stride=(1, 1))\n",
              "  )\n",
              "  (aux): AuxiliaryPSPlayers(\n",
              "    (cbr): conv2DBatchNormRelu(\n",
              "      (conv): Conv2d(1024, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (batchnorm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace)\n",
              "    )\n",
              "    (dropout): Dropout2d(p=0.1)\n",
              "    (classification): Conv2d(256, 21, kernel_size=(1, 1), stride=(1, 1))\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t5bYr52Z5Dkw"
      },
      "source": [
        "# 損失関数を定義"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C0Y6_cZh5Dkx"
      },
      "source": [
        "# 損失関数の設定\n",
        "class PSPLoss(nn.Module):\n",
        "    \"\"\"PSPNetの損失関数のクラスです。\"\"\"\n",
        "\n",
        "    def __init__(self, aux_weight=0.4):\n",
        "        super(PSPLoss, self).__init__()\n",
        "        self.aux_weight = aux_weight  # aux_lossの重み\n",
        "\n",
        "    def forward(self, outputs, targets):\n",
        "        \"\"\"\n",
        "        損失関数の計算。\n",
        "\n",
        "        Parameters\n",
        "        ----------\n",
        "        outputs : PSPNetの出力(tuple)\n",
        "            (output=torch.Size([num_batch, 21, 475, 475]), output_aux=torch.Size([num_batch, 21, 475, 475]))。\n",
        "\n",
        "        targets : [num_batch, 475, 475]\n",
        "            正解のアノテーション情報\n",
        "\n",
        "        Returns\n",
        "        -------\n",
        "        loss : テンソル\n",
        "            損失の値\n",
        "        \"\"\"\n",
        "\n",
        "        loss = F.cross_entropy(outputs[0], targets, reduction='mean')\n",
        "        loss_aux = F.cross_entropy(outputs[1], targets, reduction='mean')\n",
        "\n",
        "        return loss+self.aux_weight*loss_aux\n",
        "\n",
        "\n",
        "criterion = PSPLoss(aux_weight=0.4)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gevOccLt5Dkz"
      },
      "source": [
        "# 最適化手法を設定"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1HQuH4465Dk0"
      },
      "source": [
        "# ファインチューニングなので、学習率は小さく\n",
        "optimizer = optim.SGD([\n",
        "    {'params': net.feature_conv.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.feature_res_1.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.feature_res_2.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.feature_dilated_res_1.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.feature_dilated_res_2.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.pyramid_pooling.parameters(), 'lr': 1e-3},\n",
        "    {'params': net.decode_feature.parameters(), 'lr': 1e-2},\n",
        "    {'params': net.aux.parameters(), 'lr': 1e-2},\n",
        "], momentum=0.9, weight_decay=0.0001)\n",
        "\n",
        "\n",
        "# スケジューラーの設定\n",
        "def lambda_epoch(epoch):\n",
        "    max_epoch = 30\n",
        "    return math.pow((1-epoch/max_epoch), 0.9)\n",
        "\n",
        "\n",
        "scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_epoch)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X0C1CqUP5Dk3"
      },
      "source": [
        "# 学習・検証を実施する"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDo8kP3r5Dk3"
      },
      "source": [
        "# モデルを学習させる関数を作成\n",
        "\n",
        "\n",
        "def train_model(net, dataloaders_dict, criterion, scheduler, optimizer, num_epochs):\n",
        "\n",
        "    # GPUが使えるかを確認\n",
        "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "    print(\"使用デバイス：\", device)\n",
        "\n",
        "    # ネットワークをGPUへ\n",
        "    net.to(device)\n",
        "\n",
        "    # ネットワークがある程度固定であれば、高速化させる\n",
        "    torch.backends.cudnn.benchmark = True\n",
        "\n",
        "    # 画像の枚数\n",
        "    num_train_imgs = len(dataloaders_dict[\"train\"].dataset)\n",
        "    num_val_imgs = len(dataloaders_dict[\"val\"].dataset)\n",
        "    batch_size = dataloaders_dict[\"train\"].batch_size\n",
        "\n",
        "    # イテレーションカウンタをセット\n",
        "    iteration = 1\n",
        "    logs = []\n",
        "\n",
        "    # multiple minibatch\n",
        "    batch_multiplier = 3\n",
        "\n",
        "    # epochのループ\n",
        "    for epoch in range(num_epochs):\n",
        "\n",
        "        # 開始時刻を保存\n",
        "        t_epoch_start = time.time()\n",
        "        t_iter_start = time.time()\n",
        "        epoch_train_loss = 0.0  # epochの損失和\n",
        "        epoch_val_loss = 0.0  # epochの損失和\n",
        "\n",
        "        print('-------------')\n",
        "        print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
        "        print('-------------')\n",
        "\n",
        "        # epochごとの訓練と検証のループ\n",
        "        for phase in ['train', 'val']:\n",
        "            if phase == 'train':\n",
        "                net.train()  # モデルを訓練モードに\n",
        "                scheduler.step()  # 最適化schedulerの更新\n",
        "                optimizer.zero_grad()\n",
        "                print('（train）')\n",
        "\n",
        "            else:\n",
        "                if((epoch+1) % 5 == 0):\n",
        "                    net.eval()   # モデルを検証モードに\n",
        "                    print('-------------')\n",
        "                    print('（val）')\n",
        "                else:\n",
        "                    # 検証は5回に1回だけ行う\n",
        "                    continue\n",
        "\n",
        "            # データローダーからminibatchずつ取り出すループ\n",
        "            count = 0  # multiple minibatch\n",
        "            for imges, anno_class_imges in dataloaders_dict[phase]:\n",
        "                # ミニバッチがサイズが1だと、バッチノーマライゼーションでエラーになるのでさける\n",
        "                # issue #186より不要なのでコメントアウト\n",
        "                # if imges.size()[0] == 1:\n",
        "                #     continue\n",
        "\n",
        "                # GPUが使えるならGPUにデータを送る\n",
        "                imges = imges.to(device)\n",
        "                anno_class_imges = anno_class_imges.to(device)\n",
        "\n",
        "                \n",
        "                # multiple minibatchでのパラメータの更新\n",
        "                if (phase == 'train') and (count == 0):\n",
        "                    optimizer.step()\n",
        "                    optimizer.zero_grad()\n",
        "                    count = batch_multiplier\n",
        "\n",
        "                # 順伝搬（forward）計算\n",
        "                with torch.set_grad_enabled(phase == 'train'):\n",
        "                    outputs = net(imges)\n",
        "                    loss = criterion(\n",
        "                        outputs, anno_class_imges.long()) / batch_multiplier\n",
        "\n",
        "                    # 訓練時はバックプロパゲーション\n",
        "                    if phase == 'train':\n",
        "                        loss.backward()  # 勾配の計算\n",
        "                        count -= 1  # multiple minibatch\n",
        "\n",
        "                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示\n",
        "                            t_iter_finish = time.time()\n",
        "                            duration = t_iter_finish - t_iter_start\n",
        "                            print('イテレーション {} || Loss: {:.4f} || 10iter: {:.4f} sec.'.format(\n",
        "                                iteration, loss.item()/batch_size*batch_multiplier, duration))\n",
        "                            t_iter_start = time.time()\n",
        "\n",
        "                        epoch_train_loss += loss.item() * batch_multiplier\n",
        "                        iteration += 1\n",
        "\n",
        "                    # 検証時\n",
        "                    else:\n",
        "                        epoch_val_loss += loss.item() * batch_multiplier\n",
        "\n",
        "        # epochのphaseごとのlossと正解率\n",
        "        t_epoch_finish = time.time()\n",
        "        print('-------------')\n",
        "        print('epoch {} || Epoch_TRAIN_Loss:{:.4f} ||Epoch_VAL_Loss:{:.4f}'.format(\n",
        "            epoch+1, epoch_train_loss/num_train_imgs, epoch_val_loss/num_val_imgs))\n",
        "        print('timer:  {:.4f} sec.'.format(t_epoch_finish - t_epoch_start))\n",
        "        t_epoch_start = time.time()\n",
        "\n",
        "        # ログを保存\n",
        "        log_epoch = {'epoch': epoch+1, 'train_loss': epoch_train_loss /\n",
        "                     num_train_imgs, 'val_loss': epoch_val_loss/num_val_imgs}\n",
        "        logs.append(log_epoch)\n",
        "        df = pd.DataFrame(logs)\n",
        "        df.to_csv(\"log_output.csv\")\n",
        "\n",
        "    # 最後のネットワークを保存する\n",
        "    torch.save(net.state_dict(), 'weights/pspnet50_' +\n",
        "               str(epoch+1) + '.pth')\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1XEy-KBb5Dk5",
        "outputId": "9f1b844f-178f-45f0-80fb-b992b8b452dc"
      },
      "source": [
        "# 学習・検証を実行する\n",
        "num_epochs = 30\n",
        "train_model(net, dataloaders_dict, criterion, scheduler, optimizer, num_epochs=num_epochs)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "使用デバイス： cuda:0\n",
            "-------------\n",
            "Epoch 1/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 10 || Loss: 0.3835 || 10iter: 83.2019 sec.\n",
            "イテレーション 20 || Loss: 0.2189 || 10iter: 50.9118 sec.\n",
            "イテレーション 30 || Loss: 0.1510 || 10iter: 50.8032 sec.\n",
            "イテレーション 40 || Loss: 0.1658 || 10iter: 50.7695 sec.\n",
            "イテレーション 50 || Loss: 0.0886 || 10iter: 50.6645 sec.\n",
            "イテレーション 60 || Loss: 0.0728 || 10iter: 50.6198 sec.\n",
            "イテレーション 70 || Loss: 0.1165 || 10iter: 50.9016 sec.\n",
            "イテレーション 80 || Loss: 0.1351 || 10iter: 50.4392 sec.\n",
            "イテレーション 90 || Loss: 0.2174 || 10iter: 50.6154 sec.\n",
            "イテレーション 100 || Loss: 0.0904 || 10iter: 50.5267 sec.\n",
            "イテレーション 110 || Loss: 0.1408 || 10iter: 50.4316 sec.\n",
            "イテレーション 120 || Loss: 0.0668 || 10iter: 50.5083 sec.\n",
            "イテレーション 130 || Loss: 0.1251 || 10iter: 50.7642 sec.\n",
            "イテレーション 140 || Loss: 0.1467 || 10iter: 50.5163 sec.\n",
            "イテレーション 150 || Loss: 0.0794 || 10iter: 50.5443 sec.\n",
            "イテレーション 160 || Loss: 0.1435 || 10iter: 50.4825 sec.\n",
            "イテレーション 170 || Loss: 0.2098 || 10iter: 50.4970 sec.\n",
            "イテレーション 180 || Loss: 0.1435 || 10iter: 50.5844 sec.\n",
            "-------------\n",
            "epoch 1 || Epoch_TRAIN_Loss:0.1771 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1056.2730 sec.\n",
            "-------------\n",
            "Epoch 2/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 190 || Loss: 0.0997 || 10iter: 33.7635 sec.\n",
            "イテレーション 200 || Loss: 0.0866 || 10iter: 50.4115 sec.\n",
            "イテレーション 210 || Loss: 0.0708 || 10iter: 50.5687 sec.\n",
            "イテレーション 220 || Loss: 0.0456 || 10iter: 50.5168 sec.\n",
            "イテレーション 230 || Loss: 0.0542 || 10iter: 50.6751 sec.\n",
            "イテレーション 240 || Loss: 0.1099 || 10iter: 50.5361 sec.\n",
            "イテレーション 250 || Loss: 0.0664 || 10iter: 50.3739 sec.\n",
            "イテレーション 260 || Loss: 0.0586 || 10iter: 50.3346 sec.\n",
            "イテレーション 270 || Loss: 0.0648 || 10iter: 50.5255 sec.\n",
            "イテレーション 280 || Loss: 0.0506 || 10iter: 50.5362 sec.\n",
            "イテレーション 290 || Loss: 0.0584 || 10iter: 50.6339 sec.\n",
            "イテレーション 300 || Loss: 0.1094 || 10iter: 50.6676 sec.\n",
            "イテレーション 310 || Loss: 0.0826 || 10iter: 50.4802 sec.\n",
            "イテレーション 320 || Loss: 0.0723 || 10iter: 50.4925 sec.\n",
            "イテレーション 330 || Loss: 0.0507 || 10iter: 50.7134 sec.\n",
            "イテレーション 340 || Loss: 0.0833 || 10iter: 50.6054 sec.\n",
            "イテレーション 350 || Loss: 0.0687 || 10iter: 50.8262 sec.\n",
            "イテレーション 360 || Loss: 0.0571 || 10iter: 50.8524 sec.\n",
            "-------------\n",
            "epoch 2 || Epoch_TRAIN_Loss:0.0919 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1022.4001 sec.\n",
            "-------------\n",
            "Epoch 3/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 370 || Loss: 0.0670 || 10iter: 17.0645 sec.\n",
            "イテレーション 380 || Loss: 0.0644 || 10iter: 50.7432 sec.\n",
            "イテレーション 390 || Loss: 0.0569 || 10iter: 50.5637 sec.\n",
            "イテレーション 400 || Loss: 0.0435 || 10iter: 50.5550 sec.\n",
            "イテレーション 410 || Loss: 0.0569 || 10iter: 50.4836 sec.\n",
            "イテレーション 420 || Loss: 0.0935 || 10iter: 50.4567 sec.\n",
            "イテレーション 430 || Loss: 0.1124 || 10iter: 50.6227 sec.\n",
            "イテレーション 440 || Loss: 0.0750 || 10iter: 50.4561 sec.\n",
            "イテレーション 450 || Loss: 0.0613 || 10iter: 50.5616 sec.\n",
            "イテレーション 460 || Loss: 0.0669 || 10iter: 50.4179 sec.\n",
            "イテレーション 470 || Loss: 0.0575 || 10iter: 50.5301 sec.\n",
            "イテレーション 480 || Loss: 0.0471 || 10iter: 50.5939 sec.\n",
            "イテレーション 490 || Loss: 0.0730 || 10iter: 50.7252 sec.\n",
            "イテレーション 500 || Loss: 0.0639 || 10iter: 50.6068 sec.\n",
            "イテレーション 510 || Loss: 0.0773 || 10iter: 50.5694 sec.\n",
            "イテレーション 520 || Loss: 0.0845 || 10iter: 50.5048 sec.\n",
            "イテレーション 530 || Loss: 0.0612 || 10iter: 50.6143 sec.\n",
            "イテレーション 540 || Loss: 0.0436 || 10iter: 50.6987 sec.\n",
            "-------------\n",
            "epoch 3 || Epoch_TRAIN_Loss:0.0784 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1022.9392 sec.\n",
            "-------------\n",
            "Epoch 4/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 550 || Loss: 0.1612 || 10iter: 0.2970 sec.\n",
            "イテレーション 560 || Loss: 0.0372 || 10iter: 50.6669 sec.\n",
            "イテレーション 570 || Loss: 0.0570 || 10iter: 50.9205 sec.\n",
            "イテレーション 580 || Loss: 0.0980 || 10iter: 50.5683 sec.\n",
            "イテレーション 590 || Loss: 0.0679 || 10iter: 50.7622 sec.\n",
            "イテレーション 600 || Loss: 0.0668 || 10iter: 50.7143 sec.\n",
            "イテレーション 610 || Loss: 0.0637 || 10iter: 50.6980 sec.\n",
            "イテレーション 620 || Loss: 0.0278 || 10iter: 50.7760 sec.\n",
            "イテレーション 630 || Loss: 0.1320 || 10iter: 50.6662 sec.\n",
            "イテレーション 640 || Loss: 0.0860 || 10iter: 50.7256 sec.\n",
            "イテレーション 650 || Loss: 0.0636 || 10iter: 50.6873 sec.\n",
            "イテレーション 660 || Loss: 0.0748 || 10iter: 50.7949 sec.\n",
            "イテレーション 670 || Loss: 0.0438 || 10iter: 50.7084 sec.\n",
            "イテレーション 680 || Loss: 0.1267 || 10iter: 50.9041 sec.\n",
            "イテレーション 690 || Loss: 0.0850 || 10iter: 50.7844 sec.\n",
            "イテレーション 700 || Loss: 0.0778 || 10iter: 50.7264 sec.\n",
            "イテレーション 710 || Loss: 0.0659 || 10iter: 50.6722 sec.\n",
            "イテレーション 720 || Loss: 0.0436 || 10iter: 50.6910 sec.\n",
            "イテレーション 730 || Loss: 0.0661 || 10iter: 50.7440 sec.\n",
            "-------------\n",
            "epoch 4 || Epoch_TRAIN_Loss:0.0714 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.6763 sec.\n",
            "-------------\n",
            "Epoch 5/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 740 || Loss: 0.0746 || 10iter: 39.5013 sec.\n",
            "イテレーション 750 || Loss: 0.0995 || 10iter: 50.7592 sec.\n",
            "イテレーション 760 || Loss: 0.1130 || 10iter: 50.6954 sec.\n",
            "イテレーション 770 || Loss: 0.0331 || 10iter: 50.6704 sec.\n",
            "イテレーション 780 || Loss: 0.0709 || 10iter: 50.7510 sec.\n",
            "イテレーション 790 || Loss: 0.0830 || 10iter: 50.7307 sec.\n",
            "イテレーション 800 || Loss: 0.0665 || 10iter: 50.6446 sec.\n",
            "イテレーション 810 || Loss: 0.0518 || 10iter: 50.7922 sec.\n",
            "イテレーション 820 || Loss: 0.0583 || 10iter: 50.8131 sec.\n",
            "イテレーション 830 || Loss: 0.0921 || 10iter: 50.7893 sec.\n",
            "イテレーション 840 || Loss: 0.0990 || 10iter: 50.8343 sec.\n",
            "イテレーション 850 || Loss: 0.1138 || 10iter: 50.9268 sec.\n",
            "イテレーション 860 || Loss: 0.0342 || 10iter: 50.8274 sec.\n",
            "イテレーション 870 || Loss: 0.0757 || 10iter: 50.8006 sec.\n",
            "イテレーション 880 || Loss: 0.0332 || 10iter: 50.6353 sec.\n",
            "イテレーション 890 || Loss: 0.0353 || 10iter: 50.5433 sec.\n",
            "イテレーション 900 || Loss: 0.0350 || 10iter: 50.8261 sec.\n",
            "イテレーション 910 || Loss: 0.0546 || 10iter: 50.7091 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 5 || Epoch_TRAIN_Loss:0.0666 ||Epoch_VAL_Loss:0.0802\n",
            "timer:  1347.2602 sec.\n",
            "-------------\n",
            "Epoch 6/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 920 || Loss: 0.0559 || 10iter: 22.6745 sec.\n",
            "イテレーション 930 || Loss: 0.0487 || 10iter: 50.4602 sec.\n",
            "イテレーション 940 || Loss: 0.0906 || 10iter: 50.8003 sec.\n",
            "イテレーション 950 || Loss: 0.0604 || 10iter: 50.9325 sec.\n",
            "イテレーション 960 || Loss: 0.0573 || 10iter: 50.7144 sec.\n",
            "イテレーション 970 || Loss: 0.0493 || 10iter: 50.7799 sec.\n",
            "イテレーション 980 || Loss: 0.0817 || 10iter: 50.8684 sec.\n",
            "イテレーション 990 || Loss: 0.0916 || 10iter: 50.5585 sec.\n",
            "イテレーション 1000 || Loss: 0.1501 || 10iter: 50.6191 sec.\n",
            "イテレーション 1010 || Loss: 0.0450 || 10iter: 50.6775 sec.\n",
            "イテレーション 1020 || Loss: 0.0680 || 10iter: 50.6985 sec.\n",
            "イテレーション 1030 || Loss: 0.0592 || 10iter: 50.5961 sec.\n",
            "イテレーション 1040 || Loss: 0.0649 || 10iter: 50.6750 sec.\n",
            "イテレーション 1050 || Loss: 0.0574 || 10iter: 50.4274 sec.\n",
            "イテレーション 1060 || Loss: 0.0709 || 10iter: 50.5833 sec.\n",
            "イテレーション 1070 || Loss: 0.0381 || 10iter: 50.7320 sec.\n",
            "イテレーション 1080 || Loss: 0.0499 || 10iter: 50.6843 sec.\n",
            "イテレーション 1090 || Loss: 0.0608 || 10iter: 50.6675 sec.\n",
            "-------------\n",
            "epoch 6 || Epoch_TRAIN_Loss:0.0613 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1024.6171 sec.\n",
            "-------------\n",
            "Epoch 7/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 1100 || Loss: 0.0614 || 10iter: 5.8334 sec.\n",
            "イテレーション 1110 || Loss: 0.1200 || 10iter: 50.7367 sec.\n",
            "イテレーション 1120 || Loss: 0.0713 || 10iter: 50.6147 sec.\n",
            "イテレーション 1130 || Loss: 0.0755 || 10iter: 50.5701 sec.\n",
            "イテレーション 1140 || Loss: 0.0827 || 10iter: 50.6069 sec.\n",
            "イテレーション 1150 || Loss: 0.0365 || 10iter: 50.8411 sec.\n",
            "イテレーション 1160 || Loss: 0.0608 || 10iter: 50.9119 sec.\n",
            "イテレーション 1170 || Loss: 0.0546 || 10iter: 50.7145 sec.\n",
            "イテレーション 1180 || Loss: 0.0517 || 10iter: 51.0666 sec.\n",
            "イテレーション 1190 || Loss: 0.0936 || 10iter: 50.4782 sec.\n",
            "イテレーション 1200 || Loss: 0.0612 || 10iter: 50.7285 sec.\n",
            "イテレーション 1210 || Loss: 0.0683 || 10iter: 50.7413 sec.\n",
            "イテレーション 1220 || Loss: 0.0631 || 10iter: 50.5807 sec.\n",
            "イテレーション 1230 || Loss: 0.0512 || 10iter: 50.9287 sec.\n",
            "イテレーション 1240 || Loss: 0.0585 || 10iter: 50.9585 sec.\n",
            "イテレーション 1250 || Loss: 0.0479 || 10iter: 50.6934 sec.\n",
            "イテレーション 1260 || Loss: 0.0975 || 10iter: 50.8956 sec.\n",
            "イテレーション 1270 || Loss: 0.0371 || 10iter: 50.7362 sec.\n",
            "イテレーション 1280 || Loss: 0.0696 || 10iter: 50.9284 sec.\n",
            "-------------\n",
            "epoch 7 || Epoch_TRAIN_Loss:0.0602 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1026.3847 sec.\n",
            "-------------\n",
            "Epoch 8/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 1290 || Loss: 0.0402 || 10iter: 45.1252 sec.\n",
            "イテレーション 1300 || Loss: 0.0530 || 10iter: 50.6068 sec.\n",
            "イテレーション 1310 || Loss: 0.0382 || 10iter: 50.6951 sec.\n",
            "イテレーション 1320 || Loss: 0.0914 || 10iter: 50.5856 sec.\n",
            "イテレーション 1330 || Loss: 0.0579 || 10iter: 50.7187 sec.\n",
            "イテレーション 1340 || Loss: 0.0602 || 10iter: 50.7291 sec.\n",
            "イテレーション 1350 || Loss: 0.0652 || 10iter: 50.8340 sec.\n",
            "イテレーション 1360 || Loss: 0.0697 || 10iter: 50.8346 sec.\n",
            "イテレーション 1370 || Loss: 0.0794 || 10iter: 50.9175 sec.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "イテレーション 1380 || Loss: 0.0715 || 10iter: 50.8677 sec.\n",
            "イテレーション 1390 || Loss: 0.0787 || 10iter: 50.8265 sec.\n",
            "イテレーション 1400 || Loss: 0.0590 || 10iter: 50.7918 sec.\n",
            "イテレーション 1410 || Loss: 0.0693 || 10iter: 50.6000 sec.\n",
            "イテレーション 1420 || Loss: 0.0749 || 10iter: 50.6794 sec.\n",
            "イテレーション 1430 || Loss: 0.1086 || 10iter: 50.6710 sec.\n",
            "イテレーション 1440 || Loss: 0.0649 || 10iter: 50.8908 sec.\n",
            "イテレーション 1450 || Loss: 0.0552 || 10iter: 50.7711 sec.\n",
            "イテレーション 1460 || Loss: 0.0374 || 10iter: 50.5102 sec.\n",
            "-------------\n",
            "epoch 8 || Epoch_TRAIN_Loss:0.0574 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1026.2220 sec.\n",
            "-------------\n",
            "Epoch 9/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 1470 || Loss: 0.0475 || 10iter: 28.4563 sec.\n",
            "イテレーション 1480 || Loss: 0.0209 || 10iter: 50.8128 sec.\n",
            "イテレーション 1490 || Loss: 0.0394 || 10iter: 50.5409 sec.\n",
            "イテレーション 1500 || Loss: 0.0464 || 10iter: 50.7096 sec.\n",
            "イテレーション 1510 || Loss: 0.0371 || 10iter: 50.8713 sec.\n",
            "イテレーション 1520 || Loss: 0.0364 || 10iter: 50.9948 sec.\n",
            "イテレーション 1530 || Loss: 0.1000 || 10iter: 50.6096 sec.\n",
            "イテレーション 1540 || Loss: 0.0512 || 10iter: 50.6501 sec.\n",
            "イテレーション 1550 || Loss: 0.0871 || 10iter: 50.5721 sec.\n",
            "イテレーション 1560 || Loss: 0.0863 || 10iter: 50.6137 sec.\n",
            "イテレーション 1570 || Loss: 0.0453 || 10iter: 50.5578 sec.\n",
            "イテレーション 1580 || Loss: 0.0600 || 10iter: 50.5028 sec.\n",
            "イテレーション 1590 || Loss: 0.0567 || 10iter: 50.7364 sec.\n",
            "イテレーション 1600 || Loss: 0.0469 || 10iter: 50.9735 sec.\n",
            "イテレーション 1610 || Loss: 0.0458 || 10iter: 50.6111 sec.\n",
            "イテレーション 1620 || Loss: 0.0618 || 10iter: 50.8481 sec.\n",
            "イテレーション 1630 || Loss: 0.0553 || 10iter: 50.7452 sec.\n",
            "イテレーション 1640 || Loss: 0.0417 || 10iter: 50.7357 sec.\n",
            "-------------\n",
            "epoch 9 || Epoch_TRAIN_Loss:0.0545 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.8891 sec.\n",
            "-------------\n",
            "Epoch 10/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 1650 || Loss: 0.0488 || 10iter: 11.4086 sec.\n",
            "イテレーション 1660 || Loss: 0.0447 || 10iter: 50.8153 sec.\n",
            "イテレーション 1670 || Loss: 0.0633 || 10iter: 50.6554 sec.\n",
            "イテレーション 1680 || Loss: 0.0414 || 10iter: 50.7869 sec.\n",
            "イテレーション 1690 || Loss: 0.0477 || 10iter: 50.7128 sec.\n",
            "イテレーション 1700 || Loss: 0.0683 || 10iter: 50.7007 sec.\n",
            "イテレーション 1710 || Loss: 0.0396 || 10iter: 50.5707 sec.\n",
            "イテレーション 1720 || Loss: 0.0522 || 10iter: 50.5896 sec.\n",
            "イテレーション 1730 || Loss: 0.0466 || 10iter: 50.7883 sec.\n",
            "イテレーション 1740 || Loss: 0.0418 || 10iter: 51.0085 sec.\n",
            "イテレーション 1750 || Loss: 0.0645 || 10iter: 50.7473 sec.\n",
            "イテレーション 1760 || Loss: 0.0587 || 10iter: 50.6194 sec.\n",
            "イテレーション 1770 || Loss: 0.0693 || 10iter: 50.9931 sec.\n",
            "イテレーション 1780 || Loss: 0.0541 || 10iter: 51.2675 sec.\n",
            "イテレーション 1790 || Loss: 0.0377 || 10iter: 50.7605 sec.\n",
            "イテレーション 1800 || Loss: 0.0413 || 10iter: 50.9146 sec.\n",
            "イテレーション 1810 || Loss: 0.0581 || 10iter: 51.0811 sec.\n",
            "イテレーション 1820 || Loss: 0.0626 || 10iter: 50.7084 sec.\n",
            "イテレーション 1830 || Loss: 0.0788 || 10iter: 50.6560 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 10 || Epoch_TRAIN_Loss:0.0505 ||Epoch_VAL_Loss:0.0759\n",
            "timer:  1349.1979 sec.\n",
            "-------------\n",
            "Epoch 11/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 1840 || Loss: 0.0756 || 10iter: 50.6540 sec.\n",
            "イテレーション 1850 || Loss: 0.0557 || 10iter: 50.6343 sec.\n",
            "イテレーション 1860 || Loss: 0.0390 || 10iter: 50.6837 sec.\n",
            "イテレーション 1870 || Loss: 0.0487 || 10iter: 50.6753 sec.\n",
            "イテレーション 1880 || Loss: 0.0452 || 10iter: 50.5269 sec.\n",
            "イテレーション 1890 || Loss: 0.0526 || 10iter: 50.5669 sec.\n",
            "イテレーション 1900 || Loss: 0.0641 || 10iter: 50.6744 sec.\n",
            "イテレーション 1910 || Loss: 0.0497 || 10iter: 51.3049 sec.\n",
            "イテレーション 1920 || Loss: 0.0511 || 10iter: 50.6360 sec.\n",
            "イテレーション 1930 || Loss: 0.0393 || 10iter: 50.5260 sec.\n",
            "イテレーション 1940 || Loss: 0.0401 || 10iter: 50.6548 sec.\n",
            "イテレーション 1950 || Loss: 0.0350 || 10iter: 50.7342 sec.\n",
            "イテレーション 1960 || Loss: 0.0701 || 10iter: 50.7164 sec.\n",
            "イテレーション 1970 || Loss: 0.0374 || 10iter: 50.7997 sec.\n",
            "イテレーション 1980 || Loss: 0.0323 || 10iter: 50.6672 sec.\n",
            "イテレーション 1990 || Loss: 0.0376 || 10iter: 50.7574 sec.\n",
            "イテレーション 2000 || Loss: 0.0611 || 10iter: 50.5434 sec.\n",
            "イテレーション 2010 || Loss: 0.0380 || 10iter: 50.6344 sec.\n",
            "-------------\n",
            "epoch 11 || Epoch_TRAIN_Loss:0.0495 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1024.8894 sec.\n",
            "-------------\n",
            "Epoch 12/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2020 || Loss: 0.0462 || 10iter: 33.9444 sec.\n",
            "イテレーション 2030 || Loss: 0.0392 || 10iter: 50.7873 sec.\n",
            "イテレーション 2040 || Loss: 0.0483 || 10iter: 50.7978 sec.\n",
            "イテレーション 2050 || Loss: 0.0555 || 10iter: 50.8544 sec.\n",
            "イテレーション 2060 || Loss: 0.0341 || 10iter: 50.8273 sec.\n",
            "イテレーション 2070 || Loss: 0.0444 || 10iter: 50.8152 sec.\n",
            "イテレーション 2080 || Loss: 0.0357 || 10iter: 50.9978 sec.\n",
            "イテレーション 2090 || Loss: 0.0640 || 10iter: 50.7223 sec.\n",
            "イテレーション 2100 || Loss: 0.0586 || 10iter: 50.7397 sec.\n",
            "イテレーション 2110 || Loss: 0.0690 || 10iter: 50.9867 sec.\n",
            "イテレーション 2120 || Loss: 0.0558 || 10iter: 50.6848 sec.\n",
            "イテレーション 2130 || Loss: 0.0422 || 10iter: 50.5360 sec.\n",
            "イテレーション 2140 || Loss: 0.0337 || 10iter: 50.5375 sec.\n",
            "イテレーション 2150 || Loss: 0.0254 || 10iter: 50.6792 sec.\n",
            "イテレーション 2160 || Loss: 0.0391 || 10iter: 50.7366 sec.\n",
            "イテレーション 2170 || Loss: 0.1643 || 10iter: 50.8524 sec.\n",
            "イテレーション 2180 || Loss: 0.0543 || 10iter: 50.8645 sec.\n",
            "イテレーション 2190 || Loss: 0.0370 || 10iter: 50.8177 sec.\n",
            "-------------\n",
            "epoch 12 || Epoch_TRAIN_Loss:0.0484 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1026.8597 sec.\n",
            "-------------\n",
            "Epoch 13/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2200 || Loss: 0.0593 || 10iter: 17.0840 sec.\n",
            "イテレーション 2210 || Loss: 0.0508 || 10iter: 50.9188 sec.\n",
            "イテレーション 2220 || Loss: 0.0592 || 10iter: 50.8460 sec.\n",
            "イテレーション 2230 || Loss: 0.0513 || 10iter: 50.9855 sec.\n",
            "イテレーション 2240 || Loss: 0.0390 || 10iter: 50.8630 sec.\n",
            "イテレーション 2250 || Loss: 0.0417 || 10iter: 50.8813 sec.\n",
            "イテレーション 2260 || Loss: 0.0399 || 10iter: 51.2813 sec.\n",
            "イテレーション 2270 || Loss: 0.0353 || 10iter: 50.9823 sec.\n",
            "イテレーション 2280 || Loss: 0.0459 || 10iter: 50.9177 sec.\n",
            "イテレーション 2290 || Loss: 0.0316 || 10iter: 50.8643 sec.\n",
            "イテレーション 2300 || Loss: 0.0438 || 10iter: 50.9033 sec.\n",
            "イテレーション 2310 || Loss: 0.0341 || 10iter: 50.8878 sec.\n",
            "イテレーション 2320 || Loss: 0.0471 || 10iter: 50.8469 sec.\n",
            "イテレーション 2330 || Loss: 0.0269 || 10iter: 51.0611 sec.\n",
            "イテレーション 2340 || Loss: 0.0511 || 10iter: 51.0096 sec.\n",
            "イテレーション 2350 || Loss: 0.0260 || 10iter: 50.8879 sec.\n",
            "イテレーション 2360 || Loss: 0.0444 || 10iter: 50.8269 sec.\n",
            "イテレーション 2370 || Loss: 0.0926 || 10iter: 50.9142 sec.\n",
            "-------------\n",
            "epoch 13 || Epoch_TRAIN_Loss:0.0467 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1029.7049 sec.\n",
            "-------------\n",
            "Epoch 14/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2380 || Loss: 0.0215 || 10iter: 0.2966 sec.\n",
            "イテレーション 2390 || Loss: 0.0366 || 10iter: 50.9562 sec.\n",
            "イテレーション 2400 || Loss: 0.0376 || 10iter: 50.7793 sec.\n",
            "イテレーション 2410 || Loss: 0.0334 || 10iter: 50.8919 sec.\n",
            "イテレーション 2420 || Loss: 0.0488 || 10iter: 50.9355 sec.\n",
            "イテレーション 2430 || Loss: 0.0558 || 10iter: 50.9042 sec.\n",
            "イテレーション 2440 || Loss: 0.0580 || 10iter: 50.9065 sec.\n",
            "イテレーション 2450 || Loss: 0.0614 || 10iter: 50.8216 sec.\n",
            "イテレーション 2460 || Loss: 0.0417 || 10iter: 50.8039 sec.\n",
            "イテレーション 2470 || Loss: 0.0421 || 10iter: 50.9031 sec.\n",
            "イテレーション 2480 || Loss: 0.0304 || 10iter: 50.6135 sec.\n",
            "イテレーション 2490 || Loss: 0.0464 || 10iter: 50.6779 sec.\n",
            "イテレーション 2500 || Loss: 0.0556 || 10iter: 50.6792 sec.\n",
            "イテレーション 2510 || Loss: 0.0431 || 10iter: 50.7663 sec.\n",
            "イテレーション 2520 || Loss: 0.0439 || 10iter: 51.0113 sec.\n",
            "イテレーション 2530 || Loss: 0.0755 || 10iter: 50.9009 sec.\n",
            "イテレーション 2540 || Loss: 0.0457 || 10iter: 50.6577 sec.\n",
            "イテレーション 2550 || Loss: 0.0496 || 10iter: 50.6449 sec.\n",
            "イテレーション 2560 || Loss: 0.0286 || 10iter: 50.8029 sec.\n",
            "-------------\n",
            "epoch 14 || Epoch_TRAIN_Loss:0.0463 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1027.2999 sec.\n",
            "-------------\n",
            "Epoch 15/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2570 || Loss: 0.0361 || 10iter: 39.6952 sec.\n",
            "イテレーション 2580 || Loss: 0.0419 || 10iter: 50.8017 sec.\n",
            "イテレーション 2590 || Loss: 0.0326 || 10iter: 50.6874 sec.\n",
            "イテレーション 2600 || Loss: 0.0343 || 10iter: 50.7933 sec.\n",
            "イテレーション 2610 || Loss: 0.0457 || 10iter: 50.6328 sec.\n",
            "イテレーション 2620 || Loss: 0.0267 || 10iter: 50.6796 sec.\n",
            "イテレーション 2630 || Loss: 0.0338 || 10iter: 50.6994 sec.\n",
            "イテレーション 2640 || Loss: 0.0608 || 10iter: 50.7672 sec.\n",
            "イテレーション 2650 || Loss: 0.0495 || 10iter: 50.6358 sec.\n",
            "イテレーション 2660 || Loss: 0.0519 || 10iter: 50.4650 sec.\n",
            "イテレーション 2670 || Loss: 0.0369 || 10iter: 50.7017 sec.\n",
            "イテレーション 2680 || Loss: 0.0400 || 10iter: 50.9054 sec.\n",
            "イテレーション 2690 || Loss: 0.0268 || 10iter: 50.8580 sec.\n",
            "イテレーション 2700 || Loss: 0.0382 || 10iter: 50.5799 sec.\n",
            "イテレーション 2710 || Loss: 0.0570 || 10iter: 50.7381 sec.\n",
            "イテレーション 2720 || Loss: 0.0334 || 10iter: 50.7179 sec.\n",
            "イテレーション 2730 || Loss: 0.0478 || 10iter: 50.5734 sec.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "イテレーション 2740 || Loss: 0.0271 || 10iter: 50.7591 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 15 || Epoch_TRAIN_Loss:0.0459 ||Epoch_VAL_Loss:0.0714\n",
            "timer:  1347.2690 sec.\n",
            "-------------\n",
            "Epoch 16/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2750 || Loss: 0.0726 || 10iter: 22.7602 sec.\n",
            "イテレーション 2760 || Loss: 0.0297 || 10iter: 50.7317 sec.\n",
            "イテレーション 2770 || Loss: 0.0294 || 10iter: 50.7034 sec.\n",
            "イテレーション 2780 || Loss: 0.0386 || 10iter: 50.9082 sec.\n",
            "イテレーション 2790 || Loss: 0.0381 || 10iter: 50.8296 sec.\n",
            "イテレーション 2800 || Loss: 0.0466 || 10iter: 50.6776 sec.\n",
            "イテレーション 2810 || Loss: 0.0318 || 10iter: 50.8160 sec.\n",
            "イテレーション 2820 || Loss: 0.0314 || 10iter: 50.5198 sec.\n",
            "イテレーション 2830 || Loss: 0.0482 || 10iter: 50.7523 sec.\n",
            "イテレーション 2840 || Loss: 0.0258 || 10iter: 50.8890 sec.\n",
            "イテレーション 2850 || Loss: 0.0436 || 10iter: 50.7661 sec.\n",
            "イテレーション 2860 || Loss: 0.0503 || 10iter: 50.5520 sec.\n",
            "イテレーション 2870 || Loss: 0.0293 || 10iter: 50.5255 sec.\n",
            "イテレーション 2880 || Loss: 0.0287 || 10iter: 50.4744 sec.\n",
            "イテレーション 2890 || Loss: 0.0360 || 10iter: 50.7980 sec.\n",
            "イテレーション 2900 || Loss: 0.0362 || 10iter: 50.8190 sec.\n",
            "イテレーション 2910 || Loss: 0.0401 || 10iter: 50.7190 sec.\n",
            "イテレーション 2920 || Loss: 0.0459 || 10iter: 50.6530 sec.\n",
            "-------------\n",
            "epoch 16 || Epoch_TRAIN_Loss:0.0429 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.8803 sec.\n",
            "-------------\n",
            "Epoch 17/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 2930 || Loss: 0.0494 || 10iter: 6.0089 sec.\n",
            "イテレーション 2940 || Loss: 0.0501 || 10iter: 50.8427 sec.\n",
            "イテレーション 2950 || Loss: 0.0485 || 10iter: 50.9029 sec.\n",
            "イテレーション 2960 || Loss: 0.0537 || 10iter: 50.8587 sec.\n",
            "イテレーション 2970 || Loss: 0.0414 || 10iter: 50.6686 sec.\n",
            "イテレーション 2980 || Loss: 0.0519 || 10iter: 50.7551 sec.\n",
            "イテレーション 2990 || Loss: 0.0523 || 10iter: 50.8267 sec.\n",
            "イテレーション 3000 || Loss: 0.0404 || 10iter: 50.8585 sec.\n",
            "イテレーション 3010 || Loss: 0.0364 || 10iter: 50.9562 sec.\n",
            "イテレーション 3020 || Loss: 0.0260 || 10iter: 50.8899 sec.\n",
            "イテレーション 3030 || Loss: 0.0467 || 10iter: 50.6349 sec.\n",
            "イテレーション 3040 || Loss: 0.0386 || 10iter: 50.6311 sec.\n",
            "イテレーション 3050 || Loss: 0.0594 || 10iter: 50.7737 sec.\n",
            "イテレーション 3060 || Loss: 0.0272 || 10iter: 50.6995 sec.\n",
            "イテレーション 3070 || Loss: 0.0288 || 10iter: 50.7778 sec.\n",
            "イテレーション 3080 || Loss: 0.0704 || 10iter: 51.0129 sec.\n",
            "イテレーション 3090 || Loss: 0.0359 || 10iter: 50.7747 sec.\n",
            "イテレーション 3100 || Loss: 0.0486 || 10iter: 50.7031 sec.\n",
            "イテレーション 3110 || Loss: 0.0503 || 10iter: 50.8194 sec.\n",
            "-------------\n",
            "epoch 17 || Epoch_TRAIN_Loss:0.0448 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1027.7675 sec.\n",
            "-------------\n",
            "Epoch 18/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 3120 || Loss: 0.0293 || 10iter: 45.0682 sec.\n",
            "イテレーション 3130 || Loss: 0.0469 || 10iter: 50.6639 sec.\n",
            "イテレーション 3140 || Loss: 0.0335 || 10iter: 50.7718 sec.\n",
            "イテレーション 3150 || Loss: 0.0510 || 10iter: 50.8002 sec.\n",
            "イテレーション 3160 || Loss: 0.0273 || 10iter: 51.0128 sec.\n",
            "イテレーション 3170 || Loss: 0.0266 || 10iter: 50.4747 sec.\n",
            "イテレーション 3180 || Loss: 0.0321 || 10iter: 50.3914 sec.\n",
            "イテレーション 3190 || Loss: 0.0413 || 10iter: 50.4989 sec.\n",
            "イテレーション 3200 || Loss: 0.0973 || 10iter: 50.7168 sec.\n",
            "イテレーション 3210 || Loss: 0.0423 || 10iter: 50.7889 sec.\n",
            "イテレーション 3220 || Loss: 0.0348 || 10iter: 50.6232 sec.\n",
            "イテレーション 3230 || Loss: 0.0315 || 10iter: 50.6590 sec.\n",
            "イテレーション 3240 || Loss: 0.0424 || 10iter: 50.6628 sec.\n",
            "イテレーション 3250 || Loss: 0.0497 || 10iter: 50.6123 sec.\n",
            "イテレーション 3260 || Loss: 0.0424 || 10iter: 50.9512 sec.\n",
            "イテレーション 3270 || Loss: 0.0511 || 10iter: 50.8080 sec.\n",
            "イテレーション 3280 || Loss: 0.0326 || 10iter: 50.7627 sec.\n",
            "イテレーション 3290 || Loss: 0.0365 || 10iter: 50.6966 sec.\n",
            "-------------\n",
            "epoch 18 || Epoch_TRAIN_Loss:0.0437 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.6398 sec.\n",
            "-------------\n",
            "Epoch 19/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 3300 || Loss: 0.0448 || 10iter: 28.5637 sec.\n",
            "イテレーション 3310 || Loss: 0.0861 || 10iter: 50.9766 sec.\n",
            "イテレーション 3320 || Loss: 0.0486 || 10iter: 51.1356 sec.\n",
            "イテレーション 3330 || Loss: 0.0296 || 10iter: 50.7503 sec.\n",
            "イテレーション 3340 || Loss: 0.0647 || 10iter: 50.8836 sec.\n",
            "イテレーション 3350 || Loss: 0.0439 || 10iter: 50.8223 sec.\n",
            "イテレーション 3360 || Loss: 0.0391 || 10iter: 50.9826 sec.\n",
            "イテレーション 3370 || Loss: 0.0303 || 10iter: 51.1298 sec.\n",
            "イテレーション 3380 || Loss: 0.0330 || 10iter: 50.7030 sec.\n",
            "イテレーション 3390 || Loss: 0.0306 || 10iter: 50.5468 sec.\n",
            "イテレーション 3400 || Loss: 0.0366 || 10iter: 50.6972 sec.\n",
            "イテレーション 3410 || Loss: 0.0369 || 10iter: 50.9969 sec.\n",
            "イテレーション 3420 || Loss: 0.0486 || 10iter: 51.0305 sec.\n",
            "イテレーション 3430 || Loss: 0.0384 || 10iter: 51.1033 sec.\n",
            "イテレーション 3440 || Loss: 0.0502 || 10iter: 51.0050 sec.\n",
            "イテレーション 3450 || Loss: 0.0316 || 10iter: 50.6722 sec.\n",
            "イテレーション 3460 || Loss: 0.0291 || 10iter: 50.8300 sec.\n",
            "イテレーション 3470 || Loss: 0.0395 || 10iter: 51.0256 sec.\n",
            "-------------\n",
            "epoch 19 || Epoch_TRAIN_Loss:0.0435 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1029.5972 sec.\n",
            "-------------\n",
            "Epoch 20/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 3480 || Loss: 0.0245 || 10iter: 11.4851 sec.\n",
            "イテレーション 3490 || Loss: 0.0380 || 10iter: 50.3755 sec.\n",
            "イテレーション 3500 || Loss: 0.0588 || 10iter: 51.1811 sec.\n",
            "イテレーション 3510 || Loss: 0.0309 || 10iter: 50.9224 sec.\n",
            "イテレーション 3520 || Loss: 0.0203 || 10iter: 51.5507 sec.\n",
            "イテレーション 3530 || Loss: 0.0423 || 10iter: 50.6811 sec.\n",
            "イテレーション 3540 || Loss: 0.0387 || 10iter: 50.8168 sec.\n",
            "イテレーション 3550 || Loss: 0.0465 || 10iter: 50.8422 sec.\n",
            "イテレーション 3560 || Loss: 0.0688 || 10iter: 50.8916 sec.\n",
            "イテレーション 3570 || Loss: 0.0345 || 10iter: 51.1554 sec.\n",
            "イテレーション 3580 || Loss: 0.0348 || 10iter: 50.9648 sec.\n",
            "イテレーション 3590 || Loss: 0.0409 || 10iter: 50.6741 sec.\n",
            "イテレーション 3600 || Loss: 0.0801 || 10iter: 50.6562 sec.\n",
            "イテレーション 3610 || Loss: 0.0519 || 10iter: 50.7185 sec.\n",
            "イテレーション 3620 || Loss: 0.0519 || 10iter: 51.0284 sec.\n",
            "イテレーション 3630 || Loss: 0.0226 || 10iter: 50.9812 sec.\n",
            "イテレーション 3640 || Loss: 0.0224 || 10iter: 50.9137 sec.\n",
            "イテレーション 3650 || Loss: 0.0510 || 10iter: 50.9480 sec.\n",
            "イテレーション 3660 || Loss: 0.0397 || 10iter: 50.7722 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 20 || Epoch_TRAIN_Loss:0.0432 ||Epoch_VAL_Loss:0.0714\n",
            "timer:  1352.1267 sec.\n",
            "-------------\n",
            "Epoch 21/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 3670 || Loss: 0.0717 || 10iter: 51.0433 sec.\n",
            "イテレーション 3680 || Loss: 0.0625 || 10iter: 51.0897 sec.\n",
            "イテレーション 3690 || Loss: 0.0330 || 10iter: 50.7828 sec.\n",
            "イテレーション 3700 || Loss: 0.0219 || 10iter: 50.9731 sec.\n",
            "イテレーション 3710 || Loss: 0.0397 || 10iter: 50.8657 sec.\n",
            "イテレーション 3720 || Loss: 0.0429 || 10iter: 50.8553 sec.\n",
            "イテレーション 3730 || Loss: 0.0493 || 10iter: 50.8467 sec.\n",
            "イテレーション 3740 || Loss: 0.0337 || 10iter: 50.8568 sec.\n",
            "イテレーション 3750 || Loss: 0.0489 || 10iter: 50.8687 sec.\n",
            "イテレーション 3760 || Loss: 0.0252 || 10iter: 50.6633 sec.\n",
            "イテレーション 3770 || Loss: 0.0523 || 10iter: 50.9524 sec.\n",
            "イテレーション 3780 || Loss: 0.0670 || 10iter: 50.7681 sec.\n",
            "イテレーション 3790 || Loss: 0.0348 || 10iter: 50.8188 sec.\n",
            "イテレーション 3800 || Loss: 0.0534 || 10iter: 50.7525 sec.\n",
            "イテレーション 3810 || Loss: 0.0387 || 10iter: 50.9238 sec.\n",
            "イテレーション 3820 || Loss: 0.0484 || 10iter: 50.8013 sec.\n",
            "イテレーション 3830 || Loss: 0.0414 || 10iter: 50.8840 sec.\n",
            "イテレーション 3840 || Loss: 0.0230 || 10iter: 51.0179 sec.\n",
            "-------------\n",
            "epoch 21 || Epoch_TRAIN_Loss:0.0422 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1028.5794 sec.\n",
            "-------------\n",
            "Epoch 22/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 3850 || Loss: 0.0239 || 10iter: 33.8703 sec.\n",
            "イテレーション 3860 || Loss: 0.0603 || 10iter: 50.5754 sec.\n",
            "イテレーション 3870 || Loss: 0.0238 || 10iter: 50.8592 sec.\n",
            "イテレーション 3880 || Loss: 0.0302 || 10iter: 50.9409 sec.\n",
            "イテレーション 3890 || Loss: 0.0466 || 10iter: 50.6616 sec.\n",
            "イテレーション 3900 || Loss: 0.0386 || 10iter: 50.8154 sec.\n",
            "イテレーション 3910 || Loss: 0.0419 || 10iter: 50.7668 sec.\n",
            "イテレーション 3920 || Loss: 0.0386 || 10iter: 50.5855 sec.\n",
            "イテレーション 3930 || Loss: 0.0696 || 10iter: 50.7717 sec.\n",
            "イテレーション 3940 || Loss: 0.0286 || 10iter: 50.8771 sec.\n",
            "イテレーション 3950 || Loss: 0.0646 || 10iter: 50.7858 sec.\n",
            "イテレーション 3960 || Loss: 0.0377 || 10iter: 50.7269 sec.\n",
            "イテレーション 3970 || Loss: 0.0297 || 10iter: 50.7836 sec.\n",
            "イテレーション 3980 || Loss: 0.0201 || 10iter: 50.6229 sec.\n",
            "イテレーション 3990 || Loss: 0.0312 || 10iter: 50.8563 sec.\n",
            "イテレーション 4000 || Loss: 0.0390 || 10iter: 50.9074 sec.\n",
            "イテレーション 4010 || Loss: 0.0275 || 10iter: 50.7891 sec.\n",
            "イテレーション 4020 || Loss: 0.0302 || 10iter: 50.7700 sec.\n",
            "-------------\n",
            "epoch 22 || Epoch_TRAIN_Loss:0.0431 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1026.4953 sec.\n",
            "-------------\n",
            "Epoch 23/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4030 || Loss: 0.0527 || 10iter: 17.2508 sec.\n",
            "イテレーション 4040 || Loss: 0.0361 || 10iter: 50.8991 sec.\n",
            "イテレーション 4050 || Loss: 0.0664 || 10iter: 50.6701 sec.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "イテレーション 4060 || Loss: 0.0297 || 10iter: 50.8080 sec.\n",
            "イテレーション 4070 || Loss: 0.0244 || 10iter: 50.4723 sec.\n",
            "イテレーション 4080 || Loss: 0.0669 || 10iter: 50.6491 sec.\n",
            "イテレーション 4090 || Loss: 0.0236 || 10iter: 50.5933 sec.\n",
            "イテレーション 4100 || Loss: 0.0330 || 10iter: 50.7665 sec.\n",
            "イテレーション 4110 || Loss: 0.0284 || 10iter: 50.5859 sec.\n",
            "イテレーション 4120 || Loss: 0.0409 || 10iter: 50.5714 sec.\n",
            "イテレーション 4130 || Loss: 0.0505 || 10iter: 50.9581 sec.\n",
            "イテレーション 4140 || Loss: 0.0372 || 10iter: 50.7392 sec.\n",
            "イテレーション 4150 || Loss: 0.0457 || 10iter: 50.8464 sec.\n",
            "イテレーション 4160 || Loss: 0.0450 || 10iter: 50.8673 sec.\n",
            "イテレーション 4170 || Loss: 0.0761 || 10iter: 50.6163 sec.\n",
            "イテレーション 4180 || Loss: 0.0455 || 10iter: 50.8787 sec.\n",
            "イテレーション 4190 || Loss: 0.0393 || 10iter: 50.5992 sec.\n",
            "イテレーション 4200 || Loss: 0.0424 || 10iter: 50.7738 sec.\n",
            "-------------\n",
            "epoch 23 || Epoch_TRAIN_Loss:0.0423 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.3780 sec.\n",
            "-------------\n",
            "Epoch 24/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4210 || Loss: 0.0193 || 10iter: 0.2901 sec.\n",
            "イテレーション 4220 || Loss: 0.0379 || 10iter: 50.4680 sec.\n",
            "イテレーション 4230 || Loss: 0.0544 || 10iter: 50.5312 sec.\n",
            "イテレーション 4240 || Loss: 0.0612 || 10iter: 50.7231 sec.\n",
            "イテレーション 4250 || Loss: 0.0236 || 10iter: 50.6315 sec.\n",
            "イテレーション 4260 || Loss: 0.0300 || 10iter: 50.6389 sec.\n",
            "イテレーション 4270 || Loss: 0.0403 || 10iter: 50.6389 sec.\n",
            "イテレーション 4280 || Loss: 0.0933 || 10iter: 50.7659 sec.\n",
            "イテレーション 4290 || Loss: 0.0639 || 10iter: 50.5220 sec.\n",
            "イテレーション 4300 || Loss: 0.0395 || 10iter: 50.6218 sec.\n",
            "イテレーション 4310 || Loss: 0.0266 || 10iter: 50.7401 sec.\n",
            "イテレーション 4320 || Loss: 0.0174 || 10iter: 50.4592 sec.\n",
            "イテレーション 4330 || Loss: 0.0424 || 10iter: 50.8557 sec.\n",
            "イテレーション 4340 || Loss: 0.0549 || 10iter: 50.6149 sec.\n",
            "イテレーション 4350 || Loss: 0.0350 || 10iter: 50.8201 sec.\n",
            "イテレーション 4360 || Loss: 0.0258 || 10iter: 50.7300 sec.\n",
            "イテレーション 4370 || Loss: 0.0437 || 10iter: 50.6177 sec.\n",
            "イテレーション 4380 || Loss: 0.0587 || 10iter: 50.7678 sec.\n",
            "イテレーション 4390 || Loss: 0.0247 || 10iter: 51.2805 sec.\n",
            "-------------\n",
            "epoch 24 || Epoch_TRAIN_Loss:0.0414 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1025.1146 sec.\n",
            "-------------\n",
            "Epoch 25/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4400 || Loss: 0.0414 || 10iter: 39.4339 sec.\n",
            "イテレーション 4410 || Loss: 0.0362 || 10iter: 51.0917 sec.\n",
            "イテレーション 4420 || Loss: 0.0319 || 10iter: 50.5509 sec.\n",
            "イテレーション 4430 || Loss: 0.0395 || 10iter: 50.4922 sec.\n",
            "イテレーション 4440 || Loss: 0.0453 || 10iter: 50.7563 sec.\n",
            "イテレーション 4450 || Loss: 0.0319 || 10iter: 50.5940 sec.\n",
            "イテレーション 4460 || Loss: 0.0589 || 10iter: 50.8591 sec.\n",
            "イテレーション 4470 || Loss: 0.0232 || 10iter: 50.5475 sec.\n",
            "イテレーション 4480 || Loss: 0.0252 || 10iter: 50.8085 sec.\n",
            "イテレーション 4490 || Loss: 0.0216 || 10iter: 50.6379 sec.\n",
            "イテレーション 4500 || Loss: 0.0469 || 10iter: 50.6337 sec.\n",
            "イテレーション 4510 || Loss: 0.0449 || 10iter: 50.7443 sec.\n",
            "イテレーション 4520 || Loss: 0.0470 || 10iter: 50.8530 sec.\n",
            "イテレーション 4530 || Loss: 0.0315 || 10iter: 50.8202 sec.\n",
            "イテレーション 4540 || Loss: 0.0318 || 10iter: 50.7968 sec.\n",
            "イテレーション 4550 || Loss: 0.0402 || 10iter: 50.7136 sec.\n",
            "イテレーション 4560 || Loss: 0.0325 || 10iter: 50.6114 sec.\n",
            "イテレーション 4570 || Loss: 0.0430 || 10iter: 50.6088 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 25 || Epoch_TRAIN_Loss:0.0417 ||Epoch_VAL_Loss:0.0715\n",
            "timer:  1346.8907 sec.\n",
            "-------------\n",
            "Epoch 26/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4580 || Loss: 0.0377 || 10iter: 22.7415 sec.\n",
            "イテレーション 4590 || Loss: 0.0385 || 10iter: 50.4380 sec.\n",
            "イテレーション 4600 || Loss: 0.0502 || 10iter: 50.6882 sec.\n",
            "イテレーション 4610 || Loss: 0.0332 || 10iter: 50.7351 sec.\n",
            "イテレーション 4620 || Loss: 0.0280 || 10iter: 50.7304 sec.\n",
            "イテレーション 4630 || Loss: 0.0300 || 10iter: 50.8155 sec.\n",
            "イテレーション 4640 || Loss: 0.0427 || 10iter: 50.5334 sec.\n",
            "イテレーション 4650 || Loss: 0.0285 || 10iter: 50.4610 sec.\n",
            "イテレーション 4660 || Loss: 0.0472 || 10iter: 51.0121 sec.\n",
            "イテレーション 4670 || Loss: 0.0429 || 10iter: 50.9830 sec.\n",
            "イテレーション 4680 || Loss: 0.0560 || 10iter: 50.8920 sec.\n",
            "イテレーション 4690 || Loss: 0.0237 || 10iter: 50.7298 sec.\n",
            "イテレーション 4700 || Loss: 0.0286 || 10iter: 50.7338 sec.\n",
            "イテレーション 4710 || Loss: 0.0354 || 10iter: 50.4900 sec.\n",
            "イテレーション 4720 || Loss: 0.0487 || 10iter: 50.8863 sec.\n",
            "イテレーション 4730 || Loss: 0.0546 || 10iter: 51.0870 sec.\n",
            "イテレーション 4740 || Loss: 0.0372 || 10iter: 50.8747 sec.\n",
            "イテレーション 4750 || Loss: 0.0299 || 10iter: 51.0240 sec.\n",
            "-------------\n",
            "epoch 26 || Epoch_TRAIN_Loss:0.0405 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1026.9166 sec.\n",
            "-------------\n",
            "Epoch 27/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4760 || Loss: 0.0495 || 10iter: 5.9440 sec.\n",
            "イテレーション 4770 || Loss: 0.0392 || 10iter: 50.8529 sec.\n",
            "イテレーション 4780 || Loss: 0.0246 || 10iter: 50.9150 sec.\n",
            "イテレーション 4790 || Loss: 0.0339 || 10iter: 50.6974 sec.\n",
            "イテレーション 4800 || Loss: 0.0524 || 10iter: 50.9193 sec.\n",
            "イテレーション 4810 || Loss: 0.0313 || 10iter: 51.1367 sec.\n",
            "イテレーション 4820 || Loss: 0.0295 || 10iter: 50.9975 sec.\n",
            "イテレーション 4830 || Loss: 0.0182 || 10iter: 50.8620 sec.\n",
            "イテレーション 4840 || Loss: 0.0477 || 10iter: 51.1026 sec.\n",
            "イテレーション 4850 || Loss: 0.0601 || 10iter: 50.8497 sec.\n",
            "イテレーション 4860 || Loss: 0.0304 || 10iter: 50.5436 sec.\n",
            "イテレーション 4870 || Loss: 0.0338 || 10iter: 50.7709 sec.\n",
            "イテレーション 4880 || Loss: 0.0418 || 10iter: 50.8460 sec.\n",
            "イテレーション 4890 || Loss: 0.0294 || 10iter: 50.8742 sec.\n",
            "イテレーション 4900 || Loss: 0.0208 || 10iter: 50.8492 sec.\n",
            "イテレーション 4910 || Loss: 0.0383 || 10iter: 50.8793 sec.\n",
            "イテレーション 4920 || Loss: 0.0178 || 10iter: 50.8939 sec.\n",
            "イテレーション 4930 || Loss: 0.0358 || 10iter: 50.8207 sec.\n",
            "イテレーション 4940 || Loss: 0.0306 || 10iter: 51.1803 sec.\n",
            "-------------\n",
            "epoch 27 || Epoch_TRAIN_Loss:0.0392 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1029.3691 sec.\n",
            "-------------\n",
            "Epoch 28/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 4950 || Loss: 0.0713 || 10iter: 45.4615 sec.\n",
            "イテレーション 4960 || Loss: 0.0346 || 10iter: 50.8605 sec.\n",
            "イテレーション 4970 || Loss: 0.0341 || 10iter: 50.7551 sec.\n",
            "イテレーション 4980 || Loss: 0.0468 || 10iter: 50.7723 sec.\n",
            "イテレーション 4990 || Loss: 0.0468 || 10iter: 51.2496 sec.\n",
            "イテレーション 5000 || Loss: 0.0435 || 10iter: 50.8582 sec.\n",
            "イテレーション 5010 || Loss: 0.0357 || 10iter: 50.6008 sec.\n",
            "イテレーション 5020 || Loss: 0.0432 || 10iter: 50.6479 sec.\n",
            "イテレーション 5030 || Loss: 0.0380 || 10iter: 50.9605 sec.\n",
            "イテレーション 5040 || Loss: 0.0244 || 10iter: 50.7780 sec.\n",
            "イテレーション 5050 || Loss: 0.0252 || 10iter: 50.7124 sec.\n",
            "イテレーション 5060 || Loss: 0.0405 || 10iter: 50.8227 sec.\n",
            "イテレーション 5070 || Loss: 0.0518 || 10iter: 50.7488 sec.\n",
            "イテレーション 5080 || Loss: 0.0411 || 10iter: 50.5002 sec.\n",
            "イテレーション 5090 || Loss: 0.0252 || 10iter: 50.7622 sec.\n",
            "イテレーション 5100 || Loss: 0.0398 || 10iter: 50.7147 sec.\n",
            "イテレーション 5110 || Loss: 0.0197 || 10iter: 50.9360 sec.\n",
            "イテレーション 5120 || Loss: 0.0534 || 10iter: 51.0057 sec.\n",
            "-------------\n",
            "epoch 28 || Epoch_TRAIN_Loss:0.0409 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1027.6655 sec.\n",
            "-------------\n",
            "Epoch 29/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 5130 || Loss: 0.0306 || 10iter: 28.4291 sec.\n",
            "イテレーション 5140 || Loss: 0.0415 || 10iter: 50.8266 sec.\n",
            "イテレーション 5150 || Loss: 0.0370 || 10iter: 50.7818 sec.\n",
            "イテレーション 5160 || Loss: 0.0601 || 10iter: 50.8149 sec.\n",
            "イテレーション 5170 || Loss: 0.0503 || 10iter: 50.8799 sec.\n",
            "イテレーション 5180 || Loss: 0.0440 || 10iter: 50.9626 sec.\n",
            "イテレーション 5190 || Loss: 0.0402 || 10iter: 51.0384 sec.\n",
            "イテレーション 5200 || Loss: 0.0342 || 10iter: 51.1898 sec.\n",
            "イテレーション 5210 || Loss: 0.0257 || 10iter: 50.7419 sec.\n",
            "イテレーション 5220 || Loss: 0.0337 || 10iter: 50.7731 sec.\n",
            "イテレーション 5230 || Loss: 0.0375 || 10iter: 51.0896 sec.\n",
            "イテレーション 5240 || Loss: 0.0408 || 10iter: 50.7951 sec.\n",
            "イテレーション 5250 || Loss: 0.0581 || 10iter: 50.9631 sec.\n",
            "イテレーション 5260 || Loss: 0.0330 || 10iter: 51.1875 sec.\n",
            "イテレーション 5270 || Loss: 0.0416 || 10iter: 50.6970 sec.\n",
            "イテレーション 5280 || Loss: 0.0609 || 10iter: 51.4237 sec.\n",
            "イテレーション 5290 || Loss: 0.0284 || 10iter: 50.8623 sec.\n",
            "イテレーション 5300 || Loss: 0.0513 || 10iter: 50.8237 sec.\n",
            "-------------\n",
            "epoch 29 || Epoch_TRAIN_Loss:0.0403 ||Epoch_VAL_Loss:0.0000\n",
            "timer:  1030.5322 sec.\n",
            "-------------\n",
            "Epoch 30/30\n",
            "-------------\n",
            "（train）\n",
            "イテレーション 5310 || Loss: 0.0374 || 10iter: 11.5215 sec.\n",
            "イテレーション 5320 || Loss: 0.0410 || 10iter: 51.1535 sec.\n",
            "イテレーション 5330 || Loss: 0.0236 || 10iter: 50.8336 sec.\n",
            "イテレーション 5340 || Loss: 0.0400 || 10iter: 50.9525 sec.\n",
            "イテレーション 5350 || Loss: 0.0197 || 10iter: 50.9832 sec.\n",
            "イテレーション 5360 || Loss: 0.0253 || 10iter: 51.2067 sec.\n",
            "イテレーション 5370 || Loss: 0.0264 || 10iter: 50.6327 sec.\n",
            "イテレーション 5380 || Loss: 0.0523 || 10iter: 50.8362 sec.\n",
            "イテレーション 5390 || Loss: 0.0522 || 10iter: 50.9922 sec.\n",
            "イテレーション 5400 || Loss: 0.0436 || 10iter: 50.8677 sec.\n",
            "イテレーション 5410 || Loss: 0.0416 || 10iter: 50.6652 sec.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "イテレーション 5420 || Loss: 0.0194 || 10iter: 50.6732 sec.\n",
            "イテレーション 5430 || Loss: 0.0428 || 10iter: 50.8644 sec.\n",
            "イテレーション 5440 || Loss: 0.0494 || 10iter: 50.8474 sec.\n",
            "イテレーション 5450 || Loss: 0.0507 || 10iter: 50.8454 sec.\n",
            "イテレーション 5460 || Loss: 0.0448 || 10iter: 51.0749 sec.\n",
            "イテレーション 5470 || Loss: 0.0192 || 10iter: 51.0534 sec.\n",
            "イテレーション 5480 || Loss: 0.0663 || 10iter: 50.8508 sec.\n",
            "イテレーション 5490 || Loss: 0.0642 || 10iter: 50.8363 sec.\n",
            "-------------\n",
            "（val）\n",
            "-------------\n",
            "epoch 30 || Epoch_TRAIN_Loss:0.0399 ||Epoch_VAL_Loss:0.0701\n",
            "timer:  1352.9696 sec.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zXHjujLZ5Dk8"
      },
      "source": [
        "以上"
      ]
    }
  ]
}