SGAN_Train.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SGAN Train.ipynb",
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "438fb649c4484ead9239e946e73c75ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_01b475744e8240f683f4e086c51ab76d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_79879938ff2e43f8b8a66401b4073973",
              "IPY_MODEL_cd452113cc1c48118eadeebc03b2cfb0"
            ]
          }
        },
        "01b475744e8240f683f4e086c51ab76d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79879938ff2e43f8b8a66401b4073973": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fb1e7f36a3784c6fab6c8b39f19b3802",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Items: 500",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c9a6686ba8447d79844c95de1478db2"
          }
        },
        "cd452113cc1c48118eadeebc03b2cfb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_127555a87b3247ffa9ac2e2af965d37e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_881753281cac4ccbb76942716a2a50b1"
          }
        },
        "fb1e7f36a3784c6fab6c8b39f19b3802": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c9a6686ba8447d79844c95de1478db2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "127555a87b3247ffa9ac2e2af965d37e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "881753281cac4ccbb76942716a2a50b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "587f6c9ca8274b75b9c11c4000172b51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "state": {
            "_view_name": "VBoxView",
            "_dom_classes": [],
            "_model_name": "VBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0f1883e15c8040539177baec4d6cc962",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_33de301c07204a5f9608d8b10e4e62c9",
              "IPY_MODEL_d8c9d1b3fc46463caef091fedfd3e3d8"
            ]
          }
        },
        "0f1883e15c8040539177baec4d6cc962": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "33de301c07204a5f9608d8b10e4e62c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e31ac088c0b8492994ca1a5427831f4c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Items: 500",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e297f76b6954da8a903ab4ea8cb3bc3"
          }
        },
        "d8c9d1b3fc46463caef091fedfd3e3d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c87ba21287ce4202a2c46578cf795516",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 500,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 500,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3cd0cf6a2c2a4c739bf93ffd3b21bc2e"
          }
        },
        "e31ac088c0b8492994ca1a5427831f4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e297f76b6954da8a903ab4ea8cb3bc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c87ba21287ce4202a2c46578cf795516": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3cd0cf6a2c2a4c739bf93ffd3b21bc2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VX_vaZCdg66Q",
        "outputId": "04fa2da0-d8b9-49fd-e658-b4eb38368c65"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QvYDzQccgMg_",
        "outputId": "7e6e0c93-0df6-4fb8-a4f1-2b7841022e30"
      },
      "source": [
        "%tensorflow_version 1.x\r\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E5wGeyZtDe2p",
        "outputId": "c56d6df7-5bf0-447d-aeda-c6e8c08edb0c"
      },
      "source": [
        "!wget 'https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip'\r\n",
        "!unzip cats_and_dogs_filtered.zip >& tmp"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-12-30 17:13:31--  https://storage.googleapis.com/mledu-datasets/cats_and_dogs_filtered.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 172.217.193.128, 172.217.204.128, 172.217.203.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|172.217.193.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 68606236 (65M) [application/zip]\n",
            "Saving to: ‘cats_and_dogs_filtered.zip’\n",
            "\n",
            "cats_and_dogs_filte 100%[===================>]  65.43M   183MB/s    in 0.4s    \n",
            "\n",
            "2020-12-30 17:13:32 (183 MB/s) - ‘cats_and_dogs_filtered.zip’ saved [68606236/68606236]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y8VaukPJgclY",
        "outputId": "438c3930-b688-474f-bf1f-125c65f7c977"
      },
      "source": [
        "!git clone https://github.com/zaidalyafeai/sgan\n",
        "%cd sgan"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'stylegan2'...\n",
            "remote: Enumerating objects: 52, done.\u001b[K\n",
            "remote: Counting objects: 100% (52/52), done.\u001b[K\n",
            "remote: Compressing objects: 100% (39/39), done.\u001b[K\n",
            "remote: Total 949 (delta 25), reused 33 (delta 13), pack-reused 897\u001b[K\n",
            "Receiving objects: 100% (949/949), 107.30 MiB | 58.54 MiB/s, done.\n",
            "Resolving deltas: 100% (600/600), done.\n",
            "/content/stylegan2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZoyFYgED0wM"
      },
      "source": [
        "dim = (256, 256)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196,
          "referenced_widgets": [
            "438fb649c4484ead9239e946e73c75ee",
            "01b475744e8240f683f4e086c51ab76d",
            "79879938ff2e43f8b8a66401b4073973",
            "cd452113cc1c48118eadeebc03b2cfb0",
            "fb1e7f36a3784c6fab6c8b39f19b3802",
            "1c9a6686ba8447d79844c95de1478db2",
            "127555a87b3247ffa9ac2e2af965d37e",
            "881753281cac4ccbb76942716a2a50b1",
            "587f6c9ca8274b75b9c11c4000172b51",
            "0f1883e15c8040539177baec4d6cc962",
            "33de301c07204a5f9608d8b10e4e62c9",
            "d8c9d1b3fc46463caef091fedfd3e3d8",
            "e31ac088c0b8492994ca1a5427831f4c",
            "7e297f76b6954da8a903ab4ea8cb3bc3",
            "c87ba21287ce4202a2c46578cf795516",
            "3cd0cf6a2c2a4c739bf93ffd3b21bc2e"
          ]
        },
        "id": "3M-zNORkgjsh",
        "outputId": "15e9787a-b325-4fd3-9137-22d3c767a59e"
      },
      "source": [
        "from datasets import Dataset\r\n",
        "\r\n",
        "dataset = Dataset('../cats_and_dogs_filtered/validation/cats', dim = dim)\r\n",
        "dataset.prepare('dataset/records')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "resizing images ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "438fb649c4484ead9239e946e73c75ee",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(HTML(value=''), IntProgress(value=0, max=500)))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "creating records ...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "587f6c9ca8274b75b9c11c4000172b51",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "VBox(children=(HTML(value=''), IntProgress(value=0, max=500)))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Added 500 images.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BPyug4mhnEEz",
        "outputId": "f4efaf94-a3d8-43b7-c63c-c920ebe0d77c"
      },
      "source": [
        "from sgan import SGAN\n",
        "model = SGAN(dim  = (256, 256))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "stylegan2-ffhq-config-f.pkl: 382MB [00:04, 89.1MB/s]                          \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Constructing networks...\n",
            "Saving network-initial-config-f-256x256-0.pkl\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5mniebmwJiy",
        "outputId": "52593824-ed48-4f69-d476-33d324ae709a"
      },
      "source": [
        "model.train(data_path = 'dataset/records', out_dir = '../drive/MyDrive/Cats/ckpts/')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING: Entity <function TFRecordDataset.parse_tfrecord_tf at 0x7f02a4b1f620> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: module 'gast' has no attribute 'Num'\n",
            "Creating output directory...\n",
            "Loading training set...\n",
            "Image shape: [3, 256, 256]\n",
            "Label shape: [0]\n",
            "\n",
            "Constructing networks...\n",
            "Resuming from \"surgery.pkl\"\n",
            "Exporting sample images...\n",
            "Replicating networks across 1 GPUs...\n",
            "Initializing augmentations...\n",
            "Setting up optimizers...\n",
            "Constructing training graph...\n",
            "Finalizing training ops...\n",
            "Initializing metrics...\n",
            "Training for 10000 kimg...\n",
            "\n",
            "tick 0     kimg 0.1      time 2m 23s       sec/tick 42.9    sec/kimg 335.24  maintenance 100.5  gpumem 6.9   augment 0.001\n",
            "tick 1     kimg 10.2     time 34m 18s      sec/tick 1866.8  sec/kimg 184.61  maintenance 48.1   gpumem 6.9   augment 0.001\n",
            "tick 2     kimg 20.4     time 1h 05m 56s   sec/tick 1871.5  sec/kimg 185.08  maintenance 26.0   gpumem 6.9   augment 0.029\n",
            "tick 3     kimg 30.5     time 1h 37m 42s   sec/tick 1879.3  sec/kimg 185.84  maintenance 26.7   gpumem 6.9   augment 0.077\n",
            "tick 4     kimg 40.6     time 2h 09m 29s   sec/tick 1880.9  sec/kimg 186.00  maintenance 26.4   gpumem 6.9   augment 0.116\n",
            "tick 5     kimg 50.7     time 2h 41m 16s   sec/tick 1879.8  sec/kimg 185.90  maintenance 26.8   gpumem 6.9   augment 0.141\n",
            "tick 6     kimg 60.8     time 3h 13m 03s   sec/tick 1882.3  sec/kimg 186.14  maintenance 25.5   gpumem 6.9   augment 0.142\n",
            "tick 7     kimg 70.9     time 3h 44m 52s   sec/tick 1882.8  sec/kimg 186.20  maintenance 25.5   gpumem 6.9   augment 0.159\n",
            "tick 8     kimg 81.0     time 4h 16m 41s   sec/tick 1883.5  sec/kimg 186.27  maintenance 26.1   gpumem 6.9   augment 0.173\n",
            "tick 9     kimg 91.1     time 4h 48m 29s   sec/tick 1882.0  sec/kimg 186.11  maintenance 25.7   gpumem 6.9   augment 0.187\n",
            "tick 10    kimg 101.2    time 5h 20m 19s   sec/tick 1883.9  sec/kimg 186.30  maintenance 26.3   gpumem 6.9   augment 0.191\n",
            "tick 11    kimg 111.4    time 5h 52m 09s   sec/tick 1884.0  sec/kimg 186.31  maintenance 25.7   gpumem 6.9   augment 0.195\n",
            "tick 12    kimg 121.5    time 6h 23m 55s   sec/tick 1881.1  sec/kimg 186.02  maintenance 25.6   gpumem 6.9   augment 0.191\n",
            "tick 13    kimg 131.6    time 6h 55m 33s   sec/tick 1873.4  sec/kimg 185.27  maintenance 23.9   gpumem 6.9   augment 0.207\n",
            "tick 14    kimg 141.7    time 7h 27m 12s   sec/tick 1875.5  sec/kimg 185.47  maintenance 23.8   gpumem 6.9   augment 0.221\n",
            "tick 15    kimg 151.8    time 7h 58m 51s   sec/tick 1875.1  sec/kimg 185.43  maintenance 23.9   gpumem 6.9   augment 0.220\n",
            "tick 16    kimg 161.9    time 8h 30m 30s   sec/tick 1874.9  sec/kimg 185.42  maintenance 23.8   gpumem 6.9   augment 0.221\n",
            "tick 17    kimg 172.0    time 9h 02m 06s   sec/tick 1872.3  sec/kimg 185.16  maintenance 23.9   gpumem 6.9   augment 0.238\n",
            "tick 18    kimg 182.1    time 9h 33m 45s   sec/tick 1874.8  sec/kimg 185.41  maintenance 23.8   gpumem 6.9   augment 0.232\n",
            "tick 19    kimg 192.3    time 10h 05m 25s  sec/tick 1876.3  sec/kimg 185.55  maintenance 24.3   gpumem 6.9   augment 0.241\n",
            "tick 20    kimg 202.4    time 10h 37m 06s  sec/tick 1876.8  sec/kimg 185.60  maintenance 23.8   gpumem 6.9   augment 0.257\n",
            "tick 21    kimg 212.5    time 11h 08m 43s  sec/tick 1873.3  sec/kimg 185.26  maintenance 23.9   gpumem 6.9   augment 0.253\n",
            "tick 22    kimg 222.6    time 11h 40m 22s  sec/tick 1875.3  sec/kimg 185.46  maintenance 23.9   gpumem 6.9   augment 0.262\n",
            "tick 23    kimg 232.7    time 12h 12m 02s  sec/tick 1875.6  sec/kimg 185.48  maintenance 23.8   gpumem 6.9   augment 0.259\n",
            "tick 24    kimg 242.8    time 12h 43m 41s  sec/tick 1875.7  sec/kimg 185.49  maintenance 23.9   gpumem 6.9   augment 0.268\n",
            "tick 25    kimg 252.9    time 13h 15m 18s  sec/tick 1873.2  sec/kimg 185.24  maintenance 23.8   gpumem 6.9   augment 0.253\n",
            "tick 26    kimg 263.0    time 13h 46m 58s  sec/tick 1875.9  sec/kimg 185.51  maintenance 24.0   gpumem 6.9   augment 0.268\n",
            "tick 27    kimg 273.2    time 14h 18m 38s  sec/tick 1876.0  sec/kimg 185.52  maintenance 23.7   gpumem 6.9   augment 0.274\n",
            "tick 28    kimg 283.3    time 14h 50m 18s  sec/tick 1875.9  sec/kimg 185.52  maintenance 24.1   gpumem 6.9   augment 0.265\n",
            "tick 29    kimg 293.4    time 15h 21m 55s  sec/tick 1873.9  sec/kimg 185.32  maintenance 23.5   gpumem 6.9   augment 0.279\n",
            "tick 30    kimg 303.5    time 15h 53m 35s  sec/tick 1876.0  sec/kimg 185.52  maintenance 23.7   gpumem 6.9   augment 0.293\n",
            "tick 31    kimg 313.6    time 16h 25m 15s  sec/tick 1876.4  sec/kimg 185.56  maintenance 23.5   gpumem 6.9   augment 0.284\n",
            "tick 32    kimg 323.7    time 16h 56m 55s  sec/tick 1876.2  sec/kimg 185.54  maintenance 23.5   gpumem 6.9   augment 0.291\n",
            "tick 33    kimg 333.8    time 17h 28m 32s  sec/tick 1873.9  sec/kimg 185.31  maintenance 23.6   gpumem 6.9   augment 0.287\n",
            "tick 34    kimg 343.9    time 18h 00m 12s  sec/tick 1876.5  sec/kimg 185.57  maintenance 23.7   gpumem 6.9   augment 0.298\n",
            "tick 35    kimg 354.0    time 18h 31m 53s  sec/tick 1876.6  sec/kimg 185.58  maintenance 23.9   gpumem 6.9   augment 0.297\n",
            "tick 36    kimg 364.2    time 19h 03m 33s  sec/tick 1876.5  sec/kimg 185.57  maintenance 23.8   gpumem 6.9   augment 0.301\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}