FCPyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "PyTorch Dataset: https://pytorch.org/tutorials/beginner/data_loading_tutorial.html\n",
    "\n",
    "NOTE:\n",
    "1. The validation range is changed to small size for debug."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.utils.data import sampler\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#import torchvision.datasets as dset\n",
    "import torchvision.transforms as T\n",
    "\n",
    "import time\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device: cpu\n"
     ]
    }
   ],
   "source": [
    "image_dir = '../data/train_images'\n",
    "image_resacled_dir = '../yi_data/train_512_512_3'\n",
    "NUM_TRAIN = 6800 #8492\n",
    "\n",
    "USE_GPU = False\n",
    "dtype = torch.float32   # use float throughout the training\n",
    "print_every = 10\n",
    "\n",
    "SCALE_SZ = 512\n",
    "BATCH_SZ = 32\n",
    "\n",
    "if USE_GPU and torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "    \n",
    "print('using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Customized Dataset\n",
    "class ProstateCancerDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, csv_file, root_dir, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed\n",
    "        self.cancer_df = pd.read_csv(csv_file).sample(frac=1, random_state=1)\n",
    "        # Use DataLoader to shuffer data\n",
    "        # self.cancer_df = pd.read_csv(csv_file)\n",
    "        self.root_dir = root_dir\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.cancer_df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #start_tm = time.time()\n",
    "        img_path = os.path.join(self.root_dir, f'{self.cancer_df.iloc[idx, 0]}.tiff')\n",
    "        # (D,W,H)\n",
    "        #img = io.imread(img_path)\n",
    "        img = io.MultiImage(img_path)[-1] # conserve_memory=True  Turn off to improve performance\n",
    "        isup = self.cancer_df.iloc[idx, 2]\n",
    "        gleason = self.cancer_df.iloc[idx, 3]\n",
    "        # Recommend using downsample rate of 16 to speed up resizing\n",
    "        sample = {'image': img, 'isup_grade': isup, 'gleason_score': gleason}\n",
    "        #print(f'Elapsed loading time: {time.time()-start_tm}, {img.shape}')\n",
    "        \n",
    "        start_tm = time.time()\n",
    "        if self.transform:\n",
    "            sample = self.transform(sample)\n",
    "        #print(f'Elapsed transfer time: {time.time()-start_tm}, {img.shape}')\n",
    "        return sample        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Customized Dataset\n",
    "class ProstateCancerDatasetFast(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, csv_file, root_dir, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed\n",
    "        self.cancer_df = pd.read_csv(csv_file).sample(frac=1, random_state=1)\n",
    "        # Use DataLoader to shuffer data\n",
    "        # self.cancer_df = pd.read_csv(csv_file)\n",
    "        self.root_dir = root_dir\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.cancer_df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #start_tm = time.time()\n",
    "        img_path = os.path.join(self.root_dir, f'{self.cancer_df.iloc[idx, 0]}.png')\n",
    "        # (D,W,H)\n",
    "        img = io.imread(img_path)\n",
    "        #img = io.MultiImage(img_path)[-1] # conserve_memory=True  Turn off to improve performance\n",
    "        isup = self.cancer_df.iloc[idx, 2]\n",
    "        gleason = self.cancer_df.iloc[idx, 3]\n",
    "        # Recommend using downsample rate of 16 to speed up resizing\n",
    "        sample = {'image': img, 'isup_grade': isup, 'gleason_score': gleason}\n",
    "        #print(f'Elapsed loading time: {time.time()-start_tm}, {img.shape}')\n",
    "        \n",
    "        start_tm = time.time()\n",
    "        if self.transform:\n",
    "            sample = self.transform(sample)\n",
    "        #print(f'Elapsed transfer time: {time.time()-start_tm}, {img.shape}')\n",
    "        return sample        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Customize Transforms\n",
    "class Rescale(object):\n",
    "    \"\"\"Rescale the image sample to the given size\n",
    "    Args:\n",
    "        output_size (tuple): Desired output size. Output is matched to output_size.\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, output_size):\n",
    "        assert isinstance(output_size, tuple)\n",
    "        self.output_size = output_size\n",
    "        \n",
    "    def __call__(self, sample):\n",
    "        #start_tm = time.time()\n",
    "        img = transform.resize(sample['image'], self.output_size)\n",
    "        #print(f'Elapsed rescale time: {time.time()-start_tm}, {sample[\"image\"].shape}')\n",
    "        return {'image': img, 'isup_grade': sample['isup_grade'], 'gleason_score': sample['gleason_score']}\n",
    "    \n",
    "\n",
    "class ToTensor(object):\n",
    "    \"\"\"Convert ndarrays in sample to Tensors.\"\"\"\n",
    "    \n",
    "    def __call__(self, sample):\n",
    "        img = sample['image']        \n",
    "        # Swap color axis to [C,H,W]\n",
    "        img = img.transpose(2,0,1)\n",
    "        return {'image': img, 'isup_grade': sample['isup_grade'], 'gleason_score': sample['gleason_score']}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "train_cancer = ProstateCancerDataset(csv_file='train_512.csv',\n",
    "                                     root_dir=image_dir,\n",
    "                                     transform=Rescale((SCALE_SZ, SCALE_SZ)))\n",
    "\n",
    "print(len(train_cancer))\n",
    "for i in range(len(train_cancer)):\n",
    "    sample = train_cancer[i]\n",
    "    print(i, sample['image'].shape, sample['isup_grade'], sample['gleason_score'])\n",
    "    if i == 2: \n",
    "        plt.imshow(sample['image'])\n",
    "    if i > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Compse tranforms of rescale and totensor\n",
    "# More transformer to try: crop, normalize, etc.\n",
    "# And transformer is a useful tool for data augmentation\n",
    "biopsy_train = ProstateCancerDataset(csv_file='train_512.csv',\n",
    "                                     root_dir=image_dir,\n",
    "                                     transform=T.Compose([\n",
    "                                         Rescale((SCALE_SZ, SCALE_SZ)),\n",
    "                                         ToTensor()\n",
    "                                     ]))\n",
    "\n",
    "loader_train = DataLoader(biopsy_train, batch_size=16, num_workers=1,\n",
    "                          sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN)))\n",
    "\n",
    "loader_val = DataLoader(biopsy_train, batch_size=4, num_workers=4,\n",
    "                        sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, NUM_TRAIN+200)))  #8492  Use smaller size for debug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compse tranforms of totensor\n",
    "# More transformer to try: crop, normalize, etc.\n",
    "# And transformer is a useful tool for data augmentation\n",
    "biopsy_train = ProstateCancerDatasetFast(csv_file='train_512.csv',\n",
    "                                         root_dir=image_resacled_dir,\n",
    "                                         transform=ToTensor())\n",
    "\n",
    "loader_train = DataLoader(biopsy_train, batch_size=BATCH_SZ, num_workers=4,\n",
    "                          sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN)))\n",
    "\n",
    "loader_val = DataLoader(biopsy_train, batch_size=BATCH_SZ, num_workers=4,\n",
    "                        sampler=sampler.SubsetRandomSampler(range(NUM_TRAIN, 8492)))  #NUM_TRAIN+200  Use smaller size for debug"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "isup_distr = defaultdict(int)\n",
    "for batch_i, batch_sample in enumerate(loader_val):\n",
    "    for grade in batch_sample['isup_grade']:\n",
    "        #print(int(grade))\n",
    "        isup_distr[int(grade)] += 1\n",
    "    #print(batch_sample['image'][0].shape)\n",
    "    #for s_i, sample in enumerate(batch_sample['isup_grade']):\n",
    "    #    print(sample)\n",
    "    #print(batch_sample['isup_grade'])\n",
    "    #plt.imshow(batch_sample['image'][0].permute(1,2,0))\n",
    "print(isup_distr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Two-Layer Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_accuracy(loader, model):\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    model.eval() # Set model to evaluation mode\n",
    "    with torch.no_grad():\n",
    "        for batch in loader:\n",
    "            x = batch['image'].to(device=device, dtype=dtype)\n",
    "            y = batch['isup_grade'].to(device=device, dtype=torch.long)\n",
    "            scores = model(x)\n",
    "            _, preds = scores.max(1)\n",
    "            num_correct += (preds==y).sum()\n",
    "            num_samples += preds.size(0)\n",
    "            print(y[preds==y])\n",
    "        acc = float(num_correct) / num_samples\n",
    "        print('Got {:d}/{:d} correct {:.2f}'.format(num_correct, num_samples, acc*100))\n",
    "\n",
    "def flatten(x):\n",
    "    N = x.shape[0] # read in N, C, H, W\n",
    "    return x.view(N, -1)\n",
    "\n",
    "def train_sequential(model, optimizer, scheduler, epochs=1):\n",
    "    \"\"\"\n",
    "    Train a model using PyTorch Sequential API.\n",
    "    \n",
    "    Inputs:\n",
    "    - model: A PyTorch Module giving the model to train.\n",
    "    - optimizer: An Optimizer object we will use to train the model.\n",
    "    - epochs: The expected usage number of each image.\n",
    "    \n",
    "    Output: Print model accuracies.\n",
    "    \"\"\"\n",
    "    model = model.to(device=device)\n",
    "    for e in range(epochs):\n",
    "        print(f'Epoch {e}')\n",
    "        for t, batch in enumerate(loader_train):\n",
    "            x = batch['image'].to(device=device, dtype=dtype)\n",
    "            y = batch['isup_grade'].to(device=device, dtype=torch.long)\n",
    "            \n",
    "            scores = model(x)\n",
    "            loss = F.cross_entropy(scores, y)\n",
    "            \n",
    "            # Zero out all of the gradients for the variables which the optimizer\n",
    "            # will update.\n",
    "            optimizer.zero_grad()\n",
    "            \n",
    "            # Backward pass: compute the gradient of the loss with respect to\n",
    "            # each parameter of the model.\n",
    "            loss.backward()\n",
    "            \n",
    "            # Update the parameters of the model using the gradients computed by\n",
    "            # the backward pass.\n",
    "            optimizer.step()\n",
    "            \n",
    "            if t % print_every == 0:\n",
    "                print('Iteration {:d}, loss = {:.4f}'.format(t, loss.item()))\n",
    "                check_accuracy(loader_val, model)\n",
    "                # Decay learning rate after each validation check\n",
    "                scheduler.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0\n",
      "Iteration 0, loss = 72.1894\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2, 2, 2])\n",
      "tensor([2, 2, 2, 2, 2])\n",
      "tensor([2])\n",
      "tensor([2])\n",
      "Got 227/1692 correct 13.42\n",
      "Iteration 10, loss = 1.8002\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 1, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 466/1692 correct 27.54\n",
      "Iteration 20, loss = 1.7842\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 1, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 466/1692 correct 27.54\n",
      "Iteration 30, loss = 1.7625\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 1, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 1, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "Got 466/1692 correct 27.54\n",
      "Iteration 40, loss = 1.7866\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 1, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([1, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "Got 466/1692 correct 27.54\n",
      "Iteration 50, loss = 1.7800\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 60, loss = 93032.4219\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3])\n",
      "tensor([3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3, 3, 3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "tensor([3, 3, 3, 3])\n",
      "tensor([3, 3])\n",
      "tensor([3])\n",
      "tensor([3, 3, 3, 3, 3])\n",
      "tensor([3, 3, 3])\n",
      "Got 199/1692 correct 11.76\n",
      "Iteration 70, loss = 1.7001\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 80, loss = 1.7012\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 90, loss = 1.7304\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 100, loss = 1.7188\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 110, loss = 1.8229\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 120, loss = 1.7186\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 130, loss = 1.7994\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 140, loss = 1.7593\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 150, loss = 1.7062\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 160, loss = 1.7281\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 170, loss = 1.6595\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 180, loss = 1.6207\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 190, loss = 1.7997\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 200, loss = 1.7224\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n",
      "Iteration 210, loss = 1.8202\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
      "tensor([0, 0, 0, 0, 0, 0, 0, 0])\n",
      "Got 464/1692 correct 27.42\n"
     ]
    }
   ],
   "source": [
    "class Flatten(nn.Module):\n",
    "    def forward(self, x):\n",
    "        return flatten(x)\n",
    "    \n",
    "hidden_layer_size = 100\n",
    "learning_rate = 2e-2\n",
    "\n",
    "model = nn.Sequential(\n",
    "    Flatten(),\n",
    "    nn.Linear(3*SCALE_SZ*SCALE_SZ, hidden_layer_size),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(hidden_layer_size, 6)\n",
    ")\n",
    "\n",
    "# Use Nesterov momentum\n",
    "\"\"\"\n",
    "optimizer = optim.SGD(model.parameters(),\n",
    "                      lr=learning_rate,\n",
    "                      momentum=.9,\n",
    "                      nesterov=True)\n",
    "\"\"\"\n",
    "optimizer = optim.Adam(model.parameters(),\n",
    "                       lr=learning_rate)\n",
    "\n",
    "# Learning Rate scheduler\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9)\n",
    "\n",
    "train_sequential(model, optimizer, scheduler, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([3, 4])\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "a = torch.tensor([1, 2, 3, 4])\n",
    "b = torch.tensor([2, 1, 3, 4])\n",
    "print(a[a==b])"
   ]
  }
 ],
 "metadata": {
  "CodeCell": {
   "cm_config": {
    "lineWrapping": true
   }
  },
  "MarkdownCell": {
   "cm_config": {
    "lineWrapping": true
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}