Transfer_learning_capsnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.optim import Adam\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import csv\n",
    "from skimage import io\n",
    "\n",
    "from PIL import Image\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "\n",
    "verbose = False\n",
    "USE_CUDA = True\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "MNIST_USE = False\n",
    "CIFAR10_USE = False\n",
    "MARVEL_USE = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MARVEL_dataset(Dataset):\n",
    "    def __init__(self, dat_file,train = True, transform = None):   \n",
    "        self.root_dir = os.path.dirname(dat_file)\n",
    "        datContent = [i.strip().split(',') for i in open(dat_file).readlines()]\n",
    "        if train:\n",
    "            csv_file = os.path.join(self.root_dir, \"data_Train.csv\")\n",
    "        else:\n",
    "            csv_file = os.path.join(self.root_dir, \"data_Test.csv\")\n",
    "        with open(csv_file, \"w\") as f:\n",
    "            writer = csv.writer(f,delimiter=',')\n",
    "            writer.writerow([\"counter\", \"set\", \"class\", \"label\",\"location\"])\n",
    "            for line in datContent:\n",
    "                if train and line[1]=='1':\n",
    "                    if not(line[4] == '-'):\n",
    "                        writer.writerows([line])  \n",
    "                if not(train) and line[1] == '2':\n",
    "                    if not(line[4]=='-'):\n",
    "                        writer.writerows([line]) \n",
    "                \n",
    "        self.MARVEL_datafile = pd.read_csv(csv_file)       \n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.MARVEL_datafile)\n",
    "    \n",
    "    def __getitem__(self,idx):\n",
    "        img_name = self.MARVEL_datafile.iloc[idx,4]\n",
    "        image = self.__loadfile(img_name)\n",
    "        target = self.MARVEL_datafile.iloc[idx,2]\n",
    "        if self.transform:\n",
    "            image = Image.fromarray(image)\n",
    "            sample = self.transform(image)\n",
    "        else:\n",
    "            sample = image\n",
    "        return (sample,target)\n",
    "    \n",
    "    def __loadfile(self, data_file):\n",
    "        image = io.imread(data_file)\n",
    "        if len(image.shape)<3:\n",
    "            image = np.stack((image,)*3, axis=-1)\n",
    "        return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MARVEL\n",
      "Initializing Datasets and Dataloaders...\n",
      "Initializing Datasets and Dataloaders...\n"
     ]
    }
   ],
   "source": [
    "dataset_transform = transforms.Compose([\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),        \n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.247, 0.243, 0.261))\n",
    "])\n",
    "\n",
    "batch_size = 100\n",
    "\n",
    "\n",
    "if CIFAR10_USE: \n",
    "    NUM_CLASSES = 10\n",
    "    print(\"CIFAR10\")\n",
    "    image_datasets = {'train': datasets.CIFAR10('../data', train=True, download=True, transform=dataset_transform),'val': datasets.CIFAR10('../data', train=False, download=True, transform=dataset_transform)}\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "    dataloaders = {'train': torch.utils.data.DataLoader(image_datasets['train'], batch_size=batch_size, shuffle=True) , 'val': torch.utils.data.DataLoader(image_datasets['val'], batch_size=batch_size, shuffle=True) }\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "if MARVEL_USE: \n",
    "    NUM_CLASSES = 26\n",
    "    print(\"MARVEL\")\n",
    "    dat_file = \"/home/rita/JupyterProjects/EYE-SEA/DataSets/marveldataset2016-master/FINAL.dat\"\n",
    "\n",
    "    image_datasets = {'train': MARVEL_dataset(dat_file,train = True,transform=dataset_transform),'val': MARVEL_dataset(dat_file,train = False,transform=dataset_transform)}\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "    dataloaders = {'train': torch.utils.data.DataLoader(image_datasets['train'], batch_size=batch_size, shuffle=True) , 'val': torch.utils.data.DataLoader(image_datasets['val'], batch_size=batch_size, shuffle=True) }\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "if MNIST_USE: \n",
    "    NUM_CLASSES = 10\n",
    "    print(\"MNIST\")\n",
    "    image_datasets = {'train': datasets.MNIST('../data', train=True, download=True, transform=dataset_transform),'val': datasets.MNIST('../data', train=False, download=True, transform=dataset_transform)}\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "    dataloaders = {'train': torch.utils.data.DataLoader(image_datasets['train'], batch_size=batch_size, shuffle=True) , 'val': torch.utils.data.DataLoader(image_datasets['val'], batch_size=batch_size, shuffle=True) }\n",
    "    print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvLayer(nn.Module):\n",
    "    def __init__(self, in_channels=128,  out_channels=256, kernel_size=9):\n",
    "           \n",
    "        super(ConvLayer, self).__init__()\n",
    "\n",
    "        self.conv = nn.Conv2d(in_channels=in_channels,\n",
    "                               out_channels=out_channels,\n",
    "                               kernel_size=kernel_size,\n",
    "                               stride=1\n",
    "                             )\n",
    "\n",
    "    def forward(self, x):\n",
    "        if verbose: print( \"Conv input size{}\".format(x.size()))\n",
    "        output = F.relu(self.conv(x))\n",
    "        if verbose: print(\"Conv output feature matrix {}\".format(output.shape))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PrimaryCaps(nn.Module):\n",
    "    def __init__(self, num_capsules=8, in_channels=256, out_channels=32, kernel_size=9):\n",
    "\n",
    "        super(PrimaryCaps, self).__init__()\n",
    "\n",
    "        self.capsules = nn.ModuleList([\n",
    "            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=2, padding=0) \n",
    "                          for _ in range(num_capsules)])\n",
    "    \n",
    "    def forward(self, x, dimension = 32*6*6):\n",
    "        u = [capsule(x) for capsule in self.capsules]\n",
    "        u = torch.stack(u, dim=1)\n",
    "        if verbose: print( \"PrimaryCaps {}\".format(u.size()))\n",
    "        u = u.view(x.size(0), dimension, -1)\n",
    "        if verbose: print(\"PrimaryCaps size U {}\".format(u.size()))\n",
    "        output = self.squash(u)\n",
    "        if verbose: print(\"Primary Caps output {}\".format(output.size()))\n",
    "        return output\n",
    "    \n",
    "    def squash(self, input_tensor):\n",
    "        squared_norm = (input_tensor ** 2).sum(-1, keepdim=True)\n",
    "        output_tensor = squared_norm *  input_tensor / ((1. + squared_norm) * torch.sqrt(squared_norm))\n",
    "        if verbose: print(output_tensor.size())\n",
    "        return output_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DigitCaps(nn.Module):\n",
    "    def __init__(self, num_capsules=NUM_CLASSES, num_routes=32 * 6 * 6 , in_channels=8,  out_channels=16):\n",
    "        \n",
    "        super(DigitCaps, self).__init__()\n",
    "\n",
    "        self.in_channels = in_channels\n",
    "        self.num_routes = num_routes\n",
    "        self.num_capsules = num_capsules\n",
    "\n",
    "        self.W = nn.Parameter(torch.randn(1, num_routes, num_capsules, out_channels, in_channels))\n",
    "\n",
    "    def forward(self, x):\n",
    "        batch_size = x.size(0)\n",
    "        x = torch.stack([x] * self.num_capsules, dim=2).unsqueeze(4)\n",
    "        if verbose: print( \"DigitCaps x {}, W {}\".format(x.size(),self.W.size()))\n",
    "        W = torch.cat([self.W] * batch_size, dim=0)\n",
    "        if verbose: print(\"DigitCaps W {}\".format(W.size()))\n",
    "        u_hat = torch.matmul(W, x)\n",
    "        if verbose: print(\"DigitCaps u_hat {}\".format(u_hat.size()))\n",
    "        \n",
    "        b_ij = Variable(torch.zeros(1, self.num_routes, self.num_capsules, 1))\n",
    "        if USE_CUDA:\n",
    "            b_ij = b_ij.to(device)#cuda()\n",
    "\n",
    "        num_iterations = 3\n",
    "        for iteration in range(num_iterations):\n",
    "            c_ij = F.softmax(b_ij)\n",
    "            c_ij = torch.cat([c_ij] * batch_size, dim=0).unsqueeze(4)\n",
    "\n",
    "            s_j = (c_ij * u_hat).sum(dim=1, keepdim=True)\n",
    "            v_j = self.squash(s_j)\n",
    "            \n",
    "            if iteration < num_iterations - 1:\n",
    "                a_ij = torch.matmul(u_hat.transpose(3, 4), torch.cat([v_j] * self.num_routes, dim=1))\n",
    "                b_ij = b_ij + a_ij.squeeze(4).mean(dim=0, keepdim=True)\n",
    "\n",
    "        return v_j.squeeze(1)\n",
    "    \n",
    "    def squash(self, input_tensor):\n",
    "        squared_norm = (input_tensor ** 2).sum(-1, keepdim=True)\n",
    "        output_tensor = squared_norm *  input_tensor / ((1. + squared_norm) * torch.sqrt(squared_norm))\n",
    "        return output_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def caps_loss(data, x, target, reconstructions):\n",
    "    return margin_loss(x, target) + reconstruction_loss(data, reconstructions)\n",
    "\n",
    "def margin_loss( x, labels, size_average=True):\n",
    "    batch_size = x.size(0)\n",
    "\n",
    "    v_c = torch.sqrt((x**2).sum(dim=2, keepdim=True))\n",
    "\n",
    "    left = F.relu(0.9 - v_c).view(batch_size, -1)\n",
    "    right = F.relu(v_c - 0.1).view(batch_size, -1)\n",
    "\n",
    "    if verbose : print(\"Dimensions of labels {}, left {} right {}\".format(labels.shape,left.shape,right.shape))\n",
    "    loss = labels * left + 0.5 * (1.0 - labels) * right\n",
    "    loss = loss.sum(dim=1).mean()\n",
    "\n",
    "    return loss\n",
    "\n",
    "def reconstruction_loss( data, reconstructions):\n",
    "    mseloss = nn.MSELoss()\n",
    "    loss = mseloss(reconstructions.view(reconstructions.size(0), -1), data.view(reconstructions.size(0), -1))\n",
    "    if verbose : print(\"loss {}\".format(loss)) \n",
    "    return loss * 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def margin_loss( x, labels, size_average=True):\n",
    "    batch_size = x.size(0)\n",
    "\n",
    "    v_c = torch.sqrt((x**2).sum(dim=2, keepdim=True))\n",
    "\n",
    "    left = F.relu(0.9 - v_c).view(batch_size, -1)\n",
    "    right = F.relu(v_c - 0.1).view(batch_size, -1)\n",
    "\n",
    "    loss = labels * left + 0.5 * (1.0 - labels) * right\n",
    "    loss = loss.sum(dim=1).mean()\n",
    "\n",
    "    return loss\n",
    "\n",
    "def model_loss( x, target):\n",
    "    return margin_loss(x, target)\n",
    "\n",
    "def decoder(x, data):\n",
    "        classes = torch.sqrt((x ** 2).sum(2))\n",
    "        classes = F.softmax(classes)\n",
    "        \n",
    "        _, max_length_indices = classes.max(dim=1)\n",
    "        masked = Variable(torch.eye(NUM_CLASSES))\n",
    "        if USE_CUDA:\n",
    "            masked = masked.to(device)#cuda()\n",
    "        masked = masked.index_select(dim=0, index=max_length_indices.squeeze(1).data)\n",
    "        \n",
    "        return masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/2\n",
      "----------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:27: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:19: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:41: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train accuracy: 0.08\n",
      "masked [17  0 10  3  1  7 14  9  7 20 16 22  3 24  5  9 12  4  1 17  2 20 21 20\n",
      " 22 22 15  7 12  5  5 13 16 10 16  9  5 18 14  4 21  3  0  0 24  6 24 21\n",
      " 21  2 14 14 12 14  1 22  0 10 10 10  2 21  1 18 19 19 17 21  3 19 15  0\n",
      " 23 24 19  5 20  8 20  9  2 20  4 10 19 17 10 12 20 12 10  2 23 13  1 21\n",
      "  0  4  8  3]\n",
      "labels [17 19  6 19  0 11 24 11  7 16 11  9 24  2 13 22  2 25 17 17  8  3  2 20\n",
      "  3  3 25 22  2  9 18  3 20 16 15 17 19 10 14 18  5 13 13 17 20 17 18 12\n",
      " 12 11  3  7 22 16 10  6 18  5 24 10  2 23  3 23  0 22  8 15 23  9  3 24\n",
      " 12 20  9 10  4 21  5  6 25 10 11 18 18  5 13 12  8  2  5  6 22 23 17  1\n",
      " 10 21 22 16]\n",
      "train accuracy: 0.13\n",
      "masked [18 15 13 13 21  5  8  1 14  6  8 14  3 13 15 14 19 20  1 13 15 13  5 21\n",
      " 13 13 13 18  5  8  1 21 15  6  2 25  3  2 18  6 18 13 25 14 25  6 13 14\n",
      "  5 13  8 13  8  1 13 25  8  7  8  6 18 13 15  2 13  3  6 20 21 25 14  5\n",
      " 13 20  7 18 14 14  8  4 15 15 12  5 19 13 14 21 19 15 21  5 22  5 18  1\n",
      "  1 18  8 25]\n",
      "labels [ 6 15 17  6  4 13 20 22  6  8  4 18  8 14 15  7 16 22  4 13 15 15 24  2\n",
      " 20 23 15 14  7  3  0  0 15 11 10 25 18 10 19 15 18 11 16  2 10  3 23  4\n",
      " 22 13  0 14 17 11 17 16  8  4 11 20 20  0  9  2 16 16  9 21  3  4 18 19\n",
      "  6 22  8 20 19  5  4 25 15 17  7 13  5 15 18 21  2 23 11  1 22 15  3 20\n",
      "  6 11  1 20]\n",
      "train accuracy: 0.16\n",
      "masked [13 13 18 13 21 14  8 15 22  3  9 13 13  5 14  5 15 13  9  5  3  8  3  9\n",
      "  8 14  7 13  3 13  1  9  6 12 25  3 13 25  3 14  5  9  3  7 25 25 25 11\n",
      " 15 25 20 13  3 21 19  8  7  8  7 15 24  8 15  8 22  3 14 17  9 25  8  7\n",
      " 13 14  8  8 20 25 15 14 25  6 17 20 13 15  5  5 15  1  7 14  3 10 14 15\n",
      " 18 21 22  9]\n",
      "labels [ 4 20  3 13 18 22 17 21 11  6 25 23  5 15  7  5 18 17 11 13 22  6  4 10\n",
      "  8  1 12 17  8  6  1 15 25 12  3  8 20 25 12 14 16  2  0  6 21 22 21  2\n",
      " 15 12 23 15  1  6 18  8 20 22 22  6 20  0 15  0 25  6  2 16 18 10 21  4\n",
      " 15 14 10 17 16  5 15  3  7  8 11 20  5 15 14  8 13  4  0  1  4 15 11 18\n",
      " 10 21  3  9]\n",
      "train accuracy: 0.13\n",
      "masked [ 9 25 25 19 25  2  7 13 25 19 20 13  0  5 25  3  4  7 13  2  3 13  1 25\n",
      "  6  8 13 20 13  1 19  8  9 11 21 20  9 20 19 13  1 13 21  9  8  9  8  0\n",
      " 14  8 13 14  9  5 25  9 20  3 25 14 25 15 13 25 21 25  5 13  6  9 13  5\n",
      " 25 15  8 11  6 11  3 13 21  9 21  2  8  2 15  8 13 13 14  2 15  0  6 25\n",
      "  9 24 25  3]\n",
      "labels [18 21 25 11 19  0  6  5 20  8 16  9 21 12  1  4 14 24 18 24  2 25 12 21\n",
      " 22 10 14 15  5 11  8  3 23  7  1 23  9  4  7 22  6  5 24  9 17  9 24  1\n",
      " 23  8  2 17 18 10 10  9  0 25  4  0 23 16 25 25  0 14  3 14  7 18 19  6\n",
      " 25 15  5 10  6 10 22  5  1 15  0 22  4  2 15  5 10  1 18 20 15  2 21 19\n",
      "  7 11 12 12]\n",
      "train accuracy: 0.13\n",
      "masked [ 3  3 21 25  9 17  3 13 21 25  9 13 21 10 25  9 14 24  8 21  8 25  9  9\n",
      " 14 13  7 13 21 13 21  9  7  5 13 21  1 24  9 21 25 17  9 13 13 21 21  9\n",
      "  3 25  0  8 13 21 13 13 24  3 13  3  2 13 19 21 21  3 22 13  9  8 13 21\n",
      " 21  9 21 17  8  6 13  8 21 21  9  6  7 21 21  1  7 21  1  9 21  8 21 13\n",
      " 22 25 13  3]\n",
      "labels [ 8  6 10 19 11  5 24 23 20 25 11 19 19  0 13 15 22  8  4 21  7 21 18 15\n",
      "  3 12 17 13 23 18 25  1 24 15 23 12  0 12  1  1  4 20 24  9  5 23 21 10\n",
      " 11 21  4  8 13  1 13  9  2 12  5 16 19 23 24  4 11 16  2 11  9 23  5 20\n",
      " 24 10 24  0 19 21  9  6 20 11 16  6 21 24  1 21 20  0  1  9 21  8 12  5\n",
      " 25 16 12  1]\n",
      "train accuracy: 0.28\n",
      "masked [15  6 21  2 14 13 14 18  4 13 11  9 22  4 13  7  9 19 20  3 13  9 12  9\n",
      " 14 13 13  9 14 20  7  1  7 25  8  7 15  8 15 25  8  4 11  9  8 15  8 15\n",
      " 11  6 25 15  9 25 10 14 15  8  8  5 14 13 12 21 14 12  7 11 14 13 25 15\n",
      " 15  5 25 12  9 15  8 13  9  3  9 11  4  8 16 15  8  3  6  1  9  4  9 13\n",
      "  0  6  9 13]\n",
      "labels [15  6  1  2 17  5 16 10  6 23  7 18 22 17 19  1 18 14 22  4 13  9  6  2\n",
      " 17  5 13 16 14 20 22  2  1 10  8  1 11 20  5 10  8 22  1 14  2 12  8  3\n",
      " 11  6 20 15 18 20 23 22  9 22  2 13  8  5 12 21 22 12  1 11 22  5  6 10\n",
      " 19 24 16  4 18 15  4 13 18  3  9 11  3  3 22 15  8  2 23 25 10  4 25 11\n",
      " 10  7  9 19]\n",
      "train accuracy: 0.27\n",
      "masked [ 9  9 13 20 14 19 25  0 13 24  9 13  9 19  3  9 13  8  3 23  8 13  9 23\n",
      "  6  8  6  8 19 20  5  8 25 14  9  8 15  8 15 11 15 15  7  8  8 25 15 15\n",
      "  9  8  9 19  5 15  8  6  9  7  8  7 25 14  3  9  1  9 25 15 19 11 14  3\n",
      " 13  8 14  1 22  9 23  8 12  1 21  8  2 15 25 13 15 25 15 13  8  6  1  8\n",
      " 11 25  1  8]\n",
      "labels [ 1  9  5 20 17 16 21 25 23 24 15 13 20  7  2 25 20  8  2 17  8 23  9 13\n",
      "  7  3  6 20 19 12  5  8 21 14  6 16 10  8 15  1 12 17 12  2  8 21 21 18\n",
      " 18  8 24 21 17 12 17  6  9 12  2 11  1  5  2 18  1  9 23 20 19  4 22 18\n",
      " 10  2 19 11 21 10 13 14 22 21 24  3 24 15 25  5 20 21 15 23  2  6 11 14\n",
      " 24  0  1  8]\n",
      "train accuracy: 0.26\n",
      "masked [13 19 13  9  9  6 11  6  8  9 15  6  1 22 23  6  8  9  8 25 15  8  9  6\n",
      "  6 18 19 15 23  9 15  8 13 25  9 19  1  8 19  9  8  9 15  9 15  1  9 22\n",
      "  9 19 19  9 21  8 25 13 15 21  6 13 14 25  6  9 19 13 14 15  6  1  8 20\n",
      " 13 24  5 13  5  2 21 13  0 13 24  6 21 15  9 11  8  9  6 22  8  2 15 25\n",
      "  6  8 13 21]\n",
      "labels [23 19  5 20 10  6 11  3 18 18 19  4  6 12 16  0 14 18 17 25  6 14 17  6\n",
      "  3 25 22 10 23  3 15 17 13 25 17 24  7 14 17 15  2 18 15  4 15 11 18  1\n",
      " 21 16 12 16 13 20 17  3  3 25  4 13 14 25 12  9 16 13  3 24 12 20 20  7\n",
      " 13 17 23 14 17 25 13 13 20  5 21  4 12 12 15  1  8 21  6 22  8  3 17 25\n",
      "  6  8 21 21]\n",
      "train accuracy: 0.2\n",
      "masked [ 8  9  9 14 19  8 21  9 22 19 20  8 21  8 13 13  8 13 14 20 11 19 12 15\n",
      " 21 19  2  5 15 13  7 15 13 12  9  9  9 14 13  9  1  8  9  9 15  5  6  0\n",
      " 19  8 25  2 13 20 13 14 21  8 21 15  3 21 12 21 22 13 19 12 12 11  8 19\n",
      "  6  8 21 13  9  8  8 15  6 19 12 13 13  7  9  9  6 13 20 13 13 11  1  2\n",
      " 25 11  5 25]\n",
      "labels [ 3 18 12 14  1  3 12 22 24 19  2 20  6  8 16 23  8  5 10 17  0 24 24  0\n",
      " 24 23  2  3 17 13 20 20 19 21 22  9 17 25 16  9 11 17 15 16 25 16  6  4\n",
      "  5 11  0 22 23 22 13  2  7  8 23 25 22 12  2 11 22  5 11 21  3 11  8 23\n",
      " 11  8 21 23  2  8 14 15 12 19 21 19  5 11 15 15 12 23  7 24 23  1 11  3\n",
      " 25 16 11 24]\n",
      "train accuracy: 0.23\n",
      "masked [14 12  7 21  0  9  9 13 13 13  9 25 14  0  2  8 19 15 19 25  9  2 22 15\n",
      " 19 19 15 19 13 19 19  8 12 25 25  9 17  0 13 14 13 25 15 15 14 24 15  3\n",
      " 25 23 11  5  2  9  3  4 15 25  8 14 14  4 13  9  1  8  4 13 13  9 14 11\n",
      " 14 13  2 13 12 19  1 17 12  9 13 15  2  9 12 19  6 19  7  0  8  9  9 15\n",
      " 12 25  6 19]\n",
      "labels [ 2  5  3 10  0  9 18 23 16  5  2 25  3 20  2  2 23 17  5 24 10 21 12 15\n",
      " 16  7 17  1  5  7 12  8  3 25 10  6 17  0  0 14 20 22 16 15 14  8  1  2\n",
      " 21 13 20 14  3  9  4 24 17 11  3 17  5 23  5  2  7 22  4  2  1 18 18 11\n",
      "  6 16 12  5  4 14 12 17  4 15 16 23  0 10 18 18  6 11  7  0  3  9  9 15\n",
      "  7 17  4 19]\n",
      "train accuracy: 0.38\n",
      "masked [12  6  1  9 14 19  3 15  3 15  0  9  1  9 25 14 15 13  1  8  9  9 22  1\n",
      " 16  1  2  2 22 13 22  2  9  1 19 17  6  3  2 15  9 23 13  8 12 24  9  8\n",
      "  7 24 13  6  2  8 13 13  1  0 10 23 22 15 15 22  7 13  0  0 11 15 24  1\n",
      " 11 19 22 19  9  8 14  8 22 15  5 13 14  1 22  5  3 18 19 14 25  0 15 14\n",
      "  2 14 16 24]\n",
      "labels [10  3  6  9  3 19  3 15  2 19  0 15 21  9 10 14 17 16 22  8 18  9  4  1\n",
      " 16 12  2  2  0  9  4  3  9 24 22  6  6 18  2 15  9 13 13  3 15  0 10  8\n",
      " 17  1  5  6  2  2 16 13  0 10 12 23  3 15  5 20 17 23 12  4 11  8 24 24\n",
      " 24 19  1 25 18  8  7  8  7 11 19  1 14 11 20  6  3 18 19 14  4  4 15  2\n",
      "  2 23 16  7]\n",
      "train accuracy: 0.3\n",
      "masked [ 5 14 11  0  1  7 25 14  4 14  2 19 25  9  9  6 14  5 19 19 14 19 11  8\n",
      "  0  8  2  8 19  8 15 21 13  0  9 12 14 19 15 14 22  2 14  0  2 25 15  7\n",
      " 11 14 17 15 14  9 24 13  2  6 12 14 20 16  2 15  5 24 14  8  2  7 10 12\n",
      " 25  9  8 21  1 16  2 11 11  5 12 13 25 13  7 23  5 20 19 18 17 11 14  6\n",
      " 13 21 14  2]\n",
      "labels [12  5 22  6  1 11 25 17 20  5 23  3  4  9 18  6 17 19 24  7 18 25 11  8\n",
      "  6  8  2 14 19  8 15 12 19 25 18 12 14 12  9 25  9 14 17  0 16  1 15  1\n",
      "  1 23 17  5  6  9  0 10 14  6 17  5  5 23  2 15 11  0  0  8  3 20  5 10\n",
      " 23  9  8  1  1 16 18 11  6 22  4 21 25  0 10  6  7 20 16  9  6 20 14  6\n",
      " 17  3 20 19]\n",
      "train accuracy: 0.3\n",
      "masked [19  6 16  9  2 11 11  9 14  0 14  7  9 15 13  8 14 19 11  5 14  1  0  7\n",
      "  0 14 22  7  0 20 20  2 16  7  8  9 23 24 25 22 14  9 15  7 14 22  9 17\n",
      "  2 21  6  1  9  5  5 15 10  9  6  8 11 25  2 16  8 17 11  2  8  6  8 14\n",
      " 14 22  0 15  0  7  9  7  9 11 14 17  5 20 21 24 25  0 10 11 18  2  2 10\n",
      " 15 11 10 13]\n",
      "labels [24 12 16 18  3 25  1 18 20 22 16 25  9 10  5  8  5 16 17  5  5 22  1  7\n",
      "  7 22  2 18  0 22 17  2  7 23  8  9  7  1 12 20 20  9  9 22 23  4 18 17\n",
      "  2 12  6  1  9 24 17 15  1 18  4  8 22 25  2 16  2 17  1  3  2  6  8 10\n",
      " 22  5 11 17  0 25 18 11  9 21  8 22 19  7  0 23 25 16  1  7 18  2  3  4\n",
      " 15  1 23 13]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train accuracy: 0.29\n",
      "masked [11  0  6  1 16 15 17  6  6  6  2  8 11  8 13  1  5  1  1 23  8  8 14 11\n",
      "  9 14 13  6  8 11  1 13  6  1  6 14 22  1  1  7 11 25  9 24 13  9  2  2\n",
      " 13 25  8 17 19 19 13  0 17 11  9  2 11  1  0  2 20 14  2 25 10 25  1 11\n",
      " 11 14  1 13  8  2  6  1 19 14 11 11 17 19  9  1  1 15  9  2 14 25 13  6\n",
      "  1  2  1 11]\n",
      "labels [12  0  6  4  5 10 11 11  6  3  2  8 24  2 13 20 13 16 21 20  3  8 18 20\n",
      "  9 22 13  0  8  3  0 13  4 25  7 21 19 10 11 13 11 21  9 25 13 18 12 20\n",
      " 25 23  8 12 11 14 13  0 12 11 18  2 22 21  0  2 12 14  3 23 10 25 13 21\n",
      " 25 16  1 18  8 18  3 25 14  3 11 24 12 16 18 20  7 15 18  3  7 12  5  7\n",
      " 25  3 21 11]\n",
      "train accuracy: 0.29\n",
      "masked [14 19 15 14 11  0  0  7 14  7 14  6 11 17  8  6  2 15  9  2 13 17 19  0\n",
      "  1 14 11 20  0 25 18  0 14  0  7 13 14 11 25  1 24 22 25 21 11 11  5  1\n",
      " 25 11  2 20 11 11  1  9 20 15  6  4 19 23 23 13  6 14  9  9  0 11 21 15\n",
      "  7  6  1  5  6  6 19 19  0 15 11  6 17 13 14  1 13 14 14  9 19 15  2  6\n",
      " 15 25  1 24]\n",
      "labels [18  7 15 10 11  0 15 12 14 25 23 10 11 17  8  6 12 15 18  2 16 10  8  1\n",
      "  7 13  4 25  4 24  7  0 15  5  7  5 25 18 23 21 21 20 18 24 17  1 17  7\n",
      " 25 24  2 17  7 20 25  9 23 15 24  4 22  6 16 23  6 20  9  9 16  1 25 15\n",
      " 12 23 18 14 17 21 25 20 14 15 11 23 20 19 23  1 16 15 14  9 17 15  2 25\n",
      " 15  5 24  1]\n",
      "train accuracy: 0.35\n",
      "masked [ 3  8 25  5 19  2 25 19  9 25  8  9  3 23 14  8 20 14  8  1  8 17  0  6\n",
      "  5 15 11  9 24  6 13  6 13 14  4 14 11  9 25 11 10  1  1  1 14  6  6 10\n",
      " 11 19  0 13  5 11 19 25 11  9  1 15 15 14 11 14 25 14 16 14  1  8  6  2\n",
      "  9 11  2 25  5  6  8  6 11  0  3 15  0 14  2  7 14  5  8 14  9  6  4 11\n",
      "  9 14  9 25]\n",
      "labels [16  8  7 20 19  2 16 22  9 14  8  9  3 12 25  8  7 14 14 20  8 17 23 22\n",
      " 18 22 24  9  2  0  5 10  5 25 17 14 12  9  5 25 22 24  1  7 25 25 19 25\n",
      "  4 17 21 13 22  1  5 25  1  9 11 15 23  7 11 14  7  6  2  5 21  8  6  3\n",
      "  9 17  2 22 17  3  8 21 11 25  3 15 21  5  2 20 21 13  8  6  9  6 16 11\n",
      " 18 14 18 21]\n",
      "train accuracy: 0.32\n",
      "masked [ 0 14 17  8 23 11 11  2 16 14 25 19  0 14 15 11  5  7 18 12 10 14  1 11\n",
      "  6  6  5 23  0 17  8  2 13 14  0 25  5 14 15  9 14 25 14  0 11  6  3 24\n",
      "  8 24 21  8 15  0 19 11  7 10 25  6  0  5 15 16  0 25  9 11 13 11 23  0\n",
      " 25  2 21 21  5 15  8 11 19  9 14  8  0  6  2  0 14 14  1 19 17 19 19 11\n",
      " 21  8  0  2]\n",
      "labels [ 6 19 21  8 17 20 17  3  1 25 23 23  0 12 15  1  5  7 16  3 10 25  4 11\n",
      "  6  6  4 19 22 17  8 12  5 20  0 16  1 14 15 16 22  5 17 21 12  6 23  2\n",
      "  8 14 13  8 23 10 19 21  0  1 23 12  6  9 10 23  0 18 18 11 13  1 10  6\n",
      " 25  8 10 22  7 15  8 24 19 15  9  8  0 12  2  6  5 25 25  5 20  7 22 11\n",
      " 21  8 21  2]\n",
      "train accuracy: 0.41\n",
      "masked [15 23 17 11 14 20 10 22 11  9 24 14  8  2 14 25 20  2  6 23  9 17 25 20\n",
      " 24 11 11 13 15 13 12 25  0 18  6 25  2 25 25 19  8 13  6  9 24 14  1 14\n",
      " 14 15 14 11  6  2 25 14 17 14 14  6 17 11 11 14  7 13 10 23 21 14  0 25\n",
      " 10 23  2  6 15  9  6 15  8 24 19 19 19 14 25  9 14  2  8 25  6 14 11 25\n",
      "  0  5 19  6]\n",
      "labels [15 12 17 11 14  2 10  8 11  9 19 19  8 10  3 19  1  2  6 17 18 23 15 20\n",
      "  1  3 11 13 15 13 18  7  0 25  6 22  2 16 10 18 23  5  6  9 17 12  4  4\n",
      "  2 15 14 11  6  2 23 16 17 12 25  4 10 21 11 16 20  5 10 16 25 14  6 25\n",
      " 20  1  3  6 15 18  6 15  8  1 19  7  5  7 21  9 19  3  8 25  6 14 21 12\n",
      " 24 22 16 21]\n",
      "train accuracy: 0.31\n",
      "masked [ 2 25 11 24 11  7 25 25 13 13  9 13  6  6  6  1 23 11 14 25  0 18 23  9\n",
      " 15 17 14  9 25 19  7  9 17 11  5  2  0  9  8  1  2 23  0 23 23 13 25 14\n",
      "  1 11  1  6 17 14 23 14 17  9  0  0 11 13  4 14 21 14 11 10 24 17  6 17\n",
      " 13 17 14  6 14  6  1  1 10  6 11 19  9 24  1 11  6 17 13 23  3 21 14  5\n",
      "  2 23 25  4]\n",
      "labels [ 2 22 11 21 22  7 21 25  5 13  9  5  6  6  0 10 24 21 19 10 23 22 17 18\n",
      " 10  2 18  9 11 14 17 18  4 11  5  8  0  9  3 21  4 13 22 23 21 13 23 23\n",
      " 10 11 25  4 23 14 16 19  7  9  0  0 12 16  4 13  5 23  1 10  9 11 11  6\n",
      " 13  3 14  4 19  0  7 25 10  4  4  4  9  7  7 11  4 19 13  0 25  1 14  5\n",
      "  3 23 16 21]\n",
      "train accuracy: 0.38\n",
      "masked [ 4 11 18  1  9 19  6  6  1 20 23  8 19  1 14 13 25  9  8 19  6  8  0  7\n",
      " 20  7 13  1 11 10 10 23 13 14 21 23  9  0  8  4 10 19  9  6  3  1  1  9\n",
      "  5  7 15 13 15  8 25  1  0 17  1  5 17 19 25 15  1 25 17 14 11  6  8 23\n",
      "  8 21  1 18 15 17 11  6  1  2 17 19  1  0 25 15 11  2 19 19 19  8 21  1\n",
      " 11 23 17 14]\n",
      "labels [24 11 22 25 18 19  6  6 24  3  4  8 16 24  0  5  1 18  8 12  6  8  4 25\n",
      " 23 25 23 16 11 10  4 21  5 16  7 13  9  0  8 17 10 14  9  6 17 21 24  9\n",
      " 13 20 15 13 15  8  1 25  0 22 25 14  6 14  7 15 23 22  5 14 12  6  8 12\n",
      "  8  4 10 18  0  0  4  6  7  3 23 19 12  0  1 15 11  2  5 19 25  8 21 24\n",
      " 11 20 18 12]\n",
      "train accuracy: 0.45\n",
      "masked [25  0  6 19 17 16 25 17 17 11 11 21 13 15 14  8  8 18  9 14 10 23 11  8\n",
      " 19 20  5 23  8  1 15 25 20 13 19 21  1  0 17  0  6  1  8 19  9 11  1 17\n",
      " 21 13  1 11 20 13  8 25 17 23  2  2  0 13 17 15  1  6 21 15  1 11 17 11\n",
      " 18  7  9 19  2  1 25 13  2  0 15 20 20  8  0  8  5  6 22 23 15 20 11 14\n",
      " 21 21 11 23]\n",
      "labels [10  0  6 10 17 16 13 24 24 11 11 12  5 24 19  8  2  1 18  0 10 23 11  8\n",
      " 20 21 20  7  8  1 15  1 22 13 20 21 24  0 23  0 20  1 18 19  9 11  0 18\n",
      " 24 13 14 11  1  5  8 24 17 19  9  2  0 13 20 24  3 12 20 15  1 11 17 11\n",
      " 22 20  9  1  2 24 17 13  2  3 15  4 21 17  0 18 23  6 11 22 15  0 11 18\n",
      " 24 21 11  3]\n",
      "train accuracy: 0.38\n",
      "masked [25 11  9 19 14 20 21 13 25 17 14 14 23 15 19 21  6 15 25 14 25  6  2 14\n",
      "  8 14  5 17  0 11 17 14 20 17  8 14 21 17 10 25 25 21 20 21 21 10 17 17\n",
      " 13  6 11 10  6  1 15  6 25 11  2 23 21 14 23 21 15 23 17  4 25  0 21 15\n",
      "  0 25 23 11 25 14 15  9 10  0  3 19 20 17  0 17 16 24 11 21 21 25 14 11\n",
      " 18  9 11  1]\n",
      "labels [13 21  9 22  7 20  0 13 25 17 16 14 23 15 22 17  6 15 25 14  7 12  4 23\n",
      "  8 14  3  2 12 11 19 16 23 19  8  5 14  1 22 17 16 12  7 21 24 10 23 19\n",
      "  5 21  1 10  6  1 15  6 16 21 12 10  7  5 17 23 15 20 20 23 22 16 20 15\n",
      "  0 25 10 11 12 17 15  9  9 24 14 21 19  9  0  1 16 11 11  7 25 20 14 11\n",
      " 18 18 11  1]\n",
      "Epoch 1/2\n",
      "----------\n",
      "train accuracy: 0.28\n",
      "masked [17  9 25  0 14 18 17 21 19  1 14 11 15  1 19 14 25 22 14 14  5  9 14 14\n",
      " 16  1 15 17 25  0  1 23 21 14  6  1 19 25  2  7 18 14 15 14 14 11 19  6\n",
      "  8 14 25 14 19 14  9 18  5  4  4  7 25 25 24  0 15 18 24 19 13  6 17  2\n",
      " 16 14 19 10  0  4 19 19  9 17 19 14  7 23 20 25 21  5 15  2 11 22 23  2\n",
      "  1 11 21 10]\n",
      "labels [17 18 25  0 12  2 20 21 23 25  5 11 15 24 17 14  4  7  5  1  5  9 16 13\n",
      "  3  7 15 23 13  4 23 19 25  5  6 21 10  4  3  7 16 23 15 23  1 21  7  6\n",
      "  3  5 19 23 14  5  9 22 18 16  3 22 25 25  2  0 15 18 22  7 15 17 12  2\n",
      " 18  5 17 10  4 17  7 18  9 20 17 18 25 23 20 21 22 13 15  3 11  1  5  2\n",
      " 21  3  7 23]\n",
      "train accuracy: 0.43\n",
      "masked [16  9 20  2  6  0 14 14 14 13  0 15 11 23  5  3  0 15 15  9  6 13 14 23\n",
      " 21 15 21 25 17 17 15  2 17 25  0 17  2 21 25  2 11 24 20 19  2 21 24 19\n",
      " 10 19 21  9 19 19  1  0  9  6 11 14 20 10 20 10  4 19 10 25 19 11 11 13\n",
      "  8 15 19  2 13 15 19 18  3 24 11  9 13 21 10 17 14 21  5 21 19  8  9 20\n",
      "  6 14  1  0]\n",
      "labels [19 18  7 14  6  0 14  5 16 13  0 15 10 23 21  5  0 15 15  9  6 13 13 19\n",
      " 21 15 20  7 12 17 15  2 20  7 16  5  2 21 14 12  1  4 16 20  3 16 25 10\n",
      " 10 19 24 18 22 17 10  4  9  6 11 14  1 10 14 24  4 22  5  3 20  1 11 13\n",
      "  8 15 16  2 13 15 18 18 12  7 24 18  5 21 22 17 14 24 13 16 23  3  9 14\n",
      "  6 14  1  5]\n",
      "train accuracy: 0.44\n",
      "masked [16  1 19  7 25 11  6 20  1 25  6 13  7 17  8  5  6 13 20  2 17 13 19 10\n",
      " 21 17 19  9 23  1 15 14  1  9  1 20  8  6  9 18 25 11 10 25 25 11 14  9\n",
      "  9 20 19 14  5 11  2 25  5  5  8  2 23 25 24 20 10 19  3 19 25  0  2 14\n",
      " 14  9 20 20  6 25 11 23  0 19  2 20 21  6  9  6 19  9 11 15 10 17 14 19\n",
      " 17 13 16 11]\n",
      "labels [16 23 17  7 25  4 25 16 16 20  6 13  4 19  8 17 17  5 18  2 12 13 18 10\n",
      "  4 23 19 18 19  7 15 19  4  9 25 20  8  3  9  2 17 21 23 20 23 11 14  9\n",
      "  9  7 19 14  3  7  2 24 19 21  8  2 16  4 19 24 10 19 12 16 10  0  2 18\n",
      " 14  4 25 21  6 25 11 24  0  1  2  5 24  6  9  6  7  9 11 15  1 18 23 23\n",
      " 17 13 16 11]\n",
      "train accuracy: 0.45\n",
      "masked [13 19  9  0  6  5  4 19 22 11  2  0 17  6  1 19  0 15 14  2 13  6 25 19\n",
      "  1  2  8  2  9  6  0 19 23 19  5 11 22 19  6 14 15  0  6 15 10  5 11  9\n",
      "  0 14  6 17 15  9 20  1  8 24 14 25 14 13 25 23 16 15 13 25 25 19 25  9\n",
      "  6 22 19  5  6 15 21 25  2 21 19 17  9 15 11 13  8  3 20  2 20 11  2 25\n",
      " 20 11 19 11]\n",
      "labels [19 10  9  0  6 18  7 19  1 11  2  0  6  1 21 23  0 15 14 12 16  6 25  1\n",
      " 10  2  8  2 18  4  0  2 23 19 17 11  7  5  4 14  0  0 10 10  7 14 11 16\n",
      "  7 13  6 12 15  9 21 20  8 20 14 10 18 13 10  5 25 15  5 25 10 10 10  9\n",
      "  6 22 19 19  6 15  4 24  2  1 18 12  9 15 11 16  8 18 12  2 23 11  2 25\n",
      "  5 17 16 24]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train accuracy: 0.46\n",
      "masked [18 16  6 11 20  3 19  3 11 25  9 15  6 22 14  7  0  9 11  5 19  8 20  4\n",
      "  5 25  5 10 13  6 19  1  3 16 13 19 22  6  4 14  0 20 19 15 16 19 19  3\n",
      " 25  5 20 20  8  6  1 19 20 20 23 25 13 25 14 19  1 25  6 25 22 10 20 25\n",
      " 19  9 17 14  1  0 14 20 19 15 14  0  0 11 18 19  2  9 14  2  0  1  2  5\n",
      " 25 25 23  5]\n",
      "labels [18 16 24 11 20 12  7  4 11  0  9 20  6 21 14  1  0  9 11 25 24  8 17  4\n",
      "  5 16 14 10 13  6 23 21  3 17 23 22  7  6 21 20  0 24  1 10 10 19  7  3\n",
      " 23 22 25  4  8  6 23 22 25 12 17 10 13 22 16 24 17 25  6 25 22 10  1 25\n",
      " 19  9  2 14 21  4 14  8 17 15 14  0 21 11  9 16  3  9 14  2  0  1  2  2\n",
      " 25 23 12 20]\n",
      "train accuracy: 0.43\n",
      "masked [11 14  6 11 14 15  9  1 20  0  8  6 17  2  5  9 11 21 25 15 19  1  0 15\n",
      " 23  6  3 15 17  2  8  8 17 23 25 14  2 25 25  2 19 21 23  0 23  2 19  4\n",
      " 15 19 18 18 25 18 19 25 20 25  5  0 25  5 17 17  3 19  1 13 11  0  1 25\n",
      " 25  0 16 14  8  1 19 13 15  0 17 19  9 15 14 11 14 15 25 10 11  6 23 17\n",
      "  2 19  1  1]\n",
      "labels [11 12  6 11 16 15  9 22 23  0  2  6 12  2  7  9 11 12 10 15 20  1  0 15\n",
      " 21  4  3 24 10  2  8  3  7  5  2 14  2 23 21  3 19 24 25  0 10  2 11  6\n",
      " 15  4  9 18 25 18  3 18 19 25 16  0 25 13 10  4  3 24 24 23 20  0 24 20\n",
      " 12  0 16 14  8  7  0 13 23  0  7  3  9 15 12 24 14 15 21 10 21  6 16 24\n",
      " 12  7 20  4]\n",
      "train accuracy: 0.42\n",
      "masked [ 1  1  1 19 24 25 19 25  9 13 14 25  1  8  6  2 17  4 14 15 19  6  3 21\n",
      "  1 25 15 19  0 13 20 13 17  2 20 10 19 14 18 19 14 19 11 17  1 14  4 21\n",
      " 17 18 25 23 20 21  5 19  0 24  5 25 25 15  6  3  6  8 25 18 15 22 25  5\n",
      " 13 11  8 15 25  4 15 19  1 19  1  1 19 11  9 21  1 21 11  1 22 25  4  1\n",
      "  4 19 19  8]\n",
      "labels [ 0  1 12 18  1 25  3 25  9 13 16  4  1 14 12  2  3 10 14 15 21  6  3 21\n",
      " 12 24 15 20 24 13 20 13  5  2 12 23 22 19 18 23 14  5 11 10 25 14 21 25\n",
      " 17  9 19  5 20  7 20 20  0 17 23 21 25 15  6 14  6  8  2 18 15 23 10 10\n",
      "  5 11 13 15  0 12 15 19  1 22  1  4 23 11 20  7  1 21 11 25 18  4 17 21\n",
      " 10 16 19  2]\n",
      "train accuracy: 0.44\n",
      "masked [17 14 21  8 17  4 20 19 21  6  6 19 19 19  4 21 20 20 13  2 15 21 17  7\n",
      " 20  7 19 21  7 15 19  6 21  2 14  6 21 15  7  7 19  7  8 21 19 21 21 20\n",
      " 19  9 14 13  2 21  5 14 21 25 13 19 20 19 18  2 21  1 10  9 25  2 24  2\n",
      " 15  0 15 10  8 19 14 19 19 15 21  0 21  5 17 14 15 16 14 19 20 20  9 14\n",
      "  9  8 15 10]\n",
      "labels [21  3 20  3 17  3 20 20 25  6  6 19 16 10  6  4  8  7 13  2 10 21 18  1\n",
      "  0 22 20 21  9 15 19  6 17  2 14  6 12 15  1 16 23 12  8 24 23 24 24 23\n",
      " 16  9 17 13  2 17 10 18  3 20 13  3 16  1 18 25  1  4  7  7  4  2 24  2\n",
      "  2  0 15 25  8 12 14 19 20 15 22  0 21  5 17  3 15 16  5 19  5 20 15 14\n",
      "  9  8 15 10]\n",
      "train accuracy: 0.45\n",
      "masked [13 25 25 13 13  1  8 19  4  9 16  5  6  1  2  6  6  5  5 10  6  0 19  2\n",
      " 21 14  0 21 17 25 15 25 19 10  4 15 13 25 20 25  2 17  9 11  9  8 24  5\n",
      " 19  2 25 17  0 25 25  0  2  8 16 18  0  8  9  0 25  9 19 16 18  8 21 16\n",
      "  1 14 14  6 19  2  1 18 19  6 14  8 25 18 25 25  8 24  5 20  5 19 25 14\n",
      "  8  1  0 21]\n",
      "labels [13 21 21  5 14  3  8 19  4  9 20 10  6 21  3 10  4  5  5  2  4  0 23  2\n",
      " 10 14  0  3 22 25 15 24 23 22  7 15 13 22 17  7  2 17  9 11  9  8 14 13\n",
      " 19  2 25 16 20 22 21  0  3  8 23 18  4  8  9  1 25  9 14  5 18  8  5 25\n",
      " 20  2  8  6 19  2 24 22 17  4  5  8 24 18 25 25  8 25  5  1  1 17 25 19\n",
      "  8 17  7 22]\n",
      "train accuracy: 0.39\n",
      "masked [21  8 10 18  5 22 13 13  6 13 20 15  9 19 25 21 20 25 25 18  4 23  0 21\n",
      " 19 13 17 19 14  7 19 11 25 14 19  8 19  5  8  0 13  8 13  2 18 21 25 10\n",
      " 11  3 25 21  2  1 20 17 21  1  2 19 21  1 11 25 19 21 25 10 25 21 25 14\n",
      " 19  6 25  0 13 13 18 25 15  3 10  1 19 25 14 25 15  6  6  6 15 20 13  8\n",
      " 20  5 25 14]\n",
      "labels [11  8  7 18 16 22 13  5  6 13  0 15  9 19 18 23 17 22  7  2  4 23  0 12\n",
      " 17  5  2  5 14 20 16 11 23 14 25  8 12 14  8  0 19  8 13  2 18  4  1 14\n",
      " 11  4  4 16 21  1 21 25 17 22  3  1 20  1 11 17  0 19  7  1 14  1 22 14\n",
      "  3  6 17  0  5  5 18 22 15 14 24 21 12 24 14 25 15  6 17  4 15 24 13  8\n",
      "  4 16  1 14]\n",
      "train accuracy: 0.35\n",
      "masked [ 0 20  1  1 25 25  9 10  1 14 15 14  7 14 10 13 21 25  0 19 25 11 17 25\n",
      " 19 11 25  0  9 13  4 17 19 16 14  6 17 18 25 15  8 13 14 18 20 19 25 19\n",
      "  1 17 18 15  6  1 19 25 19 19 20 18 23  5  1  0 11 21 11 25  4 19 15 20\n",
      " 10  9  5 19 19  0  4  4  1 25 25 14 19  9  0  0 11 17 25  8 16 24 19 24\n",
      " 25  4 25  2]\n",
      "labels [ 0 14 18  1 24  4 16 17  7 23 15 16 25 22  9 13 21 16  0 21 22  1  5 25\n",
      " 16 11 24  5  9 13 12  3 23 16 14  3 22 18 24 15  8 13 14  4  1 12 25  4\n",
      "  1 25 18 15  6  4  7 13 20 16  8 11  4  5  8 12 11  1 11 23 17  4 15 25\n",
      " 11  9 13 19  8  0  7 22 18 24  7  5 16  9  7  0 11  2 17  2 19 15 19 17\n",
      "  7  4 16  2]\n",
      "train accuracy: 0.41\n",
      "masked [25 25  6  6  8  1  1 15 21 13 14 20 25 19  0 23  9  4  1  0  1 11  8  1\n",
      " 19  1  5  1  9  1 23 15 15 17 14 23  4 14 19 17  3 20 25  2 19  1  5  0\n",
      " 25 19  6  8 17 19  6 16 21 19 23  1 23 14 25 11 18  8 25 25 15 19  3 19\n",
      "  7  8 19 19 18  2 20 10 19  8  5  1  6 12 19 25  1 10  1 18  1 13 21  2\n",
      " 21  1 15  0]\n",
      "labels [25 14  6  6  8 25  7 17 22 13 14 17  7 19 25 24  9 20 21  0 24 11  2 10\n",
      " 20 21  2 17  9 24 22 15 15 16 14 20 25 14 23 16  3 20 25  2  4 17  5  0\n",
      "  1 12  6  8  5 19  4 22 16  4 15 17 16 14 23 21 18  8  8 13 15  7  2 23\n",
      " 10  8 16 19 18  2 17  2  3  8 25 18  6 12 20 23 23 15 18 18  1 13  3  2\n",
      " 12 21 15  0]\n",
      "train accuracy: 0.36\n",
      "masked [19 25  5 21 25  5 19  2 15 19  6 25 13 16 25 25  5  5  6  7 13 17 25  9\n",
      " 19 25 19 20  1 18 14 11 14 16  8  0  6  6  2  1 25  2  5 20 13  8 19 25\n",
      "  4  2 25 19 13  8  9  4 24  0  0  3 17 20 17  4 21 14 14  8  8  9  8  8\n",
      "  8 20 25  8 17 25  8  8 25 25 15  2 11  5  1 25 25 11  0  8 13 20 18 13\n",
      " 10 16 19 19]\n",
      "labels [ 3 17  4 22  1  2 19  3 15 10  4 22 13 16  0 20 19 18  6 20 13 17 21 23\n",
      " 14 25 22 20  1  9  5 11 14 19  3  0  6 24  3 21 20  3 10 21  5  8 12 17\n",
      " 25  2  3  3 13  8  2 24 12  1 12  3 24 19  3  4 18 14  3 20  8 18  8  8\n",
      "  8  3 10  8 21 16  8  8 10  4 18 18 11  5 22 22 24 11  0  8  5  3 18 13\n",
      " 10  7 25 22]\n",
      "train accuracy: 0.38\n",
      "masked [ 5 25 15 23 10 21  2  0  0  6 15 21 13 21 25 20  1 21 21 25 25  0 25  2\n",
      "  9 19 19 17  6 19 15  8 11 23  8 23 14  0  8 19  4 11  0 25 19  6 13 25\n",
      " 14 25 19 24 14  9 19  6 25 25  0 19 25 15 13 17 20  9 13  6 25  0 17  3\n",
      " 25  5 19 11 21 19 21 19 21 21 19  1 19  3 19 25 15 25 25 20 21 20 14  5\n",
      "  2 13 14 19]\n",
      "labels [21 21  0 13 10  3  8 24  0  6  7  4 13 25 12 20 13 21 12 16 16  0 17 12\n",
      "  2 19 18 19  6 17 15  8 11  3 14 23 14  4  8  4 22 11  0 24 22  6 13  1\n",
      " 22 25 23  4 14  9 22  6  7 12  0 23 10 15  5  5 20  9 13  6 12  1 19  3\n",
      " 16 20 16  1 21  1 17 19 17  1  8  7 19 12 19 20 15 21 17 17 21  3 17  2\n",
      "  2 13 16 16]\n",
      "train accuracy: 0.46\n",
      "masked [25 24 19  5 21 19 10  2  1 25 25 11 25  2 11 19 20 19  9 17 19  8 11 13\n",
      " 15 13 25  8 19  7  5  4  2 19 17 11 19 25  5  2 20 21 14  0 10 13 25 21\n",
      "  9 19 20 14  8 19 14  5 16  3  6 10 15 23  0  0 19 15 19 21  7 19 25 13\n",
      " 19 21  7  4 11 19 19 10 21 17  5  9 18 17  8 16  6  5  2  8 18 20  0 25\n",
      "  9 19  2 19]\n",
      "labels [10 21 19  5  1 18 10  2 22 24  4 11 17  2 11 23 21  8  9 18 17  8 20 13\n",
      "  1 13  7 18 20  7  5 12  2  7  4 11 16 10  5  2 18 12 14  0 10 13 23 23\n",
      "  1 23 17 14  8 19 14  5 17  3  6 10  3 21  0  6 19 23 25  4 22 12 24 13\n",
      "  4 16  4 21 11 19 16 10  4 24  5  9 18 16  8 16  6  3  2  8 18 25  0  1\n",
      " 13  5  2  7]\n",
      "train accuracy: 0.32\n",
      "masked [13 25 21 25  0 21  7 19 10 21  2  4 25 20  6  8 25 22 25  6 25 15  0 20\n",
      " 21 25 25 19 19 20  8 11 10 19 19 15 25  5 21 19 25 13 25 14 25  5 18 22\n",
      " 17  1  2 25 19  7  5 25 21  1 25 14 17 25 25 14 21 20 19  0 20  6  2 11\n",
      " 25 25 25 19 14 19 19 25 14 14 17 21 21 19 17 16 21  9 19 15 25 10 19  6\n",
      " 13 17 13 19]\n",
      "labels [13 23 12 23  0  7 12 12 16  1  2 25  4 16  6  8 23 10 16  6 21 15  0 19\n",
      "  4 17  4 20 19 21  8 11 10 24 18 15  1  5  4 16  7 16  6  5  3 25 18  8\n",
      "  5 23  2  3 16  0 24  3 13 19 22  8 17 25 16  3  1 24 14  0 20  3  3 21\n",
      " 21 23 22 19 14 17 20  5 14 20 17 21  4 19 11 25  1  9 10 15  1 10 20  6\n",
      " 13 11 13 18]\n",
      "train accuracy: 0.38\n",
      "masked [15  0 19 14 25  9 11 13 19  0  2 19 21  1 25  8 17  1 25 25 18 19 19 25\n",
      "  1 25 19  6 14  2 25  0 24 25 16 19 25 21  5 23 25  6  5  8 19  5  0 19\n",
      "  5 17 19 17 17 25  1 25 21  8 23  0  9 21 14  6 17 19 25 21  2  9  8  1\n",
      " 25 19 25 18 25 11 14 25 25 13  5 11  1  8 19 19  7  1  8 25 10 17  0  1\n",
      " 16 19  6 13]\n",
      "labels [15  0  5  5 13  2 11 13 17  0  5  7 20  7 17  8 11  0 23  7 18  5 19 21\n",
      " 22 23 19 21 14  2  5  0 16  7 14 14  1  4 13  9 19  3 14  8 12  5  0 23\n",
      "  5 24 23 12 21 12  1  4 11  8 14  0 18 25 22  6 19  5 22 10  2  9  8 21\n",
      " 22 19  7 18 17 11 14  4  4 13  2 11 10  8  3 12  7 12  8 12 10 17  0  7\n",
      " 16  7  6 13]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train accuracy: 0.45\n",
      "masked [19 19 19 16 19 15 18 11 25 19  3 16 19 25 11 25  9 25  7 11 17  9 25  5\n",
      "  6 18  8 11  2 25 21  0  6 10  4 15  8  3 13 17 20 15 25  4 11  7  7 19\n",
      "  1  8  9 15 25  0 15  5 20  5  7 11 25 17 17 19 20  8 18 25 25 19 16 14\n",
      " 18 25  2 20 10 19 11 19  9 18 23  0 18 19 11  0 25 20 25 11  8 11  0 17\n",
      " 25 17 11  5]\n",
      "labels [ 3 14 24 23 18 15 18 11  7  3 18 10 25 18 11 10  9 19 21 11  1  9  5  4\n",
      " 21 18  8 11  3  5  1 25  6 10 24 15  8  3 13 12 20 15 23  4 11 19 12 19\n",
      " 11  8  9 15 20  7 15 20 12  5 12 11 12  3  5 20 12  8 18 23 20  7 16 14\n",
      " 18  7  3 16  1 19 11 14  9 18 13  0 18 13 11  0 22 21 21 11  8 11  0 18\n",
      " 14 12 11  7]\n",
      "train accuracy: 0.42\n",
      "masked [ 2 14 19 19  6 25 24 25  6 11 20 24 23 19  5 25 25  8 10  9 15 15 20  2\n",
      " 10 15  0 19 19 19  0  0 25 19  9  8  2 11  5 13 14 19 19 25 25 20  2 18\n",
      "  6 14 19 23  2  7 10  0 19 11  6 19 18 16 19 19  0 10 25 19  0 15  0 19\n",
      " 25  6 11 14 20 10 18 10 25 19 11  0  9 22  0 19  8 18  8 21  9 11  5  6\n",
      "  6 15  8  8]\n",
      "labels [ 8 14 18 19  6 22 21 20  6 11 17  1  5 20 23 24 16  2 10  9 15 15 25 12\n",
      " 24 15  0 20  7 23  0 10 10 25 18  8  7  8  5  5 18 24 20  1 23 23  2 18\n",
      "  6 14  4 20  3 12 10  0 22 11  6 21 18 12  5 23  0  7  7 19  0 15  4  1\n",
      " 17  6 11  3 16 10 18 10 20 21 11  4  9 11 19 19  8 18 20  1  9 11 20  4\n",
      "  3 15  8  8]\n",
      "train accuracy: 0.33\n",
      "masked [ 9  3  3  0  8  9 16 10 19  5 16  5  3 14 21  1 21 19  5 17 13 19 21 14\n",
      " 20  9 25 18 18 21 21 19 24 15 16  8 11 16 25  9 25  6  7 18 17 16 15 19\n",
      "  1 14 19 25 25 19 25 19  6 19  5 21 25 25 25  5 11 25  0  1 14 14 19  8\n",
      " 10  5 18 24  5 21 18  0  5  0  3 12  8 17 13  5  5 19  8 18 17 25 15  0\n",
      " 17 11 11  0]\n",
      "labels [ 9 17  3  0  8  9 16 21 21 22 16 19  4  5 12  7  7  3  5  7  5 13  1 14\n",
      " 20 18 21 23 16  4  3 14  1 15 16  4 11  7 20  9 17  5 10 16 22 12 15 24\n",
      " 24 18 12 22 23 23 25 24  6 17  5 24 21  4 16 11 11 23  0 24 14  3 13  8\n",
      " 10 10  7 22  5  0 25  0  5  0 10  5  8 21  5  5  7 10 22  9 20  7 15  4\n",
      "  3 11 11  0]\n",
      "train accuracy: 0.4\n",
      "masked [14 17 21 21 14 25 19 19 19 14  8 21  3 19 10 25  2 21  9 25 21 18 14 19\n",
      " 11 10 21 14  5 11  1 19 13 17  9  0 17 19 15  2 23 21 25 21  2 19 25 19\n",
      "  1 16 25 24 21 10  8  0 16 20 17  4 22  0 21 13  2 21 15  2 21 13 13 21\n",
      " 13 19 19 19 25 14  7 18  7 25 15 15 21 20 14  0 16 21 25  8  6 21  9  7\n",
      "  6 20  8 19]\n",
      "labels [ 1 22  1  4  1 18 16 12  1  3  8 18 18 12 10 25  3 24  9 16 24 18  2  5\n",
      " 11 10  7 11 23 11  3 19 13 17  9  0 19 19 15  2  5 21 23 19  2 23 12 19\n",
      " 18 25 25 20  3 10  8  0 23 20 25 24 20  7 24 13  3 24 15  2 12 13 13 11\n",
      " 13 19  5 16 21 17 21 18 18 23  2 15 20 21 22  0  5 12 16  8  6 23  9 24\n",
      "  6 16  8 19]\n",
      "train accuracy: 0.41\n",
      "masked [ 6 19 11  6 19 14 25  2 14 13 25  2 19  7 19 11 10 11 10  8  7 14 14 10\n",
      "  3  8 11  5 10  1 20  6  8  3 25 25 15  6 11  5  9  6 20 18 20 13 25 15\n",
      " 11 13 19 19  4 21 20  1  6  7 11 20 16 25 15 21 19 21 19  0 20 21 15 19\n",
      " 23  2  3  6 19  7  7  6 16 19 25 14 19  7  3 19 16 21 21 25 10 19 20 21\n",
      " 19  8  4  9]\n",
      "labels [ 6  3 11  6  3 14 24  2 14  5 10 17 17 12 21 11 25  1  4 17  7 14 14 10\n",
      " 18  8 11 18  1  1 22  6  8  3 22 21 15  6 11 22 16  6 16 18  4 14 12 15\n",
      " 11 23 11 19 21  1 12  1  4 21 11 21  5 17 15  4 25 24  5  0 13 21 10 13\n",
      " 23  2 23  6 17  1 25  6 19 24 10 14 16  7  3 23 23  4 24 25  5 19 15  1\n",
      " 18  8 16  9]\n",
      "Epoch 2/2\n",
      "----------\n",
      "train accuracy: 0.4\n",
      "masked [25 14 16 11  1  5 25 20 20 11 20 20  0 25 14 25  2 25 19 18 15  6  4  2\n",
      "  3  0 20 25 18  8  9 10  6 11 25  9  4 25 25 25  5 20  8 11 14  9 19  6\n",
      "  0 25  6 20 19  0 20 25 14 20 10 20 25 11 19  9  4 23  5  8 20 16  1 15\n",
      " 14 17 11 14 11 25  0 20 25 25 20 20 25  9 11 25 23  3  3 20  2 25 25 13\n",
      " 19 14 20 13]\n",
      "labels [24  5 16 11 21  2  3 13  4  4 19  7  4 23 19 21  2 12  4 18 15  6  1  2\n",
      "  3  0  5 19 18  8 18 16  6 11 24  9 16 15 21 24 15  1  8 11 14  9 25  6\n",
      "  7 21  6  6 17  0 24  1 14 25 10  3  2 17  7  9 22  5  5  2 17 16  4 15\n",
      " 14 21 11 14 11 10  0 19 13 17 17 20 17  9 11 25  4  3 21 23 23 25 16 13\n",
      " 17 13  1 13]\n",
      "train accuracy: 0.34\n",
      "masked [14 21  2 19 25 19 11 11 17 25 25 17 19 21 15  2 10 25 25 18  9 17  5 14\n",
      "  3 19 11 10 19 20  1  6  5 25 10 18  8 25 14 15 25  5 25 14 20 16  1  5\n",
      " 19 21  2 20  9 11 19 21 20  5  4 20 25 14 20 18 20 20 11 25 15  4 10  5\n",
      " 15  0 21  6 20 21  5  2  4 20 19 13  6 21 11  6 17 15  8 20 21  6 20 20\n",
      " 20  4 25  0]\n",
      "labels [16  7  2 16 25 17 11 11 21  7  7 20 16 21 15  2  4 24 12 18  9 17 23 14\n",
      "  3  7 21  5 20  7  4 12  4 21 22 18  8 21 14 15  3 19 25  5 24 23 12 23\n",
      " 22  9  2 12  9 11  4  8  7 22 12  3 25 23 20 18  3 24 11 25 24  7 16 21\n",
      " 15  0 24  4  3  7  2  2 24 21 19 13  6 12 11  4 25 15 23  4  4  4 25 18\n",
      " 17  6 24  0]\n",
      "train accuracy: 0.36\n",
      "masked [16  6 11 20 13 10 21 15 14 21 15  8  5 19 23 11 21 25  9 19 15  0  9 13\n",
      "  1 25 21 16 19  9  8 19 10 21  3  6  2 13 25 21  2 25 17 18  6 20 19 21\n",
      " 15 19  6  4  0  5  9 25 20 19 21 15 15 19 25 21 11 14  7  6 20  8 21 16\n",
      " 21 19 18 10 15 10 21  0 21  2 25 25 19  5  6 15  6 20 22  9 13 11 19 10\n",
      "  2 17 19 20]\n",
      "labels [22  4 24  5 13 10  7  3  3 24 15  3 12 20 13 11 25 24  8 12 15  0  9 13\n",
      "  1 21 19 21 19  2  8  3 20 21  3  6 12  5  1  7  2 20 24 20  6  3 20  1\n",
      " 15  5 20  4  7 10 22  1  2 16 21 23 15 16 20 24 11 14 25  6 23  8 21  5\n",
      " 24  3 18 16 15 10 11  0  1  2 22 22 18 14 12 15 21 23 21  9 13 11  5 10\n",
      "  2 22  3  3]\n",
      "train accuracy: 0.49\n",
      "masked [ 8 11 20  2 13  9 20 19  0  6  0 19 17 19 20  9 15 11  5 14 25  5 19 20\n",
      "  9 19 25  6 14  0 25 23 19 21  6  1  0 11 25 13 11  0  6  6  8 25  6  9\n",
      " 15  6  5 11 25 14 25 25 19  1  8 23  9 15 20 25 20 14  8 19  1 25  2 13\n",
      " 25 17  5 10 11  9 25  9 11 19 10 25 15 14 19 21 20 14 25 14 21 25  2 25\n",
      " 18 25  3 25]\n",
      "labels [ 8 11 12 18 13  1 25 19  0  6  0 12  3 17  7  9 24 11 25 14 13 10 16  8\n",
      "  9 19 24  6 18  0  7 17 17 20  6  8  0 11 21 13 11  0  6  6  8 16  6  1\n",
      " 15  6 14 11 23 14 10 12  7 20  8 24  9 15 24  4 19 14  8  5 17  4  2 13\n",
      " 23 18  5 16 11  9 21  7 11 19 10 25 15 14  7 16 22 14 21 14  4 24  2  7\n",
      " 18 12  3 20]\n",
      "train accuracy: 0.56\n",
      "masked [18  1 21 24 10 25 25 16 19 11 21 20 19  1 25 11 23 20 16 19 20 18  6 10\n",
      " 15 19  8 11  5 21  6 16  7 21 10 17 19  6  5 16 14  2 13  8 20 14 25 13\n",
      " 15  9 17  8  9 25 24  6  0 19  0 25  9 17 19  0 19  5 19 16 20 21 17  8\n",
      " 23 20 19 25 10 13 19  9 11  8 25 25 13 20  9 24 19 13  0 17 15  2 16 19\n",
      "  9  6 14  5]\n",
      "labels [18  1 21  0 10  3 21 23 12 11 10 20 19 24 25 11  5 17 16  3  1 18  6 10\n",
      " 13 19  8 11 22  7  6 22 12 16 10 25 19  6 13 10 10  2 13  8 24 14 25 13\n",
      " 15  9 17  8  9 21 24  6  0 17  0 17  9 19 24  0 10 13 21 19  4 24 13  8\n",
      "  4 20 23 25 10 13 19  9 11  8 25  4 13 21  9 21 10 13  0 16  7  2 16 22\n",
      "  9  6 14 13]\n",
      "train accuracy: 0.41\n",
      "masked [11 15  2  5 14  2 19 21  7 20  7 23 17 25  9 11  7  6 21  6 21 14 17  7\n",
      " 10  0  2 25 25  6 20 23 19 25 24 20  8 21  1 19  0 14 25  6  5 20 19  8\n",
      "  0  8 25  2 13  5 19 25 19 25 23 23 20 21 10 23  6  5  0 14 20 20 21 19\n",
      " 21  8 18 20 21 20  7 19 11 20 23 18 20 25 14  9 19 11  5  0 20  2 20  9\n",
      " 15 10 25 25]\n",
      "labels [11 15  3 15 14  2 12  1  7  4 20 25 20  7  9 11 19 17  4  6  4 22 23 24\n",
      "  7  0  2 19  7  6 16 12 19 10  1  3  8 21  5 20  4 20 25  6 16 17 22  8\n",
      "  0  8 24  8 13 25  3 10  3  7 17 16 24  1 10 19  6 12  0 14 20 23  4 24\n",
      " 17  8 18 20 21 20  3 12 11 20 20 18  4 20 14 22 18 11 18  0 22  2 20  9\n",
      " 15 10  1  5]\n",
      "train accuracy: 0.45\n",
      "masked [ 2 19 10 25 25  6 20 17  9 18 13 25  1 17 19 17 14 14 18 19  6 17 20 10\n",
      "  0 23 20  8  6  6 25 25  7  2  6 11  7 25 11  2  2  9 13 25 19 16 25 25\n",
      " 19 15 14  1 25 25 15 10 25 19 25  9 25 21 25 13 17  9 25 21 11  5 25 18\n",
      " 11  6 18 15  0 18 14 11 25 20 20 25 16 20 19 25 14  6 18  8  6 10 20 20\n",
      " 25 10  8  5]\n",
      "labels [ 2 22 10  6  1 12  7 17  8  8 13  5  1 20  7 17 14 14 18 16  4 25 10  0\n",
      "  4 23 20  8  6  6 24 10  4  2 12 11 21 25 11  2  3  9  5 22 17 23 20  5\n",
      "  7 24 14 19 16 12 15 10 25 16 21  9 25 11 21 13 10  9 24 21 11 16 25 18\n",
      " 11  6  8 15  0 22 14 11 23  4 20  7 18 10 20  7 14  4 16  8  6 10 23 16\n",
      " 24 10  8 23]\n",
      "train accuracy: 0.43\n",
      "masked [13 19  4 14 21 21  3 21  8 11 10 21 10 14 21 21 19 18  8  8 21 20 10 18\n",
      " 10 19 14 21 20 21 25 17 21 21  9 19 25 13  0  6 18  0 21 15  2 19 15 23\n",
      " 25 13 14 24 19 19  3 17 21 18  2 14 13 16 15 10 19  2 14 11  0 14  4 14\n",
      " 15  0 17 17 15  8 18  8  0 21 13  4 14 19 21 19 10 21 10 21  0 17 21  0\n",
      " 18  7  5 21]\n",
      "labels [13 19 24 14 23 12  2 23  8 11 18 24 10 14 25  4 16 18  8  8 21 23 22 18\n",
      " 10 25 14 19 21 20 23 11 10  1 18 19 23  5 24  6 22  1  0 15  3 19 23  5\n",
      "  1 13 14 25 19 21  3 25 18 18  2  5  5 25 15 10 16  4 14 11  5  2 16 18\n",
      " 15  0 17 12 15  8 18  8  1 19  5  4 14 17  7 22 10  1 10 21  0  5 24  0\n",
      "  9 12 23 21]\n"
     ]
    }
   ],
   "source": [
    "from torchsummary import summary\n",
    "model = torchvision.models.resnet18(pretrained=True)\n",
    "modules = list(model.children())[:-4]\n",
    "model=nn.Sequential(*modules)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False\n",
    "    \n",
    "model.layer3 = nn.Sequential(ConvLayer(), PrimaryCaps(), DigitCaps())\n",
    "loss_train = []        \n",
    "accuracy_train = []\n",
    "\n",
    "model = model.to(device)\n",
    "optimizer = optim.Adam(model.layer3.parameters(),lr = 0.001)\n",
    "start = time.time()\n",
    "n_epochs = 3\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "        model.train() \n",
    "        \n",
    "        train_loss = 0\n",
    "        train_accuracy = 0\n",
    "        print('Epoch {}/{}'.format(epoch,3-1))\n",
    "        print('-'*10)\n",
    "        for batch_id, (inputs, labels) in enumerate(dataloaders['train']):\n",
    "            #inputs, labels = next(iter(dataloaders['train']))\n",
    "            if MARVEL_USE: labels = labels-1\n",
    "            labels =torch.eye(NUM_CLASSES).index_select(dim=0, index=labels)\n",
    "            inputs, labels = Variable(inputs), Variable(labels)\n",
    "            inputs = inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            outputs = model(inputs)\n",
    "            loss = model_loss(outputs, labels)\n",
    "            masked = decoder(outputs, inputs)\n",
    "        \n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_loss += loss.data[0]\n",
    "            train_accuracy += (sum(np.argmax(masked.data.cpu().numpy(), 1) == np.argmax(labels.data.cpu().numpy(), 1)) / float(batch_size))\n",
    "            \n",
    "            if batch_id % 100 == 0:\n",
    "                print(\"train accuracy:\", sum(np.argmax(masked.data.cpu().numpy(), 1) == \n",
    "                                       np.argmax(labels.data.cpu().numpy(), 1)) / float(batch_size))\n",
    "                if verbose: print(\"masked {}\".format(np.argmax(masked.data.cpu().numpy(), 1)))\n",
    "                if verbose: print(\"labels {}\".format(np.argmax(labels.data.cpu().numpy(), 1)))\n",
    "#                batch_accuracy.append(sum(np.argmax(preds.data.cpu().numpy(), 1) == \n",
    "#                                       np.argmax(labels.data.cpu().numpy(), 1)) / float(batch_size))\n",
    "        \n",
    "        loss_train.append(train_loss/len(dataloaders['train']))\n",
    "        accuracy_train.append(train_accuracy/len(dataloaders['train']))\n",
    "end = time.time()\n",
    "print(\"Training time execution {}\".format(end-start))\n",
    "print(\"Loss value for training phase: {}\".format(train_loss / len(dataloaders['train'])))\n",
    "print(\"Accuracy value for training phase: {}\".format(train_accuracy / len(dataloaders['train'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.eval()\n",
    "test_loss = 0\n",
    "test_accuracy = 0\n",
    "start = time.time()\n",
    "for batch_id, (inputs, labels) in enumerate(dataloaders['val']):\n",
    "    labels =torch.eye(NUM_CLASSES).index_select(dim=0, index=labels)\n",
    "    inputs, labels = Variable(inputs), Variable(labels)\n",
    "    inputs = inputs.to(device)\n",
    "    labels = labels.to(device)\n",
    "\n",
    "    outputs = model(inputs)\n",
    "    masked = decoder(outputs, inputs)\n",
    "    \n",
    "    loss = model_loss(outputs, labels)\n",
    "    test_loss += loss.data[0]\n",
    "    test_accuracy += (sum(np.argmax(masked.data.cpu().numpy(), 1) == np.argmax(labels.data.cpu().numpy(), 1)) / float(batch_size))\n",
    "    \n",
    "    if batch_id % 100 == 0:\n",
    "            print(\"test accuracy:\", sum(np.argmax(masked.data.cpu().numpy(), 1) == \n",
    "                                   np.argmax(labels.data.cpu().numpy(), 1)) / float(batch_size))\n",
    "            print(\"masked {}\".format(np.argmax(masked.data.cpu().numpy(), 1)))\n",
    "            print(\"labels {}\".format(np.argmax(labels.data.cpu().numpy(), 1)))\n",
    "        \n",
    "            \n",
    "end = time.time()   \n",
    "print(\"Validation time execution {}\".format(end-start))\n",
    "print(\"Loss value for test phase: {}\".format(test_loss / len(dataloaders['val'])))\n",
    "print(\"Accuracy value for test phase: {}\".format(test_accuracy / len(dataloaders['val'])))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(model)\n",
    "import matplotlib.pyplot as plt\n",
    "n_epochs = 3\n",
    "epochs = np.arange(1,n_epochs+1)\n",
    "plt.plot(epochs, loss_train, color='g')\n",
    "plt.plot(epochs, loss_train, color='orange')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.title('Training phase')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}