pipeline_spec-val.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__Python VERSION: 3.7.4 (default, Aug 13 2019, 20:35:49) \n",
      "[GCC 7.3.0]\n",
      "__pyTorch VERSION: 1.4.0\n",
      "__CUDA VERSION\n",
      "nvcc: NVIDIA (R) Cuda compiler driver\n",
      "Copyright (c) 2005-2017 NVIDIA Corporation\n",
      "Built on Fri_Nov__3_21:07:56_CDT_2017\n",
      "Cuda compilation tools, release 9.1, V9.1.85\n",
      "__CUDNN VERSION: 7603\n",
      "__Number CUDA Devices: 2\n",
      "__Devices\n",
      "Active CUDA Device: GPU 0\n",
      "Available devices  2\n",
      "Current cuda device  0\n",
      "3.7.4 (default, Aug 13 2019, 20:35:49) \n",
      "[GCC 7.3.0]\n",
      "24.0\n",
      "svmem(total=67477729280, available=32392196096, percent=52.0, used=31103012864, free=8581877760, active=32186486784, inactive=19827380224, buffers=267857920, cached=27524980736, shared=3240423424, slab=5197410304)\n",
      "memory GB: 0.2600975036621094\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import sys\n",
    "import torch\n",
    "from torch.utils.data.dataset import Dataset\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from sklearn import model_selection\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_auc_score, log_loss, roc_auc_score, roc_curve, auc\n",
    "from sklearn.model_selection import StratifiedKFold, ShuffleSplit, cross_val_score, train_test_split\n",
    "\n",
    "print('__Python VERSION:', sys.version)\n",
    "print('__pyTorch VERSION:', torch.__version__)\n",
    "print('__CUDA VERSION')\n",
    "from subprocess import call\n",
    "# call([\"nvcc\", \"--version\"]) does not work\n",
    "! nvcc --version\n",
    "print('__CUDNN VERSION:', torch.backends.cudnn.version())\n",
    "print('__Number CUDA Devices:', torch.cuda.device_count())\n",
    "print('__Devices')\n",
    "# call([\"nvidia-smi\", \"--format=csv\", \"--query-gpu=index,name,driver_version,memory.total,memory.used,memory.free\"])\n",
    "print('Active CUDA Device: GPU', torch.cuda.current_device())\n",
    "\n",
    "print ('Available devices ', torch.cuda.device_count())\n",
    "print ('Current cuda device ', torch.cuda.current_device())\n",
    "\n",
    "import numpy\n",
    "import numpy as np\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "Tensor = FloatTensor\n",
    "\n",
    "import pandas\n",
    "import pandas as pd\n",
    "\n",
    "import logging\n",
    "handler=logging.basicConfig(level=logging.INFO)\n",
    "lgr = logging.getLogger(__name__)\n",
    "%matplotlib inline\n",
    "\n",
    "# !pip install psutil\n",
    "import psutil\n",
    "import os\n",
    "def cpuStats():\n",
    "        print(sys.version)\n",
    "        print(psutil.cpu_percent())\n",
    "        print(psutil.virtual_memory())  # physical memory usage\n",
    "        pid = os.getpid()\n",
    "        py = psutil.Process(pid)\n",
    "        memoryUse = py.memory_info()[0] / 2. ** 30  # memory use in GB...I think\n",
    "        print('memory GB:', memoryUse)\n",
    "\n",
    "cpuStats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "# use_cuda = False\n",
    "\n",
    "FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor\n",
    "Tensor = FloatTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/raznem/projects_artur\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/raznem/projects/audio-pipelines\n"
     ]
    }
   ],
   "source": [
    "cd ../projects/audio-pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eval_set  test\n"
     ]
    }
   ],
   "source": [
    "!cd VoiceChallengeTestSet-8k/data_eval/sejm_voicelab_eval;ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-01-19 00:52:42--  http://naan.mimuw.edu.pl/~pzuk/targets\n",
      "Resolving naan.mimuw.edu.pl (naan.mimuw.edu.pl)... 193.0.109.30\n",
      "Connecting to naan.mimuw.edu.pl (naan.mimuw.edu.pl)|193.0.109.30|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 57998 (57K)\n",
      "Saving to: ‘targets.1’\n",
      "\n",
      "targets.1           100%[===================>]  56,64K  --.-KB/s    in 0,005s  \n",
      "\n",
      "2020-01-19 00:52:42 (11,3 MB/s) - ‘targets.1’ saved [57998/57998]\n",
      "\n",
      "hyps  score.py\ttargets  targets.1  text  utt2spk  wav.scp\n"
     ]
    }
   ],
   "source": [
    "# !cd VoiceChallengeTestSet-8k/data_eval/sejm_voicelab_eval/eval_set;wget http://naan.mimuw.edu.pl/~pzuk/targets;ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hyps  score.py\ttargets  text  utt2spk\twav.scp\n"
     ]
    }
   ],
   "source": [
    "!cd VoiceChallengeTestSet-8k/data_eval/sejm_voicelab_eval/eval_set;rm targets.1;ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = 'VoiceChallengeTestSet-8k/data_eval/sejm_voicelab_eval/eval_set/'\n",
    "\n",
    "hyps = pd.read_csv(base+\"hyps\", names=['audio_id'])\n",
    "hyps[['audio_id','hyps']] = hyps[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "targets = pd.read_csv(base+\"targets\", names=['audio_id'])\n",
    "targets[['audio_id','target']] = targets[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "text = pd.read_csv(base+\"text\", names=['audio_id'])\n",
    "text[['audio_id','text']] = text[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "speakers = pd.read_csv(base+\"utt2spk\", names=['audio_id'])\n",
    "speakers[['audio_id','speaker']] = speakers[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "images = pd.read_csv(base+\"wav.scp\", names=['audio_id'])\n",
    "images[['audio_id','audio_path']] = images[\"audio_id\"].str.split(\" \", 1, expand=True)\n",
    "\n",
    "dataset = pd.merge(hyps, images, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_absolute(string, prefix='/home/raznem/projects/audio-pipelines/data/'):\n",
    "    if string.startswith(prefix):\n",
    "        string = string[len(prefix):]\n",
    "    return string\n",
    "\n",
    "dataset = pd.merge(hyps, text, how=\"left\")\n",
    "dataset = pd.merge(dataset, speakers, how=\"left\")\n",
    "dataset = pd.merge(dataset, images, how=\"left\")\n",
    "dataset = pd.merge(dataset, targets, how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>audio_id</th>\n",
       "      <th>hyps</th>\n",
       "      <th>text</th>\n",
       "      <th>speaker</th>\n",
       "      <th>audio_path</th>\n",
       "      <th>target</th>\n",
       "      <th>image_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Adam_Andruszkiewicz_4_25_33-3</td>\n",
       "      <td>stronę dzień referendalny ustawy pel wydrukuje...</td>\n",
       "      <td>stronę dzień referendalny kropka p 50 wydrukuj...</td>\n",
       "      <td>Adam_Andruszkiewicz</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Andrusz...</td>\n",
       "      <td>1</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Andrusz...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Adam_Cyranski_2_2_56-6</td>\n",
       "      <td>czy nato propozycje klubu nowoczesna idą w dwó...</td>\n",
       "      <td>szczyt nato propozycje klubu nowoczesna idą w ...</td>\n",
       "      <td>Adam_Cyranski</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "      <td>0</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Adam_Cyranski_4_26_31-8</td>\n",
       "      <td>mechanizmy zostaną wprowadzone aby żołnierze m...</td>\n",
       "      <td>mechanizmy zostaną wprowadzone aby żołnierze m...</td>\n",
       "      <td>Adam_Cyranski</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "      <td>1</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Adam_Cyranski_4_42_16-1</td>\n",
       "      <td>szeregowych dzisiaj wprowadzamy ustawę która w...</td>\n",
       "      <td>szeregowych dzisiaj wprowadzamy ustawę która w...</td>\n",
       "      <td>Adam_Cyranski</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "      <td>1</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Adam_Cyranski_4_50_4-0</td>\n",
       "      <td>panie pośle proszę zadawać pytanie potem pan b...</td>\n",
       "      <td>panie pośle proszę zadawać pytanie potem pan b...</td>\n",
       "      <td>Adam_Cyranski</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "      <td>0</td>\n",
       "      <td>data_eval/sejm_voicelab_eval/test/Adam_Cyransk...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        audio_id  \\\n",
       "0  Adam_Andruszkiewicz_4_25_33-3   \n",
       "1         Adam_Cyranski_2_2_56-6   \n",
       "2        Adam_Cyranski_4_26_31-8   \n",
       "3        Adam_Cyranski_4_42_16-1   \n",
       "4         Adam_Cyranski_4_50_4-0   \n",
       "\n",
       "                                                hyps  \\\n",
       "0  stronę dzień referendalny ustawy pel wydrukuje...   \n",
       "1  czy nato propozycje klubu nowoczesna idą w dwó...   \n",
       "2  mechanizmy zostaną wprowadzone aby żołnierze m...   \n",
       "3  szeregowych dzisiaj wprowadzamy ustawę która w...   \n",
       "4  panie pośle proszę zadawać pytanie potem pan b...   \n",
       "\n",
       "                                                text              speaker  \\\n",
       "0  stronę dzień referendalny kropka p 50 wydrukuj...  Adam_Andruszkiewicz   \n",
       "1  szczyt nato propozycje klubu nowoczesna idą w ...        Adam_Cyranski   \n",
       "2  mechanizmy zostaną wprowadzone aby żołnierze m...        Adam_Cyranski   \n",
       "3  szeregowych dzisiaj wprowadzamy ustawę która w...        Adam_Cyranski   \n",
       "4  panie pośle proszę zadawać pytanie potem pan b...        Adam_Cyranski   \n",
       "\n",
       "                                          audio_path target  \\\n",
       "0  data_eval/sejm_voicelab_eval/test/Adam_Andrusz...      1   \n",
       "1  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...      0   \n",
       "2  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...      1   \n",
       "3  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...      1   \n",
       "4  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...      0   \n",
       "\n",
       "                                          image_path  \n",
       "0  data_eval/sejm_voicelab_eval/test/Adam_Andrusz...  \n",
       "1  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...  \n",
       "2  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...  \n",
       "3  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...  \n",
       "4  data_eval/sejm_voicelab_eval/test/Adam_Cyransk...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(400, 400)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from PIL import Image\n",
    "im = Image.open('data_eval/sejm_voicelab_eval/test/Adam_Cyranski_4_26_31-8.png')\n",
    "im.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset.to_csv('data/dataset_val.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data_eval/sejm_voicelab_eval/test/Adam_Andruszkiewicz_4_25_33-3.png'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['image_path'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.modeling:Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n",
      "INFO:pytorch_transformers.modeling_bert:Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n",
      "INFO:pytorch_transformers.modeling_xlnet:Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n",
      "INFO:allennlp.common.registrable:instantiating registered subclass relu of <class 'allennlp.nn.activations.Activation'>\n",
      "INFO:allennlp.common.registrable:instantiating registered subclass relu of <class 'allennlp.nn.activations.Activation'>\n",
      "INFO:allennlp.common.registrable:instantiating registered subclass relu of <class 'allennlp.nn.activations.Activation'>\n",
      "INFO:allennlp.common.registrable:instantiating registered subclass relu of <class 'allennlp.nn.activations.Activation'>\n"
     ]
    }
   ],
   "source": [
    "from allennlp.commands.elmo import ElmoEmbedder\n",
    "from torchvision import transforms\n",
    "\n",
    "try:\n",
    "    from PIL import Image\n",
    "except ImportError:\n",
    "    import Image\n",
    "    \n",
    "toImg = transforms.ToPILImage()\n",
    "toTensor = transforms.ToTensor()\n",
    "    \n",
    "class WavImagesLoader(Dataset):\n",
    "    def __init__(self, csv_path, transform=None):\n",
    "        self.dataset = pd.read_csv(csv_path)\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        image_path = self.dataset['image_path'][key]\n",
    "        image = Image.open(image_path)\n",
    "        image = image.convert('RGB')\n",
    "        image = toTensor(image)\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "        \n",
    "        return image\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "    \n",
    "    @staticmethod        \n",
    "    def imshow(img):\n",
    "        img = img / 2 + 0.5     # unnormalize\n",
    "        npimg = img.numpy()\n",
    "        plt.imshow(np.transpose(npimg, (1, 2, 0)))    \n",
    "\n",
    "#     @staticmethod    \n",
    "#     def flaotTensorToImage(img, mean=0, std=1):\n",
    "#         \"\"\"convert a tensor to an image\"\"\"\n",
    "#         img = np.transpose(img.numpy(), (1, 2, 0))\n",
    "#         img = (img*std+ mean)*255\n",
    "#         img = img.astype(np.uint8)    \n",
    "#         return img    \n",
    "    \n",
    "    \n",
    "class ElmoWavImagesLoader(WavImagesLoader):\n",
    "    def __init__(self, csv_path, elmo_model, transform=None):\n",
    "        super().__init__(csv_path, transform=None)\n",
    "        self.elmo_model = elmo_model\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        image_path = self.dataset['image_path'][key]\n",
    "        image = Image.open(path)\n",
    "        image = img.convert('RGB')\n",
    "        image = toTensor(image)\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(image)\n",
    "            \n",
    "        text = self.dataset['hyps'][key]\n",
    "        text = text.split(' ')\n",
    "        embedding = self.elmo_model(text)\n",
    "        embedding = embedding.sum(axis=0)\n",
    "\n",
    "        return image, embedding, target\n",
    "    \n",
    "    \n",
    "class ElmoWavVecLoader(ElmoWavImagesLoader):\n",
    "    def __init__(self, csv_path, elmo_model, transform=None):\n",
    "        super().__init__(csv_path, elmo_model, transform=None)\n",
    "        \n",
    "    def __getitem__(self, key):\n",
    "        target = self.dataset['target'][key]\n",
    "        image_vec = self.dataset['image_vec'][key]\n",
    "        text = self.dataset['hyps'][key]\n",
    "        text = text.split(' ')\n",
    "        embedding = self.elmo_model(text)\n",
    "        embedding = embedding.sum(axis=0)\n",
    "\n",
    "        return image_vec, embedding, target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conv net to get vector from Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout = torch.nn.Dropout(p=0.30)\n",
    "class ConvRes(nn.Module):\n",
    "    def __init__(self, insize, outsize):\n",
    "        super(ConvRes, self).__init__()\n",
    "        drate = .3\n",
    "        self.math = nn.Sequential(\n",
    "            nn.BatchNorm2d(insize),\n",
    "            # nn.Dropout(drate),\n",
    "            torch.nn.Conv2d(insize, outsize, kernel_size=2, padding=2),\n",
    "            nn.PReLU(),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.math(x)\n",
    "\n",
    "\n",
    "class ConvCNN(nn.Module):\n",
    "    def __init__(self, insize, outsize, kernel_size=7, padding=2, pool=2, avg=True):\n",
    "        super(ConvCNN, self).__init__()\n",
    "        self.avg = avg\n",
    "        self.math = torch.nn.Sequential(\n",
    "            torch.nn.Conv2d(insize, outsize, kernel_size=kernel_size, padding=padding),\n",
    "            torch.nn.BatchNorm2d(outsize),\n",
    "            torch.nn.LeakyReLU(),\n",
    "            torch.nn.MaxPool2d(pool, pool),\n",
    "        )\n",
    "        self.avgpool = torch.nn.AvgPool2d(pool, pool)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.math(x)\n",
    "        if self.avg is True:\n",
    "            x = self.avgpool(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, output_dim):\n",
    "        super(Net, self).__init__()\n",
    "\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d(1)\n",
    "\n",
    "        self.cnn1 = ConvCNN(3, 32, kernel_size=7, pool=4, avg=False)\n",
    "        self.cnn2 = ConvCNN(32, 32, kernel_size=5, pool=2, avg=True)\n",
    "        self.cnn3 = ConvCNN(32, 32, kernel_size=5, pool=2, avg=True)\n",
    "\n",
    "        self.res1 = ConvRes(32, 64)\n",
    "\n",
    "        self.features = nn.Sequential(\n",
    "            self.cnn1, dropout,\n",
    "            self.cnn2,\n",
    "            self.cnn3,\n",
    "            self.res1,\n",
    "        )\n",
    "\n",
    "        self.classifier = torch.nn.Sequential(\n",
    "            nn.Linear(5184, output_dim),\n",
    "        )\n",
    "        self.sig = nn.Sigmoid()\n",
    "  \n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        x = self.sig(x)\n",
    "        return x\n",
    "\n",
    "    \n",
    "# if use_cuda:\n",
    "#     lgr.info (\"Using the GPU\")\n",
    "#     model = Net(output_dim).cuda() # On GPU\n",
    "# else:\n",
    "#     lgr.info (\"Using the CPU\")\n",
    "#     model = Net(output_dim) # On CPU\n",
    "\n",
    "# lgr.info('Model {}'.format(model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gc\n",
    "def image_trainer(loader_train, model, optimizer, save_every=None, print_every=None, epochs=1, use_gpu=True, \n",
    "                 dtype=torch.float32):\n",
    "    if use_gpu and torch.cuda.is_available():\n",
    "        device = torch.device('cuda:1')\n",
    "    else:\n",
    "        device = torch.device('cpu')\n",
    "        \n",
    "    model.train()\n",
    "    model.to(device=device)\n",
    "\n",
    "#     for e in range(epochs):\n",
    "#         print('Epoch %d' %e)\n",
    "#         for t, (x, y) in enumerate(loader_train):\n",
    "#             x = x.to(device=device, dtype=dtype)\n",
    "#             y = y.to(device=device, dtype=dtype)\n",
    "\n",
    "#             y_pred = model(x)\n",
    "#             y_pred = torch.squeeze(y_pred)\n",
    "#             loss = loss_fn(y_pred, y)\n",
    "\n",
    "#             optimizer.zero_grad()\n",
    "#             loss.backward()\n",
    "\n",
    "#             optimizer.step()\n",
    "\n",
    "#             if print_every is not None and t % print_every == 0:\n",
    "#                 print('Iteration %d, loss = %.4f; epoch %d' % (t, loss.item(), e))\n",
    "                \n",
    "#         if save_every is not None and t % save_every == 0:\n",
    "#             torch.save(model.state_dict(), 'models/e%d_dupa_cnn.pt' % e)\n",
    "#             gc.collect()\n",
    "    torch.save(model.state_dict(), '/tmp/inference_draft.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = torch.nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "LEARNING_RATE = 1e-4\n",
    "USE_GPU = True\n",
    "TE_DTYPE = torch.float32\n",
    "TARGET_DIM = 1\n",
    "NUM_EPOCHS = 100\n",
    "\n",
    "csv_path = 'data/dataset_val.csv'\n",
    "model = Net(TARGET_DIM)\n",
    "image_data = WavImagesLoader(csv_path)\n",
    "dataloader = DataLoader(image_data, batch_size=BATCH_SIZE, shuffle=False, num_workers=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda:0\"\n",
    "model.load_state_dict(torch.load('models/image_cnn_100e_e70_image_cnn.pt'))\n",
    "model = model.eval().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_list = []\n",
    "with torch.no_grad():\n",
    "    for i, batch in enumerate(dataloader):\n",
    "        pred = model(batch.to(device)).to(\"cpu\")\n",
    "        prediction_list.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = torch.squeeze(torch.cat(prediction_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result = (result > 0.9).type(torch.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(csv_path)\n",
    "df.head()\n",
    "df[\"result\"] = result\n",
    "result = df[[\"audio_id\", \"hyps\", \"text\", \"target\", \"result\"]].copy()\n",
    "# # result.to_csv(\"submission_2239.txt\", header=False, sep=\" \", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def rval_acc(df):\n",
    "    df['acc'] = df['target'] == df['result']\n",
    "    acc = df['acc'].sum()/len(df['acc'])\n",
    "    return acc\n",
    "\n",
    "def find_thresh(df):\n",
    "    max_acc = 0.0\n",
    "    best = 0\n",
    "    for tr in np.arange(0.1,1,0.1):\n",
    "        df_0 = (df > tr).type(torch.int)\n",
    "        df_1 = pd.read_csv(csv_path)\n",
    "        df_1[\"result\"] = df_0\n",
    "        df_2 = df_1[[\"audio_id\", \"hyps\", \"text\", \"target\", \"result\"]].copy()\n",
    "        acc = rval_acc(df_2)\n",
    "        if acc>max_acc:\n",
    "            max_acc = acc\n",
    "            best = tr\n",
    "        print(f'Threshold: {tr}, acc: {acc}')\n",
    "    return(max_acc, best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +: 'int' and 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-ad009ee88bee>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'int' and 'str'"
     ]
    }
   ],
   "source": [
    "sum(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([[1.1295e-01],\n",
       "         [3.1939e-01],\n",
       "         [4.7646e-03],\n",
       "         [8.2300e-01],\n",
       "         [3.6138e-01],\n",
       "         [6.7834e-01],\n",
       "         [8.9241e-01],\n",
       "         [9.9536e-01],\n",
       "         [6.7618e-01],\n",
       "         [9.9773e-01],\n",
       "         [1.6123e-03],\n",
       "         [6.1922e-02],\n",
       "         [7.0179e-01],\n",
       "         [9.9996e-01],\n",
       "         [8.5671e-01],\n",
       "         [5.4975e-02],\n",
       "         [9.9960e-01],\n",
       "         [9.9009e-01],\n",
       "         [5.2179e-02],\n",
       "         [9.0659e-01],\n",
       "         [4.8412e-01],\n",
       "         [9.9521e-01],\n",
       "         [4.7270e-01],\n",
       "         [1.7834e-02],\n",
       "         [9.8809e-01],\n",
       "         [2.9266e-06],\n",
       "         [3.4489e-02],\n",
       "         [9.5738e-01],\n",
       "         [2.0221e-03],\n",
       "         [5.0348e-01],\n",
       "         [9.8034e-01],\n",
       "         [9.9976e-01],\n",
       "         [9.7026e-01],\n",
       "         [8.4788e-01],\n",
       "         [2.2412e-01],\n",
       "         [1.1598e-01],\n",
       "         [9.7839e-01],\n",
       "         [1.3701e-01],\n",
       "         [9.9257e-01],\n",
       "         [9.9957e-01],\n",
       "         [9.9253e-01],\n",
       "         [5.1313e-02],\n",
       "         [2.6000e-02],\n",
       "         [8.0212e-03],\n",
       "         [1.0224e-02],\n",
       "         [1.4106e-01],\n",
       "         [9.6926e-01],\n",
       "         [9.9928e-01],\n",
       "         [9.9974e-01],\n",
       "         [1.0688e-02],\n",
       "         [1.3873e-01],\n",
       "         [1.7288e-04],\n",
       "         [8.1184e-06],\n",
       "         [9.9891e-01],\n",
       "         [9.0922e-01],\n",
       "         [5.0865e-04],\n",
       "         [1.7785e-01],\n",
       "         [1.4152e-05],\n",
       "         [1.1311e-01],\n",
       "         [6.4180e-01],\n",
       "         [7.5510e-01],\n",
       "         [8.2324e-01],\n",
       "         [9.9940e-01],\n",
       "         [7.6918e-01],\n",
       "         [5.0152e-02],\n",
       "         [7.8358e-02],\n",
       "         [3.2180e-02],\n",
       "         [9.9866e-01],\n",
       "         [3.1348e-02],\n",
       "         [7.1295e-04],\n",
       "         [9.3360e-01],\n",
       "         [9.8739e-01],\n",
       "         [2.0079e-02],\n",
       "         [9.5176e-01],\n",
       "         [2.5136e-01],\n",
       "         [9.8879e-01],\n",
       "         [1.3907e-01],\n",
       "         [5.4635e-01],\n",
       "         [9.7910e-07],\n",
       "         [1.1070e-03],\n",
       "         [1.2949e-01],\n",
       "         [8.2666e-01],\n",
       "         [9.9162e-01],\n",
       "         [8.2129e-02],\n",
       "         [1.6905e-08],\n",
       "         [3.3664e-01],\n",
       "         [1.7506e-01],\n",
       "         [9.8268e-01],\n",
       "         [9.0610e-03],\n",
       "         [9.9479e-01],\n",
       "         [9.9065e-01],\n",
       "         [5.8808e-04],\n",
       "         [1.4416e-03],\n",
       "         [4.0452e-04],\n",
       "         [9.9096e-01],\n",
       "         [9.3457e-02],\n",
       "         [3.1784e-04],\n",
       "         [8.1014e-01],\n",
       "         [6.2497e-02],\n",
       "         [2.1261e-03],\n",
       "         [8.7750e-01],\n",
       "         [3.3837e-03],\n",
       "         [1.8666e-06],\n",
       "         [7.7115e-01],\n",
       "         [1.8815e-04],\n",
       "         [1.3035e-04],\n",
       "         [3.0550e-01],\n",
       "         [7.2440e-03],\n",
       "         [4.4278e-03],\n",
       "         [8.3827e-01],\n",
       "         [1.7148e-04],\n",
       "         [3.5964e-05],\n",
       "         [9.9039e-01],\n",
       "         [3.0632e-04],\n",
       "         [8.3340e-01],\n",
       "         [1.5691e-02],\n",
       "         [9.9952e-01],\n",
       "         [6.4697e-01],\n",
       "         [2.5130e-02],\n",
       "         [1.0963e-02],\n",
       "         [3.6394e-04],\n",
       "         [9.9937e-01],\n",
       "         [2.8983e-03],\n",
       "         [9.1914e-01],\n",
       "         [1.0347e-02],\n",
       "         [4.2113e-03],\n",
       "         [9.9990e-01],\n",
       "         [5.4157e-01]]), tensor([[7.3285e-02],\n",
       "         [9.0388e-01],\n",
       "         [1.6076e-05],\n",
       "         [9.9996e-01],\n",
       "         [5.5434e-01],\n",
       "         [1.2642e-04],\n",
       "         [6.5675e-02],\n",
       "         [1.7793e-02],\n",
       "         [5.8295e-03],\n",
       "         [6.1235e-03],\n",
       "         [6.7502e-03],\n",
       "         [3.2010e-01],\n",
       "         [1.2942e-01],\n",
       "         [7.2328e-04],\n",
       "         [3.1293e-03],\n",
       "         [7.5994e-03],\n",
       "         [9.9945e-01],\n",
       "         [1.1774e-03],\n",
       "         [3.4169e-01],\n",
       "         [2.3632e-02],\n",
       "         [2.2512e-03],\n",
       "         [4.4221e-01],\n",
       "         [3.3360e-01],\n",
       "         [1.3672e-02],\n",
       "         [4.9131e-05],\n",
       "         [9.2301e-05],\n",
       "         [1.3226e-01],\n",
       "         [9.9112e-01],\n",
       "         [1.3219e-03],\n",
       "         [3.1779e-03],\n",
       "         [3.2702e-03],\n",
       "         [1.8949e-01],\n",
       "         [5.2932e-04],\n",
       "         [4.3312e-02],\n",
       "         [3.8644e-01],\n",
       "         [9.5436e-01],\n",
       "         [1.2473e-01],\n",
       "         [9.8628e-01],\n",
       "         [2.8469e-01],\n",
       "         [2.3232e-02],\n",
       "         [1.8071e-03],\n",
       "         [5.0382e-04],\n",
       "         [9.9979e-01],\n",
       "         [7.0480e-04],\n",
       "         [9.5764e-01],\n",
       "         [1.1994e-01],\n",
       "         [9.9426e-02],\n",
       "         [6.8356e-01],\n",
       "         [1.5819e-01],\n",
       "         [9.7407e-01],\n",
       "         [8.3284e-01],\n",
       "         [9.9949e-01],\n",
       "         [1.2372e-01],\n",
       "         [4.3136e-06],\n",
       "         [1.1425e-01],\n",
       "         [5.1660e-03],\n",
       "         [5.5275e-05],\n",
       "         [6.3814e-01],\n",
       "         [9.9124e-01],\n",
       "         [3.5100e-01],\n",
       "         [9.9412e-01],\n",
       "         [5.4658e-02],\n",
       "         [1.0869e-02],\n",
       "         [9.9963e-01],\n",
       "         [8.7752e-01],\n",
       "         [2.8961e-01],\n",
       "         [9.9845e-01],\n",
       "         [7.4051e-01],\n",
       "         [9.9883e-01],\n",
       "         [1.1248e-02],\n",
       "         [2.1572e-01],\n",
       "         [9.1898e-01],\n",
       "         [2.5605e-01],\n",
       "         [8.1539e-02],\n",
       "         [1.7546e-02],\n",
       "         [9.9974e-01],\n",
       "         [8.5505e-01],\n",
       "         [2.2845e-03],\n",
       "         [9.5067e-01],\n",
       "         [9.0013e-03],\n",
       "         [3.3085e-01],\n",
       "         [9.8297e-01],\n",
       "         [1.5825e-02],\n",
       "         [1.7491e-02],\n",
       "         [9.4980e-01],\n",
       "         [2.9311e-02],\n",
       "         [9.8042e-01],\n",
       "         [2.1405e-01],\n",
       "         [8.3552e-02],\n",
       "         [6.0689e-03],\n",
       "         [9.2404e-03],\n",
       "         [8.8123e-01],\n",
       "         [1.8157e-02],\n",
       "         [9.2133e-01],\n",
       "         [9.2361e-01],\n",
       "         [3.1035e-02],\n",
       "         [9.7485e-01],\n",
       "         [4.3338e-02],\n",
       "         [8.7621e-01],\n",
       "         [2.2650e-02],\n",
       "         [2.9348e-03],\n",
       "         [9.2823e-01],\n",
       "         [1.0680e-01],\n",
       "         [2.5155e-02],\n",
       "         [4.7594e-01],\n",
       "         [2.0938e-02],\n",
       "         [1.4975e-01],\n",
       "         [3.1339e-01],\n",
       "         [4.3990e-01],\n",
       "         [1.6096e-02],\n",
       "         [5.3203e-02],\n",
       "         [9.3932e-01],\n",
       "         [9.9159e-01],\n",
       "         [9.3516e-01],\n",
       "         [4.0147e-02],\n",
       "         [9.7851e-01],\n",
       "         [3.7784e-01],\n",
       "         [6.1954e-05],\n",
       "         [3.5581e-03],\n",
       "         [5.7569e-03],\n",
       "         [3.5119e-01],\n",
       "         [3.0923e-01],\n",
       "         [9.5839e-03],\n",
       "         [3.7351e-03],\n",
       "         [9.7089e-01],\n",
       "         [1.1998e-01],\n",
       "         [7.5962e-02],\n",
       "         [7.3670e-03]]), tensor([[4.0364e-06],\n",
       "         [4.2165e-01],\n",
       "         [4.8840e-03],\n",
       "         [9.9801e-01],\n",
       "         [9.9489e-01],\n",
       "         [1.6214e-01],\n",
       "         [1.0419e-02],\n",
       "         [7.7110e-02],\n",
       "         [9.1010e-01],\n",
       "         [7.0764e-03],\n",
       "         [9.2283e-01],\n",
       "         [3.8537e-03],\n",
       "         [8.5239e-02],\n",
       "         [4.2042e-04],\n",
       "         [6.2525e-01],\n",
       "         [1.1444e-01],\n",
       "         [1.8345e-01],\n",
       "         [4.2363e-02],\n",
       "         [1.2359e-02],\n",
       "         [6.4412e-05],\n",
       "         [9.6929e-01],\n",
       "         [9.9595e-01],\n",
       "         [9.9619e-01],\n",
       "         [3.6471e-01],\n",
       "         [6.3678e-01],\n",
       "         [1.6392e-03],\n",
       "         [2.0683e-03],\n",
       "         [9.5550e-01],\n",
       "         [1.7101e-01],\n",
       "         [9.7132e-01],\n",
       "         [1.9563e-05],\n",
       "         [1.7798e-02],\n",
       "         [2.1492e-04],\n",
       "         [9.4955e-01],\n",
       "         [3.0661e-07],\n",
       "         [9.9960e-01],\n",
       "         [1.2302e-01],\n",
       "         [9.2182e-05],\n",
       "         [2.5105e-03],\n",
       "         [9.9716e-01],\n",
       "         [7.4090e-03],\n",
       "         [2.1587e-01],\n",
       "         [3.8795e-01],\n",
       "         [3.8520e-02],\n",
       "         [6.0683e-01],\n",
       "         [9.9680e-01],\n",
       "         [6.7019e-01],\n",
       "         [5.1228e-01],\n",
       "         [9.3645e-01],\n",
       "         [6.7035e-05],\n",
       "         [2.4793e-03],\n",
       "         [2.0373e-05],\n",
       "         [1.5453e-01],\n",
       "         [3.4931e-01],\n",
       "         [1.5368e-01],\n",
       "         [9.9958e-01],\n",
       "         [9.9994e-01],\n",
       "         [2.3957e-04],\n",
       "         [3.1129e-01],\n",
       "         [3.1243e-03],\n",
       "         [5.9358e-01],\n",
       "         [9.9853e-01],\n",
       "         [4.4674e-01],\n",
       "         [9.9450e-01],\n",
       "         [9.9801e-01],\n",
       "         [2.1386e-05],\n",
       "         [7.2014e-01],\n",
       "         [6.5823e-01],\n",
       "         [6.7988e-07],\n",
       "         [4.9441e-02],\n",
       "         [9.8182e-01],\n",
       "         [4.3564e-01],\n",
       "         [4.9593e-05],\n",
       "         [9.5883e-01],\n",
       "         [1.5129e-01],\n",
       "         [4.1699e-04],\n",
       "         [9.9620e-01],\n",
       "         [6.1562e-01],\n",
       "         [3.0517e-01],\n",
       "         [9.1590e-01],\n",
       "         [2.9813e-01],\n",
       "         [9.7884e-02],\n",
       "         [3.6321e-01],\n",
       "         [9.5679e-01],\n",
       "         [3.6667e-03],\n",
       "         [1.0000e+00],\n",
       "         [1.5586e-01],\n",
       "         [8.5550e-03],\n",
       "         [4.9380e-01],\n",
       "         [8.6913e-01],\n",
       "         [1.1921e-01],\n",
       "         [6.5359e-01],\n",
       "         [9.9223e-01],\n",
       "         [4.1995e-05],\n",
       "         [2.9758e-03],\n",
       "         [1.0669e-04],\n",
       "         [2.1105e-02],\n",
       "         [2.5973e-01],\n",
       "         [2.1775e-03],\n",
       "         [1.3545e-02],\n",
       "         [4.1346e-03],\n",
       "         [2.7240e-05],\n",
       "         [1.5261e-02],\n",
       "         [9.2759e-01],\n",
       "         [1.1987e-01],\n",
       "         [8.0351e-01],\n",
       "         [6.4050e-03],\n",
       "         [2.5044e-01],\n",
       "         [1.7473e-02],\n",
       "         [8.7337e-01],\n",
       "         [6.1239e-01],\n",
       "         [1.4718e-02],\n",
       "         [8.5230e-04],\n",
       "         [4.1819e-01],\n",
       "         [1.2942e-02],\n",
       "         [8.3506e-01],\n",
       "         [6.5879e-02],\n",
       "         [2.4211e-01],\n",
       "         [4.5341e-04],\n",
       "         [4.3143e-02],\n",
       "         [5.0456e-03],\n",
       "         [3.2061e-01],\n",
       "         [9.9984e-01],\n",
       "         [6.5055e-01],\n",
       "         [2.3598e-02],\n",
       "         [1.2283e-01],\n",
       "         [9.9304e-01],\n",
       "         [1.1796e-01]]), tensor([[9.8785e-01],\n",
       "         [9.9672e-01],\n",
       "         [1.7602e-01],\n",
       "         [9.9858e-01],\n",
       "         [9.8955e-01],\n",
       "         [9.6773e-01],\n",
       "         [5.4342e-02],\n",
       "         [9.5488e-01],\n",
       "         [3.2220e-03],\n",
       "         [9.2633e-01],\n",
       "         [4.2981e-01],\n",
       "         [4.1346e-06],\n",
       "         [3.8384e-01],\n",
       "         [3.5489e-03],\n",
       "         [2.8385e-02],\n",
       "         [2.7250e-03],\n",
       "         [5.6048e-01],\n",
       "         [4.0452e-03],\n",
       "         [4.0806e-01],\n",
       "         [3.0950e-02],\n",
       "         [5.3040e-05],\n",
       "         [2.2897e-04],\n",
       "         [7.1880e-06],\n",
       "         [5.4514e-01],\n",
       "         [2.9782e-03],\n",
       "         [9.6287e-01],\n",
       "         [8.2397e-02],\n",
       "         [7.8118e-01],\n",
       "         [1.7142e-02],\n",
       "         [2.0288e-02],\n",
       "         [2.3379e-02],\n",
       "         [9.7492e-01],\n",
       "         [9.3587e-01],\n",
       "         [9.6633e-01],\n",
       "         [7.9274e-01],\n",
       "         [5.6920e-01],\n",
       "         [9.2386e-01],\n",
       "         [9.9969e-01],\n",
       "         [9.9992e-01],\n",
       "         [5.5533e-04],\n",
       "         [9.7070e-01],\n",
       "         [1.1410e-04],\n",
       "         [9.9642e-01],\n",
       "         [6.8681e-01],\n",
       "         [9.9901e-01],\n",
       "         [8.9817e-01],\n",
       "         [1.6484e-01],\n",
       "         [9.8075e-04],\n",
       "         [9.0857e-01],\n",
       "         [9.9771e-01],\n",
       "         [1.2960e-01],\n",
       "         [7.2737e-01],\n",
       "         [2.2946e-01],\n",
       "         [9.9276e-01],\n",
       "         [2.0697e-02],\n",
       "         [9.1144e-01],\n",
       "         [1.1462e-03],\n",
       "         [1.3291e-02],\n",
       "         [4.7627e-01],\n",
       "         [2.3814e-02],\n",
       "         [9.1459e-01],\n",
       "         [2.7180e-03],\n",
       "         [1.8062e-01],\n",
       "         [1.6608e-05],\n",
       "         [5.1957e-01],\n",
       "         [2.3835e-02],\n",
       "         [4.6865e-05],\n",
       "         [3.7125e-04],\n",
       "         [9.9441e-01],\n",
       "         [3.9991e-06],\n",
       "         [5.8092e-07],\n",
       "         [3.9956e-01],\n",
       "         [1.6719e-02],\n",
       "         [2.2039e-02],\n",
       "         [1.9516e-03],\n",
       "         [4.9364e-01],\n",
       "         [1.7274e-04],\n",
       "         [9.9036e-01],\n",
       "         [3.2967e-01],\n",
       "         [9.6322e-01],\n",
       "         [9.9967e-01],\n",
       "         [2.7630e-02],\n",
       "         [3.5574e-01],\n",
       "         [4.2344e-03],\n",
       "         [3.9076e-02],\n",
       "         [7.8464e-04],\n",
       "         [7.6821e-02],\n",
       "         [2.0345e-01],\n",
       "         [9.2568e-02],\n",
       "         [9.9372e-01],\n",
       "         [8.5102e-02],\n",
       "         [5.9489e-01],\n",
       "         [1.6442e-01],\n",
       "         [7.8660e-04],\n",
       "         [9.3812e-01],\n",
       "         [3.0716e-01],\n",
       "         [5.0760e-01],\n",
       "         [1.6431e-01],\n",
       "         [9.3070e-01],\n",
       "         [1.2165e-02],\n",
       "         [7.9093e-01],\n",
       "         [8.4402e-01],\n",
       "         [8.9902e-05],\n",
       "         [9.9229e-01],\n",
       "         [7.8779e-01],\n",
       "         [8.8847e-01],\n",
       "         [9.9739e-01],\n",
       "         [1.3804e-01],\n",
       "         [9.9103e-01],\n",
       "         [4.2711e-02],\n",
       "         [6.3133e-01],\n",
       "         [3.8255e-05],\n",
       "         [2.2515e-01],\n",
       "         [5.5418e-04],\n",
       "         [1.2975e-04],\n",
       "         [3.9622e-02],\n",
       "         [9.9945e-01],\n",
       "         [8.6052e-02],\n",
       "         [8.8707e-03],\n",
       "         [9.8825e-01],\n",
       "         [8.1496e-03],\n",
       "         [3.2512e-01],\n",
       "         [8.7353e-01],\n",
       "         [5.2426e-02],\n",
       "         [3.4520e-02],\n",
       "         [9.8013e-01],\n",
       "         [4.1829e-02],\n",
       "         [4.6922e-02]]), tensor([[4.1272e-03],\n",
       "         [1.3458e-01],\n",
       "         [1.0718e-02],\n",
       "         [9.9980e-01],\n",
       "         [5.0060e-04],\n",
       "         [1.3353e-03],\n",
       "         [8.0119e-01],\n",
       "         [9.3802e-03],\n",
       "         [6.9705e-02],\n",
       "         [9.8989e-01],\n",
       "         [2.8119e-03],\n",
       "         [4.6724e-03],\n",
       "         [5.4131e-02],\n",
       "         [6.7777e-01],\n",
       "         [8.0535e-01],\n",
       "         [8.2687e-01],\n",
       "         [9.7969e-01],\n",
       "         [1.2466e-01],\n",
       "         [3.8112e-04],\n",
       "         [1.1762e-02],\n",
       "         [9.1427e-01],\n",
       "         [2.2230e-01],\n",
       "         [7.7870e-01],\n",
       "         [9.9419e-01],\n",
       "         [7.0783e-01],\n",
       "         [1.1375e-01],\n",
       "         [2.3165e-02],\n",
       "         [6.0894e-02],\n",
       "         [5.5800e-01],\n",
       "         [8.7777e-01],\n",
       "         [7.0615e-03],\n",
       "         [7.2394e-01],\n",
       "         [9.7525e-01],\n",
       "         [6.7279e-01],\n",
       "         [9.9735e-01],\n",
       "         [1.0328e-01],\n",
       "         [4.9529e-05],\n",
       "         [9.5291e-01],\n",
       "         [9.8651e-01],\n",
       "         [1.0332e-01],\n",
       "         [2.2669e-01],\n",
       "         [9.9695e-01],\n",
       "         [9.7656e-01],\n",
       "         [9.8685e-01],\n",
       "         [2.4724e-01],\n",
       "         [8.3016e-01],\n",
       "         [1.2276e-01],\n",
       "         [9.9834e-01],\n",
       "         [1.6271e-03],\n",
       "         [9.5749e-01],\n",
       "         [4.5828e-04],\n",
       "         [8.9773e-01],\n",
       "         [7.9731e-02],\n",
       "         [1.2294e-01],\n",
       "         [5.4739e-01],\n",
       "         [7.6935e-01],\n",
       "         [5.0459e-01],\n",
       "         [9.5313e-04],\n",
       "         [3.5315e-04],\n",
       "         [6.0183e-01],\n",
       "         [3.9198e-01],\n",
       "         [1.7069e-01],\n",
       "         [9.1440e-01],\n",
       "         [9.6781e-01],\n",
       "         [6.9380e-02],\n",
       "         [3.9048e-01],\n",
       "         [9.8239e-01],\n",
       "         [9.2061e-01],\n",
       "         [7.9590e-01],\n",
       "         [3.3530e-01],\n",
       "         [1.8054e-01],\n",
       "         [2.8915e-01],\n",
       "         [2.0953e-03],\n",
       "         [4.0429e-01],\n",
       "         [5.1868e-03],\n",
       "         [4.7581e-05],\n",
       "         [6.9674e-02],\n",
       "         [4.0923e-01],\n",
       "         [9.2745e-01],\n",
       "         [3.4548e-01],\n",
       "         [9.9999e-01],\n",
       "         [3.9196e-01],\n",
       "         [6.8288e-02],\n",
       "         [2.2463e-01],\n",
       "         [5.4473e-04],\n",
       "         [1.9573e-02],\n",
       "         [9.9315e-01],\n",
       "         [1.2847e-01],\n",
       "         [2.4367e-01],\n",
       "         [1.7200e-01],\n",
       "         [2.6735e-02],\n",
       "         [1.9912e-01],\n",
       "         [4.8883e-02],\n",
       "         [9.9882e-01],\n",
       "         [4.4223e-04],\n",
       "         [5.0834e-01],\n",
       "         [2.7066e-04],\n",
       "         [9.9833e-01],\n",
       "         [1.1853e-02],\n",
       "         [1.2319e-01],\n",
       "         [4.1630e-01],\n",
       "         [9.4872e-01],\n",
       "         [4.0997e-03],\n",
       "         [8.4043e-01],\n",
       "         [9.9910e-01],\n",
       "         [9.9929e-01],\n",
       "         [2.6098e-03],\n",
       "         [9.9757e-01],\n",
       "         [1.1000e-02],\n",
       "         [9.7542e-01],\n",
       "         [9.9215e-01],\n",
       "         [7.0521e-01],\n",
       "         [7.8964e-01],\n",
       "         [9.4388e-01],\n",
       "         [1.0357e-03],\n",
       "         [6.2663e-02],\n",
       "         [8.9449e-01],\n",
       "         [4.9932e-01],\n",
       "         [4.8009e-02],\n",
       "         [8.2439e-01],\n",
       "         [3.3708e-04],\n",
       "         [3.4856e-03],\n",
       "         [8.0539e-01],\n",
       "         [9.9914e-01],\n",
       "         [4.5095e-03],\n",
       "         [5.6341e-01],\n",
       "         [9.0626e-01],\n",
       "         [1.4726e-02]]), tensor([[6.0455e-01],\n",
       "         [4.8792e-01],\n",
       "         [9.6381e-01],\n",
       "         [9.6822e-01],\n",
       "         [9.1238e-01],\n",
       "         [6.2472e-01],\n",
       "         [9.9336e-01],\n",
       "         [9.9703e-01],\n",
       "         [9.6544e-01],\n",
       "         [1.5861e-01],\n",
       "         [5.0600e-02],\n",
       "         [9.9838e-01],\n",
       "         [3.8095e-02],\n",
       "         [2.9299e-01],\n",
       "         [9.0236e-04],\n",
       "         [9.8090e-01],\n",
       "         [9.7080e-01],\n",
       "         [3.8811e-04],\n",
       "         [7.4142e-01],\n",
       "         [9.5794e-01],\n",
       "         [9.9927e-01],\n",
       "         [9.8702e-01],\n",
       "         [9.7315e-01],\n",
       "         [1.3477e-06],\n",
       "         [1.2602e-01],\n",
       "         [5.2309e-04],\n",
       "         [1.0446e-02],\n",
       "         [6.8513e-01],\n",
       "         [1.8301e-02],\n",
       "         [9.6393e-01],\n",
       "         [3.3787e-02],\n",
       "         [8.4476e-01],\n",
       "         [2.3336e-02],\n",
       "         [9.8768e-01],\n",
       "         [2.8933e-02],\n",
       "         [9.7991e-01],\n",
       "         [3.9585e-01],\n",
       "         [6.6560e-02],\n",
       "         [7.9923e-01],\n",
       "         [1.2957e-01],\n",
       "         [1.1221e-04],\n",
       "         [2.8926e-04],\n",
       "         [6.4348e-03],\n",
       "         [4.1224e-02],\n",
       "         [9.9497e-03],\n",
       "         [9.4104e-01],\n",
       "         [9.2715e-01],\n",
       "         [9.7916e-01],\n",
       "         [9.9871e-01],\n",
       "         [2.1904e-05],\n",
       "         [3.3030e-03],\n",
       "         [9.8046e-01],\n",
       "         [9.7235e-01],\n",
       "         [6.6112e-03],\n",
       "         [1.7420e-02],\n",
       "         [9.8928e-01],\n",
       "         [2.6625e-03],\n",
       "         [4.1981e-01],\n",
       "         [3.1105e-04],\n",
       "         [3.3554e-01],\n",
       "         [9.7513e-01],\n",
       "         [3.0730e-04],\n",
       "         [3.8756e-01],\n",
       "         [8.3578e-03],\n",
       "         [7.8093e-02],\n",
       "         [6.5064e-01],\n",
       "         [5.6059e-02],\n",
       "         [2.7919e-02],\n",
       "         [1.3953e-02],\n",
       "         [9.8842e-01],\n",
       "         [7.5871e-01],\n",
       "         [8.6132e-01],\n",
       "         [9.9640e-01],\n",
       "         [1.4777e-02],\n",
       "         [1.7008e-02],\n",
       "         [9.9970e-01],\n",
       "         [1.2173e-02],\n",
       "         [7.7108e-01],\n",
       "         [8.8050e-02],\n",
       "         [9.8564e-01],\n",
       "         [6.9731e-02],\n",
       "         [8.8582e-02],\n",
       "         [3.4704e-02],\n",
       "         [5.2430e-01],\n",
       "         [2.7237e-01],\n",
       "         [5.4518e-02],\n",
       "         [2.0809e-02],\n",
       "         [3.4205e-01],\n",
       "         [1.2195e-03],\n",
       "         [4.4321e-03],\n",
       "         [1.0303e-03],\n",
       "         [4.0698e-02],\n",
       "         [2.7260e-02],\n",
       "         [8.4483e-01],\n",
       "         [9.9985e-01],\n",
       "         [7.5448e-02],\n",
       "         [2.3134e-02],\n",
       "         [7.2501e-03],\n",
       "         [9.7713e-01],\n",
       "         [7.0920e-02],\n",
       "         [6.7986e-01],\n",
       "         [1.4052e-02],\n",
       "         [7.0017e-01],\n",
       "         [9.5025e-01],\n",
       "         [9.6343e-01],\n",
       "         [1.5833e-01],\n",
       "         [1.6853e-01],\n",
       "         [4.1952e-04],\n",
       "         [7.9716e-01],\n",
       "         [9.8939e-01],\n",
       "         [2.4942e-02],\n",
       "         [9.9069e-01],\n",
       "         [7.9903e-01],\n",
       "         [1.3123e-03],\n",
       "         [9.9659e-01],\n",
       "         [2.4989e-01],\n",
       "         [1.0646e-02],\n",
       "         [2.1072e-01],\n",
       "         [9.0228e-01],\n",
       "         [1.4879e-01],\n",
       "         [1.1082e-02],\n",
       "         [8.2287e-01],\n",
       "         [9.7020e-01],\n",
       "         [2.1854e-02],\n",
       "         [7.9537e-01],\n",
       "         [2.4721e-01],\n",
       "         [9.9778e-01],\n",
       "         [2.7896e-03]]), tensor([[2.1506e-01],\n",
       "         [9.8735e-01],\n",
       "         [5.7911e-02],\n",
       "         [4.5437e-02],\n",
       "         [9.3899e-01],\n",
       "         [1.4195e-01],\n",
       "         [2.1293e-02],\n",
       "         [3.8187e-03],\n",
       "         [3.3744e-05],\n",
       "         [1.9742e-02],\n",
       "         [2.8009e-01],\n",
       "         [9.9483e-01],\n",
       "         [4.5553e-06],\n",
       "         [8.3672e-02],\n",
       "         [6.2360e-01],\n",
       "         [9.2376e-01],\n",
       "         [1.9284e-02],\n",
       "         [9.9543e-01],\n",
       "         [3.7222e-02],\n",
       "         [9.1993e-02],\n",
       "         [9.2864e-01],\n",
       "         [9.9974e-01],\n",
       "         [1.4648e-01],\n",
       "         [9.6991e-01],\n",
       "         [9.3257e-01],\n",
       "         [2.7609e-01],\n",
       "         [9.7057e-01],\n",
       "         [9.8950e-01],\n",
       "         [9.2889e-01],\n",
       "         [3.8894e-01],\n",
       "         [9.8040e-01],\n",
       "         [1.0462e-05],\n",
       "         [5.0793e-03],\n",
       "         [8.7756e-01],\n",
       "         [1.1548e-01],\n",
       "         [7.9614e-06],\n",
       "         [9.9795e-01],\n",
       "         [2.1715e-01],\n",
       "         [7.0229e-01],\n",
       "         [8.6208e-01],\n",
       "         [4.0460e-04],\n",
       "         [5.6073e-01],\n",
       "         [9.9979e-01],\n",
       "         [8.3533e-07],\n",
       "         [2.4779e-06],\n",
       "         [5.6495e-01],\n",
       "         [8.4823e-01],\n",
       "         [2.5163e-01],\n",
       "         [1.6556e-02],\n",
       "         [2.0630e-03],\n",
       "         [8.6688e-01],\n",
       "         [6.5350e-01],\n",
       "         [8.7400e-03],\n",
       "         [6.5135e-01],\n",
       "         [1.8386e-03],\n",
       "         [9.6959e-01],\n",
       "         [9.3915e-01],\n",
       "         [2.2515e-01],\n",
       "         [1.7113e-03],\n",
       "         [3.0687e-02],\n",
       "         [7.0217e-01],\n",
       "         [9.1868e-01],\n",
       "         [1.2357e-01],\n",
       "         [6.8029e-01],\n",
       "         [6.9126e-01],\n",
       "         [2.6929e-02],\n",
       "         [3.3602e-01],\n",
       "         [9.9443e-01],\n",
       "         [9.9320e-01],\n",
       "         [1.9278e-02],\n",
       "         [1.4012e-06],\n",
       "         [4.7815e-02],\n",
       "         [9.9518e-01],\n",
       "         [9.9399e-01],\n",
       "         [2.6051e-03],\n",
       "         [7.0035e-01],\n",
       "         [9.8625e-01],\n",
       "         [1.1866e-01],\n",
       "         [1.5474e-02],\n",
       "         [5.8668e-01],\n",
       "         [2.7279e-04],\n",
       "         [1.3020e-02],\n",
       "         [9.7277e-01],\n",
       "         [2.2404e-01],\n",
       "         [1.1338e-01],\n",
       "         [1.1489e-01],\n",
       "         [5.4199e-03],\n",
       "         [5.6517e-03],\n",
       "         [4.3458e-01],\n",
       "         [9.9859e-01],\n",
       "         [9.6826e-01],\n",
       "         [2.4729e-02],\n",
       "         [5.4110e-04],\n",
       "         [3.3542e-02],\n",
       "         [2.4553e-01],\n",
       "         [2.3525e-02],\n",
       "         [9.9984e-01],\n",
       "         [9.0517e-01],\n",
       "         [9.8620e-01],\n",
       "         [7.6884e-02],\n",
       "         [1.9101e-02],\n",
       "         [7.6755e-02],\n",
       "         [2.1330e-03],\n",
       "         [1.3836e-02],\n",
       "         [9.6385e-01],\n",
       "         [2.0448e-01],\n",
       "         [3.7197e-05],\n",
       "         [2.1310e-02],\n",
       "         [1.0654e-01],\n",
       "         [3.0409e-03],\n",
       "         [9.8689e-01],\n",
       "         [3.2754e-02],\n",
       "         [9.8082e-01],\n",
       "         [3.3021e-01],\n",
       "         [3.9229e-02],\n",
       "         [1.2536e-03],\n",
       "         [9.9004e-01],\n",
       "         [8.3587e-01],\n",
       "         [3.1812e-01],\n",
       "         [2.9287e-01],\n",
       "         [6.6535e-01],\n",
       "         [9.9985e-01],\n",
       "         [9.9812e-01],\n",
       "         [9.3740e-01],\n",
       "         [2.9556e-01],\n",
       "         [3.3998e-01],\n",
       "         [1.8928e-02],\n",
       "         [1.1265e-04]]), tensor([[4.0885e-04],\n",
       "         [8.7058e-04],\n",
       "         [9.6120e-01],\n",
       "         [6.6576e-01],\n",
       "         [8.8940e-01],\n",
       "         [9.1769e-01],\n",
       "         [1.9356e-01],\n",
       "         [1.9409e-01],\n",
       "         [8.4276e-04],\n",
       "         [1.1978e-01],\n",
       "         [8.7799e-01],\n",
       "         [8.2739e-01],\n",
       "         [9.9091e-01],\n",
       "         [2.9417e-02],\n",
       "         [8.2365e-01],\n",
       "         [9.9812e-01],\n",
       "         [9.9531e-01],\n",
       "         [2.3352e-04],\n",
       "         [7.4504e-02],\n",
       "         [2.3211e-03],\n",
       "         [9.2135e-01],\n",
       "         [1.0252e-02],\n",
       "         [9.2414e-01],\n",
       "         [4.9486e-01],\n",
       "         [9.2562e-01],\n",
       "         [9.9989e-01],\n",
       "         [6.9139e-03],\n",
       "         [2.8828e-02],\n",
       "         [1.3008e-01],\n",
       "         [9.9970e-01],\n",
       "         [1.7955e-02],\n",
       "         [1.1366e-02],\n",
       "         [9.9988e-01],\n",
       "         [9.8347e-01],\n",
       "         [1.0000e+00],\n",
       "         [5.8447e-01],\n",
       "         [1.6691e-02],\n",
       "         [4.8763e-01],\n",
       "         [8.3329e-03],\n",
       "         [2.0913e-01],\n",
       "         [2.0088e-01],\n",
       "         [7.1529e-01],\n",
       "         [5.0474e-01],\n",
       "         [3.0153e-06],\n",
       "         [1.7882e-04],\n",
       "         [3.6468e-03],\n",
       "         [1.1238e-02],\n",
       "         [2.2405e-06],\n",
       "         [1.3726e-02],\n",
       "         [1.4122e-01],\n",
       "         [2.5710e-02],\n",
       "         [1.3804e-02],\n",
       "         [8.2319e-02],\n",
       "         [6.3020e-02],\n",
       "         [6.4917e-01],\n",
       "         [5.4969e-02],\n",
       "         [9.8005e-01],\n",
       "         [1.8288e-01],\n",
       "         [1.0532e-01],\n",
       "         [3.0178e-01],\n",
       "         [3.3967e-05],\n",
       "         [9.9989e-01],\n",
       "         [4.5347e-01],\n",
       "         [9.9782e-01],\n",
       "         [1.4821e-03],\n",
       "         [4.4971e-01],\n",
       "         [9.3186e-02],\n",
       "         [1.4879e-03],\n",
       "         [4.3238e-03],\n",
       "         [1.0963e-01],\n",
       "         [9.4263e-01],\n",
       "         [9.4685e-01],\n",
       "         [9.9831e-01],\n",
       "         [4.2826e-01],\n",
       "         [2.7969e-02],\n",
       "         [1.0310e-04],\n",
       "         [9.9757e-01],\n",
       "         [9.7877e-01],\n",
       "         [4.1889e-04],\n",
       "         [3.5024e-02],\n",
       "         [9.9142e-01],\n",
       "         [8.1540e-01],\n",
       "         [9.8798e-01],\n",
       "         [2.1393e-01],\n",
       "         [7.5173e-01],\n",
       "         [1.1627e-01],\n",
       "         [7.9113e-01],\n",
       "         [3.7747e-02],\n",
       "         [9.5967e-01],\n",
       "         [3.6406e-03],\n",
       "         [9.9020e-01],\n",
       "         [6.9649e-02],\n",
       "         [8.2097e-01],\n",
       "         [9.3400e-01],\n",
       "         [5.1252e-04],\n",
       "         [9.4827e-01],\n",
       "         [9.1488e-01],\n",
       "         [9.9898e-01],\n",
       "         [8.4278e-01],\n",
       "         [8.1161e-01],\n",
       "         [5.9310e-01],\n",
       "         [7.5202e-01],\n",
       "         [1.8121e-01],\n",
       "         [4.1512e-02],\n",
       "         [1.1086e-03],\n",
       "         [1.1608e-01],\n",
       "         [8.0559e-02],\n",
       "         [4.4438e-01],\n",
       "         [7.0220e-01],\n",
       "         [1.8604e-02],\n",
       "         [4.0373e-04],\n",
       "         [2.5200e-03],\n",
       "         [1.0000e+00],\n",
       "         [1.2612e-02],\n",
       "         [7.9268e-01],\n",
       "         [1.3851e-03],\n",
       "         [1.2334e-02],\n",
       "         [9.7853e-01],\n",
       "         [1.0000e+00],\n",
       "         [9.9121e-01],\n",
       "         [6.1594e-01],\n",
       "         [9.9873e-01],\n",
       "         [9.5956e-01],\n",
       "         [7.2987e-01],\n",
       "         [9.6576e-01],\n",
       "         [5.0918e-01],\n",
       "         [7.8498e-01],\n",
       "         [7.4198e-01]]), tensor([[1.1127e-01],\n",
       "         [4.6025e-01],\n",
       "         [9.9586e-01],\n",
       "         [6.9123e-01],\n",
       "         [6.5828e-01],\n",
       "         [8.7243e-02],\n",
       "         [3.5818e-01],\n",
       "         [1.2600e-03],\n",
       "         [9.9471e-01],\n",
       "         [8.8297e-01],\n",
       "         [9.8566e-01],\n",
       "         [3.4780e-04],\n",
       "         [9.4924e-01],\n",
       "         [6.6614e-01],\n",
       "         [1.1706e-02],\n",
       "         [8.5644e-02],\n",
       "         [2.3626e-02],\n",
       "         [8.0432e-01],\n",
       "         [1.5471e-03],\n",
       "         [8.3894e-01],\n",
       "         [9.8315e-01],\n",
       "         [4.3791e-02],\n",
       "         [1.8088e-02],\n",
       "         [6.6149e-01],\n",
       "         [6.8755e-05],\n",
       "         [2.9300e-01],\n",
       "         [1.1069e-02],\n",
       "         [7.8630e-01],\n",
       "         [9.4981e-01],\n",
       "         [9.3572e-02],\n",
       "         [5.2775e-02],\n",
       "         [1.4895e-01],\n",
       "         [7.5545e-01],\n",
       "         [9.2060e-01],\n",
       "         [1.0000e+00],\n",
       "         [2.0694e-03],\n",
       "         [9.9031e-01],\n",
       "         [1.0000e+00],\n",
       "         [4.6220e-04],\n",
       "         [1.6625e-06],\n",
       "         [9.6164e-01],\n",
       "         [1.0000e+00],\n",
       "         [9.9231e-01],\n",
       "         [7.4700e-05],\n",
       "         [2.7364e-01],\n",
       "         [8.1190e-02],\n",
       "         [8.7469e-01],\n",
       "         [1.2993e-01],\n",
       "         [7.9545e-04],\n",
       "         [9.2834e-01],\n",
       "         [9.9993e-01],\n",
       "         [1.8181e-01],\n",
       "         [9.9060e-01],\n",
       "         [9.9965e-01],\n",
       "         [2.8776e-04],\n",
       "         [7.7304e-03],\n",
       "         [5.0899e-01],\n",
       "         [3.1220e-01],\n",
       "         [4.5274e-02],\n",
       "         [9.2205e-01],\n",
       "         [1.9479e-02],\n",
       "         [7.8601e-04],\n",
       "         [2.3396e-01],\n",
       "         [1.2366e-02],\n",
       "         [1.4060e-06],\n",
       "         [2.1460e-03],\n",
       "         [7.9528e-03],\n",
       "         [7.7880e-03],\n",
       "         [1.1422e-04],\n",
       "         [9.4258e-01],\n",
       "         [9.0520e-01],\n",
       "         [2.9295e-06],\n",
       "         [5.0035e-01],\n",
       "         [5.2568e-01],\n",
       "         [3.2455e-01],\n",
       "         [9.8435e-01],\n",
       "         [5.4644e-03],\n",
       "         [3.3121e-02],\n",
       "         [9.1489e-01],\n",
       "         [1.3881e-03],\n",
       "         [2.7894e-01],\n",
       "         [2.7165e-01],\n",
       "         [9.4451e-01],\n",
       "         [1.0649e-02],\n",
       "         [4.1551e-01],\n",
       "         [9.6881e-01],\n",
       "         [7.3406e-01],\n",
       "         [9.9147e-01],\n",
       "         [9.0268e-01],\n",
       "         [8.6848e-01],\n",
       "         [2.7416e-04],\n",
       "         [9.2655e-02],\n",
       "         [9.8890e-01],\n",
       "         [7.5070e-01],\n",
       "         [5.5586e-01],\n",
       "         [8.8064e-04],\n",
       "         [5.5077e-02],\n",
       "         [1.8127e-01],\n",
       "         [9.8665e-01],\n",
       "         [5.9667e-01],\n",
       "         [1.9860e-01],\n",
       "         [1.4647e-01],\n",
       "         [9.3654e-02],\n",
       "         [4.7434e-01],\n",
       "         [1.2664e-01],\n",
       "         [8.2305e-01],\n",
       "         [4.1607e-02],\n",
       "         [6.0895e-02],\n",
       "         [8.9979e-01],\n",
       "         [9.2842e-01],\n",
       "         [9.9474e-01],\n",
       "         [9.9994e-01],\n",
       "         [9.3626e-01],\n",
       "         [9.9999e-01],\n",
       "         [6.8189e-01],\n",
       "         [2.9186e-02],\n",
       "         [7.0742e-01],\n",
       "         [1.9931e-01],\n",
       "         [9.4246e-01],\n",
       "         [9.8526e-01],\n",
       "         [1.4582e-03],\n",
       "         [8.3479e-04],\n",
       "         [2.4715e-03],\n",
       "         [6.4974e-01],\n",
       "         [5.8592e-01],\n",
       "         [8.0259e-01],\n",
       "         [8.2094e-01],\n",
       "         [4.0169e-04]]), tensor([[1.1562e-02],\n",
       "         [7.0359e-02],\n",
       "         [9.0307e-01],\n",
       "         [1.5874e-01],\n",
       "         [4.4749e-01],\n",
       "         [3.0797e-01],\n",
       "         [9.8009e-01],\n",
       "         [8.4053e-01],\n",
       "         [1.6425e-03],\n",
       "         [6.0414e-01],\n",
       "         [9.9384e-02],\n",
       "         [2.2279e-03],\n",
       "         [4.4288e-02],\n",
       "         [5.0639e-01],\n",
       "         [4.0610e-01],\n",
       "         [9.8402e-01],\n",
       "         [9.9266e-01],\n",
       "         [1.7243e-03],\n",
       "         [1.1776e-02],\n",
       "         [2.8832e-01],\n",
       "         [1.4209e-01],\n",
       "         [9.7496e-01],\n",
       "         [4.7930e-02],\n",
       "         [2.3079e-02],\n",
       "         [7.5806e-05],\n",
       "         [1.3896e-04],\n",
       "         [3.1903e-01],\n",
       "         [1.1796e-02],\n",
       "         [9.0710e-01],\n",
       "         [7.9546e-01],\n",
       "         [2.0388e-03],\n",
       "         [2.2575e-04],\n",
       "         [6.2241e-01],\n",
       "         [7.0451e-03],\n",
       "         [4.3863e-04],\n",
       "         [2.2684e-02],\n",
       "         [9.9110e-01],\n",
       "         [6.2314e-01],\n",
       "         [2.9367e-01],\n",
       "         [6.8464e-03],\n",
       "         [9.4288e-01],\n",
       "         [8.6783e-02],\n",
       "         [9.9293e-01],\n",
       "         [2.1880e-01],\n",
       "         [9.8886e-01],\n",
       "         [2.8629e-01],\n",
       "         [9.9921e-01],\n",
       "         [5.2743e-01],\n",
       "         [9.5294e-01],\n",
       "         [2.0741e-02],\n",
       "         [8.2911e-01],\n",
       "         [9.8419e-01],\n",
       "         [1.0000e+00],\n",
       "         [9.9057e-01],\n",
       "         [1.7954e-03],\n",
       "         [2.3843e-02],\n",
       "         [9.9993e-01],\n",
       "         [2.9904e-02],\n",
       "         [9.9902e-01],\n",
       "         [9.9884e-01],\n",
       "         [9.2623e-02],\n",
       "         [3.9574e-01],\n",
       "         [9.9863e-01],\n",
       "         [9.8207e-01],\n",
       "         [2.7025e-04],\n",
       "         [2.9250e-02],\n",
       "         [9.3043e-01],\n",
       "         [1.9236e-03],\n",
       "         [3.1578e-02],\n",
       "         [7.4326e-01],\n",
       "         [4.2547e-01],\n",
       "         [1.1376e-01],\n",
       "         [9.9614e-01],\n",
       "         [9.9921e-01],\n",
       "         [2.1312e-02],\n",
       "         [3.9866e-01],\n",
       "         [9.9997e-01],\n",
       "         [9.9999e-01],\n",
       "         [5.0480e-04],\n",
       "         [7.2934e-03],\n",
       "         [9.5678e-01],\n",
       "         [9.9991e-01],\n",
       "         [5.9613e-02],\n",
       "         [1.7531e-01],\n",
       "         [1.7001e-02],\n",
       "         [9.9502e-01],\n",
       "         [9.7801e-01],\n",
       "         [1.9539e-02],\n",
       "         [9.9940e-01],\n",
       "         [4.3964e-01],\n",
       "         [9.3610e-01],\n",
       "         [1.5452e-02],\n",
       "         [7.6923e-01],\n",
       "         [7.5510e-01],\n",
       "         [3.3967e-02],\n",
       "         [3.7818e-01],\n",
       "         [5.5703e-01],\n",
       "         [9.6996e-01],\n",
       "         [2.0490e-02],\n",
       "         [8.2835e-01],\n",
       "         [1.1632e-02],\n",
       "         [4.7945e-01],\n",
       "         [9.9497e-01],\n",
       "         [5.6663e-01],\n",
       "         [2.6638e-01],\n",
       "         [9.9116e-01],\n",
       "         [9.9358e-01],\n",
       "         [9.8476e-01],\n",
       "         [4.9432e-02],\n",
       "         [8.8874e-04],\n",
       "         [9.9991e-01],\n",
       "         [1.3947e-01],\n",
       "         [4.9135e-04],\n",
       "         [5.2635e-01],\n",
       "         [7.8936e-01],\n",
       "         [1.4413e-03],\n",
       "         [6.7142e-02],\n",
       "         [9.9200e-01],\n",
       "         [5.7281e-03],\n",
       "         [4.7991e-01],\n",
       "         [8.3547e-02],\n",
       "         [9.5166e-01],\n",
       "         [9.5605e-01],\n",
       "         [7.6156e-01],\n",
       "         [1.8487e-02],\n",
       "         [2.0716e-01],\n",
       "         [2.5904e-03],\n",
       "         [1.2776e-02]])]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_list[:10]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}