datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: Tesla K20Xm (CNMeM is disabled, cuDNN not available)\n"
     ]
    }
   ],
   "source": [
    "import datasets\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'datasets' from 'datasets.pyc'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found:  /home/jmj418/vae_ssl/datasets/mnist/proc-mnist.h5\n"
     ]
    }
   ],
   "source": [
    "data = datasets.SemiSupervisedMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SemiSupervisedMNIST nclasses=10, dim_observations=784: NestD{\n",
       "   test: SemiSupervisedDataEvaluate: NestD{\n",
       "     U: NestD{\n",
       "       Y: (10000,)\n",
       "       X: (10000, 784)}\n",
       "     L: NestD{\n",
       "       Y: (10000,)\n",
       "       X: (10000, 784)}}\n",
       "   train: SemiSupervisedDataTrain: NestD{\n",
       "     U: NestD{\n",
       "       Y: (50000,)\n",
       "       X: (50000, 784)}\n",
       "     L: NestD{\n",
       "       Y: (100,)\n",
       "       X: (100, 784)}}\n",
       "   valid: SemiSupervisedDataEvaluate: NestD{\n",
       "     U: NestD{\n",
       "       Y: (10000,)\n",
       "       X: (10000, 784)}\n",
       "     L: NestD{\n",
       "       Y: (10000,)\n",
       "       X: (10000, 784)}}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NestD{\n",
       " U: NestD{\n",
       "   Y: array([5, 7, 1, 7, 1, 1, 6, 3, 0, 2, 9, 3, 1, 1, 0, 4, 9, 2, 0, 0, 2, 0, 2,\n",
       "       7, 1, 8, 6, 4, 1, 6, 3, 4, 5, 9, 1, 3, 3, 8, 5, 4, 7, 7, 4, 2, 8, 5,\n",
       "       8, 6, 7, 3, 4, 6, 1, 9, 9, 6, 0, 3, 7, 2, 8, 2, 9, 4, 4, 6, 4, 9, 7,\n",
       "       0, 9, 2, 9, 5, 1, 5, 9, 1, 2, 3, 2, 3, 5, 9, 1, 7, 6, 2, 8, 2, 2, 5,\n",
       "       0, 7, 4, 9, 7, 8, 3, 2], dtype=int32)\n",
       "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
       " L: NestD{\n",
       "   Y: array([3, 4, 2, 6, 3, 8, 4, 4, 6, 4, 3, 0, 3, 3, 6, 3, 5, 5, 2, 8, 8, 7, 4,\n",
       "       0, 2, 6, 5, 0, 9, 8, 5, 0, 4, 1, 8, 1, 0, 4, 9, 6, 5, 4, 8, 8, 4, 7,\n",
       "       0, 5, 5, 6, 5, 8, 7, 8, 9, 9, 1, 0, 9, 7, 7, 2, 8, 3, 0, 3, 7, 7, 0,\n",
       "       9, 7, 1, 1, 9, 9, 9, 6, 9, 4, 0, 8, 3, 1, 9, 5, 0, 8, 2, 6, 3, 1, 6,\n",
       "       9, 0, 5, 3, 6, 8, 9, 2], dtype=int32)\n",
       "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.train[100:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NestD{\n",
       " U: NestD{\n",
       "   Y: array([7, 0, 9, 1, 6, 2, 8, 3, 6, 4, 9, 5, 8, 6, 8, 7, 8, 8, 6, 9, 1, 7, 6,\n",
       "       0, 9, 6, 7, 0, 9, 7, 1, 3, 6, 8, 4, 6, 1, 7, 5, 1, 3, 3, 5, 7, 9, 9,\n",
       "       6, 7, 3, 4, 1, 0, 4, 2, 4, 5, 0, 0, 1, 6, 6, 4, 7, 9, 4, 6, 5, 2, 6,\n",
       "       9, 8, 8, 8, 5, 9, 3, 8, 9, 8, 8, 8, 3, 4, 4, 3, 0, 9, 5, 4, 4, 1, 8,\n",
       "       0, 6, 1, 3, 2, 0, 8, 6])\n",
       "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
       " L: NestD{\n",
       "   Y: array([7, 0, 9, 1, 6, 2, 8, 3, 6, 4, 9, 5, 8, 6, 8, 7, 8, 8, 6, 9, 1, 7, 6,\n",
       "       0, 9, 6, 7, 0, 9, 7, 1, 3, 6, 8, 4, 6, 1, 7, 5, 1, 3, 3, 5, 7, 9, 9,\n",
       "       6, 7, 3, 4, 1, 0, 4, 2, 4, 5, 0, 0, 1, 6, 6, 4, 7, 9, 4, 6, 5, 2, 6,\n",
       "       9, 8, 8, 8, 5, 9, 3, 8, 9, 8, 8, 8, 3, 4, 4, 3, 0, 9, 5, 4, 4, 1, 8,\n",
       "       0, 6, 1, 3, 2, 0, 8, 6])\n",
       "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       ..., \n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.valid[100:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SemiSupervisedDataEvaluate: NestD{\n",
       "   U: NestD{\n",
       "     Y: (10000,)\n",
       "     X: (10000, 784)}\n",
       "   L: NestD{\n",
       "     Y: (10000,)\n",
       "     X: (10000, 784)}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import models.dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'models.dataloader' from 'models/dataloader.pyc'>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(models.dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "loader = models.dataloader.DataLoader(data.train,batchsize=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\n",
      "NestD{\n",
      " U: NestD{\n",
      "   Y: array([5, 2, 4, 5, 0], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
      " L: NestD{\n",
      "   Y: array([7, 6, 8, 3, 8], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}\n",
      "minibatch['L']['Y']: [7 6 8 3 8]\n",
      "[1]\n",
      "NestD{\n",
      " U: NestD{\n",
      "   Y: array([8, 9, 9, 7, 3], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
      " L: NestD{\n",
      "   Y: array([3, 1, 7, 8, 4], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}\n",
      "minibatch['L']['Y']: [3 1 7 8 4]\n",
      "[2]\n",
      "NestD{\n",
      " U: NestD{\n",
      "   Y: array([1, 3, 9, 4, 1], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
      " L: NestD{\n",
      "   Y: array([7, 0, 7, 1, 3], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}\n",
      "minibatch['L']['Y']: [7 0 7 1 3]\n",
      "[3]\n",
      "NestD{\n",
      " U: NestD{\n",
      "   Y: array([5, 2, 6, 2, 6], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}\n",
      " L: NestD{\n",
      "   Y: array([7, 1, 0, 6, 1], dtype=int32)\n",
      "   X: array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])}}\n",
      "minibatch['L']['Y']: [7 1 0 6 1]\n"
     ]
    }
   ],
   "source": [
    "for i,minibatch in enumerate(loader):\n",
    "    if i > 3:\n",
    "        break\n",
    "    print '[%s]' % i\n",
    "    print minibatch\n",
    "    print \"minibatch['L']['Y']:\",minibatch['L']['Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}