3.4_dataset_split.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset,DataLoader\n",
    "\n",
    "from glob import glob\n",
    "\n",
    "from torchvision.datasets import ImageFolder\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from torchvision import transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SCALE = 224\n",
    "DATA = \"/Users/zhangxiaochen/Downloads/data/content_clean/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transform = transforms.Compose([transforms.Resize((SCALE,SCALE)),transforms.ToTensor(),transforms.Normalize([.5,.5,.5],[.5,.5,.5])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img = Image.open(\"/Users/zhangxiaochen/Downloads/data/content_clean/bike/012b8915381597.562905b9c71a5.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# resize = transforms.Resize((SCALE,SCALE))\n",
    "\n",
    "# resize(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "img_folder = ImageFolder(DATA,transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "DL = DataLoader(img_folder,batch_size=2,shuffle=True)\n",
    "\n",
    "gen = iter(DL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = next(gen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "c2i = img_folder.class_to_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "i2c =dict((v,k) for k,v in c2i.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/Users/zhangxiaochen/Downloads/data/content_clean/animal/0005ac19211849.562d696675b1b.jpg',\n",
       "  0),\n",
       " ('/Users/zhangxiaochen/Downloads/data/content_clean/animal/002d6a52378649.590ee2f628592.jpg',\n",
       "  0),\n",
       " ('/Users/zhangxiaochen/Downloads/data/content_clean/animal/007e7a23659201.563269cfeae07.jpg',\n",
       "  0),\n",
       " ('/Users/zhangxiaochen/Downloads/data/content_clean/animal/008d3020492975.560428a00c91f.jpg',\n",
       "  0),\n",
       " ('/Users/zhangxiaochen/Downloads/data/content_clean/animal/009c2121724315.56306af4eccff.jpg',\n",
       "  0)]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_folder.imgs[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = ImageFolder(DATA,transform=transform)\n",
    "valid = ImageFolder(DATA,transform=transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_list = np.random.rand(len(img_folder.imgs))>.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid.imgs = np.array(valid.imgs)[valid_list].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.imgs = np.array(train.imgs)[~valid_list].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid.samples = valid.imgs\n",
    "train.samples = train.imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = DataLoader(train ,batch_size = 2, shuffle = True)\n",
    "valid_dl = DataLoader(valid ,batch_size = 2, shuffle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}