3.2_understand_rnn_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from torch.utils.data.dataset import TensorDataset\n",
    "from torch.optim import Adam\n",
    "from torch.autograd import Variable\n",
    "from torch import cuda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.datasets import reuters\n",
    "from keras.preprocessing.sequence import  pad_sequences\n",
    "from tqdm import trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/text-datasets/reuters.npz\n",
      "2113536/2110848 [==============================] - 5s 3us/step\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = reuters.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (x_train, y_train), (x_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_len = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_p,x_test_p = tuple(np.clip(np.array(pad_sequences(a,maxlen=200,value = vocab_len-1 )),0,vocab_len-1) for a in [x_train,x_test])\n",
    "\n",
    "y_train_oh, y_test_oh = tuple(np.eye(max(y_train)+1)[a] for a in [y_train,y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ds = TensorDataset(torch.from_numpy(x_train_p),torch.from_numpy(y_train))\n",
    "test_ds = TensorDataset(torch.from_numpy(x_test_p),torch.from_numpy(y_test))\n",
    "\n",
    "train_dl = DataLoader(train_ds, batch_size=128, num_workers=2,shuffle=True)\n",
    "test_dl = DataLoader(test_ds, batch_size=128, num_workers=2,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class senti(nn.Module):\n",
    "    def __init__(self,seq_len,in_size,hidden,rnnmodel,nb_class,fc1_size=256):\n",
    "        super(senti,self).__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.in_size = in_size\n",
    "        self.hidden = hidden\n",
    "        self.fc1_size = fc1_size\n",
    "        self.nb_class = nb_class\n",
    "        \n",
    "        self.ebd = nn.Embedding(vocab_len,self.in_size)\n",
    "        \n",
    "        self.rnn = rnnmodel(seq_len=self.seq_len,in_size=self.in_size,hidden=self.hidden)\n",
    "\n",
    "        self.bn1 = nn.BatchNorm1d(self.hidden)\n",
    "        self.fc1 = nn.Linear(self.hidden,self.fc1_size)\n",
    "        \n",
    "        self.bn2 = nn.BatchNorm1d(self.fc1_size)\n",
    "        self.fc2 = nn.Linear(self.fc1_size,self.nb_class)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        x = self.ebd(x)\n",
    "        x = self.rnn(x)\n",
    "        \n",
    "        x = self.bn1(x)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.bn2(x)\n",
    "        x = F.softmax(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "class rnn(nn.Module):\n",
    "    def __init__(self,seq_len,in_size,hidden):\n",
    "        super(rnn,self).__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.hidden = hidden\n",
    "        self.in_size = in_size\n",
    "        \n",
    "        self.dense1 = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "    def forward(self,xs):\n",
    "        bs = int(xs.size()[0])\n",
    "        cellout = Variable(torch.zeros(bs,self.hidden))\n",
    "        \n",
    "        for i in range(self.seq_len):\n",
    "            ipt = torch.cat([xs[:,i],cellout],1)\n",
    "            cellout += F.tanh(self.dense1(ipt))\n",
    "        return cellout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class rnn_forget(nn.Module):\n",
    "    def __init__(self,seq_len,in_size,hidden):\n",
    "        super(rnn_forget,self).__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.hidden = hidden\n",
    "        self.in_size = in_size\n",
    "        \n",
    "        self.gate_fgt = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "        self.dense1 = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "    def forward(self,xs):\n",
    "        bs = int(xs.size()[0])\n",
    "        cellout = Variable(torch.zeros(bs,self.hidden))\n",
    "        \n",
    "        for i in range(self.seq_len):\n",
    "            ipt = torch.cat([xs[:,i],cellout],1)\n",
    "            fgt = F.sigmoid(self.gate_fgt(ipt))\n",
    "            \n",
    "            cellout = cellout * fgt\n",
    "            \n",
    "            cellout += F.tanh(self.dense1(ipt))\n",
    "        return cellout\n",
    "    \n",
    "class rnn_in(nn.Module):\n",
    "    def __init__(self,seq_len,in_size,hidden):\n",
    "        super(rnn_in,self).__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.hidden = hidden\n",
    "        self.in_size = in_size\n",
    "        \n",
    "#         self.gate_fgt = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        self.gate_in = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "        self.dense1 = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "    def forward(self,xs):\n",
    "        bs = int(xs.size()[0])\n",
    "        cellout = Variable(torch.zeros(bs,self.hidden))\n",
    "        \n",
    "        for i in range(self.seq_len):\n",
    "            ipt = torch.cat([xs[:,i],cellout],1)\n",
    "#             fgt = F.sigmoid(self.gate_fgt(ipt))\n",
    "            igt = F.sigmoid(self.gate_in(ipt))\n",
    "            \n",
    "#             cellout = cellout * fgt\n",
    "            \n",
    "            cellout += igt * F.tanh(self.dense1(ipt))\n",
    "        return cellout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class lstm(nn.Module):\n",
    "    def __init__(self,seq_len,in_size,hidden):\n",
    "        \"\"\"Long short term memory\"\"\"\n",
    "        super(lstm,self).__init__()\n",
    "        self.seq_len = seq_len\n",
    "        self.hidden = hidden\n",
    "        self.in_size = in_size\n",
    "        \n",
    "        self.gate_fgt = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        self.gate_in = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        self.gate_out = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        \n",
    "        self.dense1 = nn.Linear(self.in_size+self.hidden,self.hidden)\n",
    "        self.dense2 = nn.Linear(self.hidden,self.hidden)\n",
    "        \n",
    "    def forward(self,xs):\n",
    "        bs = int(xs.size()[0])\n",
    "        cell = Variable(torch.zeros(bs,self.hidden))\n",
    "        h = Variable(torch.ones(bs,self.hidden))\n",
    "        \n",
    "        for i in range(self.seq_len):\n",
    "            ipt = torch.cat([xs[:,i],h],1)\n",
    "            \n",
    "            # 3 gates\n",
    "            fgt = F.sigmoid(self.gate_fgt(ipt))\n",
    "            igt = F.sigmoid(self.gate_in(ipt))\n",
    "            ogt = F.sigmoid(self.gate_out(ipt))\n",
    "            \n",
    "            cell = cell * fgt\n",
    "            \n",
    "            cell += igt * F.tanh(self.dense2(F.relu(self.dense1(ipt))))\n",
    "            \n",
    "            h = ogt * F.tanh(cell)\n",
    "            \n",
    "        return h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class lstm_cell(nn.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = senti(seq_len=200,in_size=50,hidden=60,nb_class=46,rnnmodel=lstm)\n",
    "if CUDA:\n",
    "    model.CUDA()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "ce_loss = nn.CrossEntropyLoss()\n",
    "opt = Adam(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(y_,y):\n",
    "    y_max,y_idx = torch.max(y_,1)\n",
    "    acc = torch.mean(torch.eq(y_idx,y).type(torch.FloatTensor))\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "CUDA = cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/71 [00:00<?, ?it/s]\u001b[A\n",
      "ep0_bt69\t cross_ent:\t3.54\tacc:\t0.328: 100%|██████████| 71/71 [01:41<00:00,  1.43s/it]\n",
      "ep1_bt69\t cross_ent:\t3.53\tacc:\t0.338: 100%|██████████| 71/71 [01:24<00:00,  1.19s/it]\n"
     ]
    }
   ],
   "source": [
    "window = 5\n",
    "for epoch in range(2):\n",
    "    train_gen = iter(train_dl)\n",
    "    test_gen = iter(test_dl)\n",
    "    r_acc = 0\n",
    "    r_ce = 0\n",
    "    t=trange(len(train_dl))\n",
    "    for i in t:\n",
    "        x,y = train_gen.__next__()\n",
    "        x,y = Variable(x.type(torch.LongTensor)),Variable(y)\n",
    "        if CUDA:\n",
    "            x.cuda()\n",
    "            y.cuda()\n",
    "        \n",
    "        y_ = model(x)\n",
    "        ce = ce_loss(y_,y)\n",
    "        # calc accuracy\n",
    "        y_max,y_idx = torch.max(y_,1)\n",
    "        acc = accuracy(y_,y)\n",
    "        \n",
    "        opt.zero_grad()\n",
    "        \n",
    "        ce.backward()\n",
    "        \n",
    "        opt.step()\n",
    "        r_acc += acc.data[0]\n",
    "        r_ce += ce.data[0]\n",
    "        if i % window ==(window-1):\n",
    "            t.set_description(\"ep%s_bt%s\\t cross_ent:\\t%.2f\\tacc:\\t%.3f\"%(epoch,\n",
    "                                             i,\n",
    "                                             r_ce/window,\n",
    "                                             r_acc/window,\n",
    "                                             ))\n",
    "            r_acc,r_ce = 0,0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "rnn, relu:\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.50\tacc:\t0.383: 100%|██████████| 71/71 [00:56<00:00,  1.26it/s]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.52\tacc:\t0.345: 100%|██████████| 71/71 [00:57<00:00,  1.24it/s]\n",
    "\n",
    "rnn_forget:\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.72\tacc:\t0.205: 100%|██████████| 71/71 [01:02<00:00,  1.14it/s]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.66\tacc:\t0.208: 100%|██████████| 71/71 [01:00<00:00,  1.17it/s]\n",
    "\n",
    "rnn,tanh\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.57\tacc:\t0.295: 100%|██████████| 71/71 [00:58<00:00,  1.22it/s]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.53\tacc:\t0.342: 100%|██████████| 71/71 [01:00<00:00,  1.17it/s]\n",
    "\n",
    "rnn_in, tanh\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.53\tacc:\t0.347: 100%|██████████| 71/71 [01:05<00:00,  1.09it/s]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.47\tacc:\t0.397: 100%|██████████| 71/71 [01:06<00:00,  1.07it/s]\n",
    "\n",
    "lstm, hidden:60\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.54\tacc:\t0.336: 100%|██████████| 71/71 [01:27<00:00,  1.24s/it]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.51\tacc:\t0.359: 100%|██████████| 71/71 [01:36<00:00,  1.36s/it]\n",
    "\n",
    "lstm, with vocab length:5000\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.63\tacc:\t0.355: 100%|██████████| 71/71 [01:32<00:00,  1.30s/it]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.50\tacc:\t0.364: 100%|██████████| 71/71 [01:30<00:00,  1.28s/it]\n",
    "\n",
    "ep0_bt69\t cross_ent:\t3.52\tacc:\t0.347: 100%|██████████| 71/71 [01:34<00:00,  1.33s/it]\n",
    "\n",
    "ep1_bt69\t cross_ent:\t3.49\tacc:\t0.377: 100%|██████████| 71/71 [01:35<00:00,  1.35s/it]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}