RNN Language Model with LSTM on Harry Potter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import nltk\n",
    "import random\n",
    "import numpy as np\n",
    "from collections import Counter, OrderedDict\n",
    "from copy import deepcopy\n",
    "from tqdm import tqdm\n",
    "from pprint import pprint\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "random.seed(1994)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "print(USE_CUDA)\n",
    "if USE_CUDA:\n",
    "    gpus = [0]\n",
    "    torch.cuda.set_device(0)\n",
    "\n",
    "FloatTensor = torch.cuda.FloatTensor if USE_CUDA else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if USE_CUDA else torch.LongTensor\n",
    "ByteTensor = torch.cuda.ByteTensor if USE_CUDA else torch.ByteTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_sequence(seq, to_index):\n",
    "    idxs = list(map(lambda w: to_index[w] if to_index.get(w) is not None else to_index[\"<unk>\"], seq))\n",
    "    return LongTensor(idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_ptb_dataset(filename, word2index = None):\n",
    "    corpus = open(filename, 'r', encoding = 'cp1252').readlines()\n",
    "    corpus = flatten([co.strip().split() + ['<\\s'] for co in corpus])\n",
    "    if word2index is None:\n",
    "        vocab = list(set(corpus))\n",
    "        word2index = {'<unk>' : 0}\n",
    "        for vo in vocab:\n",
    "            if word2index.get(vo) is None:\n",
    "                word2index[vo] = len(word2index)\n",
    "    return prepare_sequence(corpus, word2index), word2index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batchify(data, bsz):\n",
    "    nbatch = data.size(0) // bsz\n",
    "    data = data.narrow(0, 0, nbatch * bsz)\n",
    "    data = data.view(bsz, -1).contiguous()\n",
    "    if USE_CUDA:\n",
    "        data = data.cuda()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getBatch(data, seq_length):\n",
    "    for i in range(0, data.size(1) - seq_length, seq_length):\n",
    "        inputs = Variable(data[:, i: i + seq_length])\n",
    "        targets = Variable(data[:, (i + 1): (i + 1) + seq_length].contiguous())\n",
    "        yield inputs, targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, word2index = prepare_ptb_dataset(\"Harry Potter 1 - Sorcerer's Stone.txt\",)\n",
    "dev_data, _ = prepare_ptb_dataset(\"Harry Potter 2 - Chamber of Secrets.txt\", word2index)\n",
    "test_data, _ = prepare_ptb_dataset(\"Harry Potter 3 - The Prisoner Of Azkaban.txt\", word2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11899"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(word2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "index2word = {v: k for k, v in word2index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LanguageModel(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, embedding_dim, hidden_size, n_layers=1, dropout_p = 0.5):\n",
    "        super(LanguageModel, self).__init__()\n",
    "        self.n_layers = n_layers\n",
    "        self.hidden_size = hidden_size\n",
    "        self.embed = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.rnn = nn.LSTM(embedding_dim, hidden_size, n_layers, batch_first=True)\n",
    "        self.linear = nn.Linear(hidden_size, vocab_size)\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "        \n",
    "    def init_weight(self):\n",
    "        self.embed.weight = nn.init.xavier_uniform_(self.embed.weight)\n",
    "        self.linear.weight = nn.init.xavier_uniform_(self.linear.weight)\n",
    "        self.linear.bias.data.fill_(0)\n",
    "        \n",
    "    def init_hidden(self, batch_size):\n",
    "        hidden = Variable(torch.zeros(self.n_layers, batch_size, self.hidden_size))\n",
    "        context = Variable(torch.zeros(self.n_layers, batch_size, self.hidden_size))\n",
    "        return (hidden.cuda(), context.cuda()) if USE_CUDA else (hidden, context)\n",
    "    \n",
    "    def detach_hidden(self, hiddens):\n",
    "        return tuple([hidden.detach() for hidden in hiddens])\n",
    "    \n",
    "    def forward(self, inputs, hidden, is_training=False):\n",
    "        embeds = self.embed(inputs)\n",
    "        if is_training:\n",
    "            embeds = self.dropout(embeds)\n",
    "        out, hidden = self.rnn(embeds, hidden)\n",
    "        return self.linear(out.contiguous().view(out.size(0) * out.size(1), -1)), hidden\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBED_DIM = 128\n",
    "HIDDEN_SIZE = 1024\n",
    "NUM_LAYER = 1\n",
    "LR = 0.01\n",
    "SEQ_LENGTH = 30 \n",
    "BATCH_SIZE = 20\n",
    "EPOCH = 40\n",
    "RESCHEDULED = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = batchify(train_data, BATCH_SIZE)\n",
    "dev_data = batchify(dev_data, BATCH_SIZE//2)\n",
    "test_data = batchify(test_data, BATCH_SIZE//2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LanguageModel(len(word2index), EMBED_DIM, HIDDEN_SIZE, NUM_LAYER, 0.5)\n",
    "model.init_weight()\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "loss_function = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr = LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[00/40] mean_loss : 7.34, Perplexity : 1545.71\n",
      "[00/40] mean_loss : 6.93, Perplexity : 1026.98\n",
      "[01/40] mean_loss : 7.16, Perplexity : 1283.28\n",
      "[01/40] mean_loss : 6.27, Perplexity : 530.85\n",
      "[02/40] mean_loss : 6.02, Perplexity : 412.16\n",
      "[02/40] mean_loss : 5.85, Perplexity : 346.58\n",
      "[03/40] mean_loss : 5.65, Perplexity : 284.58\n",
      "[03/40] mean_loss : 5.45, Perplexity : 233.81\n",
      "[04/40] mean_loss : 5.29, Perplexity : 197.81\n",
      "[04/40] mean_loss : 5.08, Perplexity : 161.46\n",
      "[05/40] mean_loss : 4.95, Perplexity : 141.48\n",
      "[05/40] mean_loss : 4.75, Perplexity : 115.79\n",
      "[06/40] mean_loss : 4.64, Perplexity : 103.06\n",
      "[06/40] mean_loss : 4.41, Perplexity : 82.64\n",
      "[07/40] mean_loss : 4.26, Perplexity : 71.14\n",
      "[07/40] mean_loss : 4.11, Perplexity : 61.08\n",
      "[08/40] mean_loss : 3.94, Perplexity : 51.38\n",
      "[08/40] mean_loss : 3.76, Perplexity : 43.09\n",
      "[09/40] mean_loss : 3.61, Perplexity : 36.87\n",
      "[09/40] mean_loss : 3.43, Perplexity : 30.95\n",
      "[10/40] mean_loss : 3.33, Perplexity : 27.88\n",
      "[10/40] mean_loss : 3.18, Perplexity : 24.11\n",
      "[11/40] mean_loss : 3.05, Perplexity : 21.10\n",
      "[11/40] mean_loss : 2.86, Perplexity : 17.46\n",
      "[12/40] mean_loss : 2.78, Perplexity : 16.10\n",
      "[12/40] mean_loss : 2.62, Perplexity : 13.78\n",
      "[13/40] mean_loss : 2.52, Perplexity : 12.47\n",
      "[13/40] mean_loss : 2.44, Perplexity : 11.43\n",
      "[14/40] mean_loss : 2.31, Perplexity : 10.06\n",
      "[14/40] mean_loss : 2.24, Perplexity : 9.39\n",
      "[15/40] mean_loss : 2.13, Perplexity : 8.40\n",
      "[15/40] mean_loss : 2.01, Perplexity : 7.49\n",
      "[16/40] mean_loss : 1.98, Perplexity : 7.25\n",
      "[16/40] mean_loss : 1.86, Perplexity : 6.44\n",
      "[17/40] mean_loss : 1.78, Perplexity : 5.91\n",
      "[17/40] mean_loss : 1.72, Perplexity : 5.57\n",
      "[18/40] mean_loss : 1.65, Perplexity : 5.19\n",
      "[18/40] mean_loss : 1.59, Perplexity : 4.92\n",
      "[19/40] mean_loss : 1.62, Perplexity : 5.07\n",
      "[19/40] mean_loss : 1.42, Perplexity : 4.15\n",
      "[20/40] mean_loss : 1.45, Perplexity : 4.25\n",
      "[20/40] mean_loss : 1.31, Perplexity : 3.71\n",
      "[21/40] mean_loss : 1.33, Perplexity : 3.79\n",
      "[21/40] mean_loss : 1.20, Perplexity : 3.33\n",
      "[22/40] mean_loss : 1.23, Perplexity : 3.43\n",
      "[22/40] mean_loss : 1.17, Perplexity : 3.22\n",
      "[23/40] mean_loss : 1.12, Perplexity : 3.08\n",
      "[23/40] mean_loss : 1.02, Perplexity : 2.78\n",
      "[24/40] mean_loss : 0.98, Perplexity : 2.66\n",
      "[24/40] mean_loss : 1.03, Perplexity : 2.79\n",
      "[25/40] mean_loss : 0.96, Perplexity : 2.61\n",
      "[25/40] mean_loss : 0.91, Perplexity : 2.47\n",
      "[26/40] mean_loss : 0.96, Perplexity : 2.60\n",
      "[26/40] mean_loss : 0.80, Perplexity : 2.23\n",
      "[27/40] mean_loss : 0.91, Perplexity : 2.49\n",
      "[27/40] mean_loss : 0.72, Perplexity : 2.06\n",
      "[28/40] mean_loss : 0.78, Perplexity : 2.18\n",
      "[28/40] mean_loss : 0.70, Perplexity : 2.01\n",
      "[29/40] mean_loss : 0.65, Perplexity : 1.92\n",
      "[29/40] mean_loss : 0.66, Perplexity : 1.93\n",
      "[30/40] mean_loss : 0.63, Perplexity : 1.89\n",
      "[30/40] mean_loss : 0.61, Perplexity : 1.84\n",
      "[31/40] mean_loss : 0.57, Perplexity : 1.77\n",
      "[31/40] mean_loss : 0.57, Perplexity : 1.77\n",
      "[32/40] mean_loss : 0.48, Perplexity : 1.62\n",
      "[32/40] mean_loss : 0.50, Perplexity : 1.65\n",
      "[33/40] mean_loss : 0.54, Perplexity : 1.72\n",
      "[33/40] mean_loss : 0.46, Perplexity : 1.58\n",
      "[34/40] mean_loss : 0.48, Perplexity : 1.62\n",
      "[34/40] mean_loss : 0.40, Perplexity : 1.48\n",
      "[35/40] mean_loss : 0.40, Perplexity : 1.49\n",
      "[35/40] mean_loss : 0.38, Perplexity : 1.46\n",
      "[36/40] mean_loss : 0.36, Perplexity : 1.43\n",
      "[36/40] mean_loss : 0.33, Perplexity : 1.39\n",
      "[37/40] mean_loss : 0.35, Perplexity : 1.42\n",
      "[37/40] mean_loss : 0.38, Perplexity : 1.47\n",
      "[38/40] mean_loss : 0.35, Perplexity : 1.41\n",
      "[38/40] mean_loss : 0.28, Perplexity : 1.32\n",
      "[39/40] mean_loss : 0.31, Perplexity : 1.37\n",
      "[39/40] mean_loss : 0.29, Perplexity : 1.34\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCH):\n",
    "    total_loss = 0\n",
    "    losses = []\n",
    "    hidden = model.init_hidden(BATCH_SIZE)\n",
    "    for i, batch in enumerate(getBatch(train_data, BATCH_SIZE)):\n",
    "        inputs, targets = batch\n",
    "        hidden = model.detach_hidden(hidden)\n",
    "        model.zero_grad()\n",
    "        preds, hidden = model(inputs, hidden, True)\n",
    "        \n",
    "        loss = loss_function(preds, targets.view(-1))\n",
    "        losses.append(loss.data.cpu().numpy())\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n",
    "        optimizer.step()\n",
    "        if i > 0 and i % 100 == 0:  \n",
    "            print(\"[%02d/%d] mean_loss : %0.2f, Perplexity : %0.2f\" % (epoch,EPOCH, np.mean(losses), np.exp(np.mean(losses))))\n",
    "            losses = [] \n",
    "    if RESCHEDULED == False and epoch == EPOCH//2:\n",
    "        LR *= 0.1\n",
    "        optimizer = optim.Adam(model.parameters(), lr=LR)\n",
    "        RESCHEDULED = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Perpelexity : 24889.33\n"
     ]
    }
   ],
   "source": [
    "total_loss = 0\n",
    "hidden = model.init_hidden(BATCH_SIZE//2)\n",
    "for batch in getBatch(test_data, SEQ_LENGTH):\n",
    "    inputs,targets = batch\n",
    "        \n",
    "    hidden = model.detach_hidden(hidden)\n",
    "    model.zero_grad()\n",
    "    preds, hidden = model(inputs, hidden)\n",
    "    total_loss += inputs.size(1) * loss_function(preds, targets.view(-1)).data\n",
    "\n",
    "total_loss = total_loss/test_data.size(1)\n",
    "print(\"Test Perpelexity : %5.2f\" % (np.exp(total_loss.data.cpu().numpy())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our training set was Harry Potter 1st book, our dev set was Harry Potter 2nd book, and our test set was Harry Potter 3rd book. This shows that the books are different from each other not only in terms of events, but also in terms of language modeling. I hypothesize that since there are words in all books which are not easily identiable with \"English\" language modeling and they change from book to book, the algorithm had difficulties identifying structure on the test set when trained on a totally different book. The other conclusion is that we might be overfitting. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next step is to divide one book into train/dev/test to see if it make any improvements."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}