Seq2Seqipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ChatbotPROD.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hood-boi/world-news-chatbot/blob/master/Seq2Seqipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QN6xgjtZ9qye",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import torch.optim as optim\n",
        "import torchtext\n",
        "import random\n",
        "import requests\n",
        "import json\n",
        "import time"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rY5Pu52_-B2l",
        "colab_type": "code",
        "outputId": "80e839d1-6ddd-4c31-afa7-095540f36274",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "swLeAlWu93D-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_field = torchtext.data.Field(sequential=True,      # text sequence\n",
        "                                  tokenize=lambda x: x, # because are building a character-RNN\n",
        "                                  include_lengths=True, # to track the length of sequences, for batching\n",
        "                                  batch_first=True,\n",
        "                                  use_vocab=True,\n",
        "                                  init_token=\"<BOS>\",\n",
        "                                  eos_token=\"<EOS>\"\n",
        "                                 )       # to turn each character into an integer index\n",
        "label_field = torchtext.data.Field(sequential=True,    # text sequence\n",
        "                                   use_vocab=True,     # don't need to track vocabulary\n",
        "                                   is_target=True,      \n",
        "                                   batch_first=True,\n",
        "                                   tokenize=lambda x: x,\n",
        "                                   preprocessing=lambda x: x,\n",
        "                                   init_token=\"<BOS>\",\n",
        "                                   eos_token=\"<EOS>\"\n",
        "                                  ) \n",
        "\n",
        "fields = [('reply', label_field), ('context', text_field)]\n",
        "dataset1 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rtodayilearned.txt\", \"tsv\", fields); # name of the file\n",
        "dataset2 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rAskReddit.txt\", \"tsv\", fields); # name of the file\n",
        "dataset3 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rnews.txt\", \"tsv\", fields); # name of the file\n",
        "dataset4 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rworldnews.txt\", \"tsv\", fields); # name of the file\n",
        "dataset5 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/routoftheloop.txt\", \"tsv\", fields); # name of the file\n",
        "dataset6 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rnottheonion.txt\", \"tsv\", fields); # name of the file\n",
        "dataset7 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/rCasualConversation.txt\", \"tsv\", fields); # name of the file\n",
        "dataset8 = torchtext.data.TabularDataset(\"/content/gdrive/My Drive/Chatbot/text.txt\", \"tsv\", fields);\n",
        "dataset = torch.utils.data.ConcatDataset([dataset1, dataset2, dataset3, dataset4, dataset5, dataset6, dataset7, dataset8])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2pzb3t2uTApY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! cat '/content/gdrive/My Drive/Chatbot/text.txt'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aLAl_ZT4AFYr",
        "colab_type": "code",
        "outputId": "a8067d42-7817-4250-bbc3-fd212503a1d1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "train = torchtext.data.Dataset(dataset, fields)\n",
        "text_field.build_vocab(train)\n",
        "label_field.build_vocab(train)\n",
        "print(text_field.vocab.itos)\n",
        "print(label_field.vocab.itos)\n",
        "input_vocab_size = len(text_field.vocab.itos)\n",
        "reply_vocab_size = len(label_field.vocab.itos)\n",
        "print(\"Input Vocab Size: \", input_vocab_size)\n",
        "print(\"Reply Vocab size: \", reply_vocab_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['<unk>', '<pad>', '<BOS>', '<EOS>', ' ', 'e', 't', 'a', 'o', 'i', 'n', 's', 'r', 'h', 'l', 'd', 'u', 'c', 'm', 'g', 'y', 'p', 'f', 'w', 'b', '.', 'k', 'v', ',', 'I', 'T', 'S', 'A', 'C', 'W', '-', '/', '0', '?', 'M', 'x', 'j', 'P', 'H', 'B', 'N', 'D', 'E', '1', ':', 'R', 'F', 'O', '2', 'z', 'G', 'L', 'U', '*', '!', 'Y', 'J', 'K', '5', ')', '(', 'q', '3', '4', ';', '9', '8', '6', '7', 'V', '_', '$', ']', '[', '%', '=', '^', '~', 'Z', '&', 'Q', 'X', '#', '+', '\\\\', '|', '@', '`', '{', '}']\n",
            "['<unk>', '<pad>', '<BOS>', '<EOS>', ' ', 'e', 't', 'o', 'a', 'i', 'n', 's', 'r', 'h', 'l', 'd', 'u', 'c', 'm', 'y', 'g', 'w', 'p', 'f', '.', 'b', 'k', 'v', 'I', ',', 'T', 'A', 'S', '/', '?', 'j', 'W', '-', 'x', 'H', 'C', '0', 'M', 'N', '!', 'B', 'E', 'O', 'D', 'P', 'R', '1', 'Y', ':', '*', 'F', 'z', 'L', 'G', '2', 'U', ')', '(', 'q', ';', 'J', '5', '3', 'K', '4', '9', '_', '8', '6', '7', 'V', ']', '[', '^', '$', '%', '=', '~', '&', 'Q', 'Z', 'X', '#', '\\\\', '+', '@', '|', '`', '}', '{']\n",
            "Input Vocab Size:  95\n",
            "Reply Vocab size:  95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VKtPCLEWBQIE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ChatBot(nn.Module):\n",
        "    def __init__(self, \n",
        "                 context_vocab_size,  \n",
        "                 reply_vocab_size, \n",
        "                 encoder_hidden_size = 100,\n",
        "                 generator_hidden_size = 100, \n",
        "                 encoder_layers = 1, \n",
        "                 generator_layers = 1):\n",
        "        \n",
        "        super(ChatBot, self).__init__()\n",
        "        \n",
        "        self.encoder_layers = encoder_layers;\n",
        "        self.generator_layers =generator_layers;\n",
        "        self.encoder_hidden_size = encoder_hidden_size;\n",
        "        self.generator_hidden_size = generator_hidden_size;\n",
        "        \n",
        "        # >>> Encoder\n",
        "        self.context_ident = torch.eye(context_vocab_size)\n",
        "        self.encode_rnn = nn.LSTM(context_vocab_size, encoder_hidden_size, encoder_layers, batch_first=True)\n",
        "        \n",
        "        # >>> Generator\n",
        "        self.reply_ident = torch.eye(reply_vocab_size)\n",
        "        self.decode_rnn = nn.LSTM(reply_vocab_size, generator_hidden_size, generator_layers, batch_first=True)\n",
        "        self.fcnn = nn.Linear(generator_hidden_size, reply_vocab_size)\n",
        "        \n",
        "    def forward(self, context, response, hidden=None):\n",
        "        \n",
        "        # >>> Encoder\n",
        "        context_tensor = self.context_ident[context] # Type: batch.context[0] | Size: (batch size, sequence size)\n",
        "        encode_out, encode_last_hidden = self.encode_rnn(context_tensor)\n",
        "        \n",
        "        # >>> Generator\n",
        "        reply_tensor = self.reply_ident[response] #Type: batch.reply\n",
        "        if(hidden == None):\n",
        "            gen_out, gen_last_hidden = self.decode_rnn(reply_tensor, encode_last_hidden)\n",
        "        else:\n",
        "            gen_out, gen_last_hidden = self.decode_rnn(reply_tensor, hidden)\n",
        "        out = self.fcnn(gen_out)\n",
        "        return out, gen_last_hidden"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yo2hAQN-lMfn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = ChatBot(input_vocab_size, reply_vocab_size, encoder_layers = 2, generator_layers = 2)\n",
        "model = model.cuda();\n",
        "model.context_ident = model.context_ident.cuda();\n",
        "model.reply_ident = model.reply_ident.cuda();"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sJn3trg9HrgM",
        "colab_type": "code",
        "outputId": "5e9b5c7d-4efc-4505-f790-0cb5798c7a87",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "len(dataset)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1321762"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dcR-6mLdl_M6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_fcn(model, data, reply_vocab_size, batch_size=1, num_epochs=1, lr=0.001, print_every=100):\n",
        "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    it = 0\n",
        "    \n",
        "    data_iter = torchtext.data.BucketIterator(data,\n",
        "                                              batch_size=batch_size,\n",
        "                                              sort_key=lambda x: len(x.context),\n",
        "                                              sort_within_batch=True)\n",
        "    \n",
        "    iters, losses = [], [];\n",
        "    \n",
        "    \n",
        "    for e in range(num_epochs):\n",
        "        torch.save(model.state_dict(), '/content/gdrive/My Drive/Chatbot/MM_WEIGHTS_'+str(e))\n",
        "        print(\"Epoch : \", e);\n",
        "        # get training set\n",
        "        avg_loss = 0\n",
        "        for batch in data_iter:\n",
        "            optimizer.zero_grad();\n",
        "            inp = batch.context[0] # BOS + EOS\n",
        "            inp = inp.cuda()\n",
        "            reply = batch.reply[:, :-1]\n",
        "            reply = reply.cuda()\n",
        "            \n",
        "            out, hidden = model(inp, reply) # no eos\n",
        "            out_reshaped = out.reshape(-1,reply_vocab_size)\n",
        "            target = (batch.reply[:,1:]).reshape(-1) # no bos\n",
        "            target = target.cuda()\n",
        "            #print(batch.reply[:,1:])\n",
        "            loss = criterion(out_reshaped, target)\n",
        "            \n",
        "            loss.backward();\n",
        "            optimizer.step();\n",
        "            \n",
        "            avg_loss += loss\n",
        "            it += 1 # increment iteration count\n",
        "            iters.append(it)\n",
        "            losses.append(float(loss)/batch_size)\n",
        "            if it % print_every == 0:\n",
        "                print(\"[Iter %d] Loss %f\" % (it+1, float(avg_loss/print_every)))\n",
        "                avg_loss = 0\n",
        "        torch.save(model.state_dict(), '/content/gdrive/My Drive/Chatbot/MM_WEIGHTS_'+str(e))\n",
        "    \n",
        "    np.savetxt(\"{}_train_loss.csv\".format('LSTM'), losses);"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Meh9s6ctmVws",
        "colab_type": "code",
        "outputId": "e91d72ce-0a35-482c-c6ee-21fad67dee0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "train_fcn(model, train, reply_vocab_size, batch_size =512, num_epochs = 50)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch :  0\n",
            "[Iter 101] Loss 1.661044\n",
            "[Iter 201] Loss 0.797015\n",
            "[Iter 301] Loss 0.760498\n",
            "[Iter 401] Loss 0.749910\n",
            "[Iter 501] Loss 0.743037\n",
            "[Iter 601] Loss 0.715566\n",
            "[Iter 701] Loss 0.688625\n",
            "[Iter 801] Loss 0.660529\n",
            "[Iter 901] Loss 0.624745\n",
            "[Iter 1001] Loss 0.602346\n",
            "[Iter 1101] Loss 0.590690\n",
            "[Iter 1201] Loss 0.571602\n",
            "[Iter 1301] Loss 0.560419\n",
            "[Iter 1401] Loss 0.547505\n",
            "[Iter 1501] Loss 0.536420\n",
            "[Iter 1601] Loss 0.527483\n",
            "[Iter 1701] Loss 0.518374\n",
            "[Iter 1801] Loss 0.507543\n",
            "[Iter 1901] Loss 0.500341\n",
            "[Iter 2001] Loss 0.495979\n",
            "[Iter 2101] Loss 0.486021\n",
            "[Iter 2201] Loss 0.481268\n",
            "[Iter 2301] Loss 0.473040\n",
            "[Iter 2401] Loss 0.468271\n",
            "[Iter 2501] Loss 0.463335\n",
            "Epoch :  1\n",
            "[Iter 2601] Loss 0.080763\n",
            "[Iter 2701] Loss 0.459426\n",
            "[Iter 2801] Loss 0.441515\n",
            "[Iter 2901] Loss 0.446407\n",
            "[Iter 3001] Loss 0.439496\n",
            "[Iter 3101] Loss 0.439194\n",
            "[Iter 3201] Loss 0.439778\n",
            "[Iter 3301] Loss 0.429015\n",
            "[Iter 3401] Loss 0.419408\n",
            "[Iter 3501] Loss 0.422324\n",
            "[Iter 3601] Loss 0.424536\n",
            "[Iter 3701] Loss 0.413139\n",
            "[Iter 3801] Loss 0.420559\n",
            "[Iter 3901] Loss 0.411753\n",
            "[Iter 4001] Loss 0.402666\n",
            "[Iter 4101] Loss 0.414029\n",
            "[Iter 4201] Loss 0.407145\n",
            "[Iter 4301] Loss 0.403517\n",
            "[Iter 4401] Loss 0.398303\n",
            "[Iter 4501] Loss 0.398288\n",
            "[Iter 4601] Loss 0.397614\n",
            "[Iter 4701] Loss 0.400806\n",
            "[Iter 4801] Loss 0.388283\n",
            "[Iter 4901] Loss 0.398496\n",
            "[Iter 5001] Loss 0.385046\n",
            "[Iter 5101] Loss 0.389309\n",
            "Epoch :  2\n",
            "[Iter 5201] Loss 0.133451\n",
            "[Iter 5301] Loss 0.392293\n",
            "[Iter 5401] Loss 0.380625\n",
            "[Iter 5501] Loss 0.386329\n",
            "[Iter 5601] Loss 0.378535\n",
            "[Iter 5701] Loss 0.384430\n",
            "[Iter 5801] Loss 0.371917\n",
            "[Iter 5901] Loss 0.385141\n",
            "[Iter 6001] Loss 0.369762\n",
            "[Iter 6101] Loss 0.378859\n",
            "[Iter 6201] Loss 0.370124\n",
            "[Iter 6301] Loss 0.376399\n",
            "[Iter 6401] Loss 0.363580\n",
            "[Iter 6501] Loss 0.373441\n",
            "[Iter 6601] Loss 0.369306\n",
            "[Iter 6701] Loss 0.372129\n",
            "[Iter 6801] Loss 0.364826\n",
            "[Iter 6901] Loss 0.366952\n",
            "[Iter 7001] Loss 0.368333\n",
            "[Iter 7101] Loss 0.370146\n",
            "[Iter 7201] Loss 0.359208\n",
            "[Iter 7301] Loss 0.365441\n",
            "[Iter 7401] Loss 0.359485\n",
            "[Iter 7501] Loss 0.357729\n",
            "[Iter 7601] Loss 0.370016\n",
            "[Iter 7701] Loss 0.357965\n",
            "Epoch :  3\n",
            "[Iter 7801] Loss 0.189180\n",
            "[Iter 7901] Loss 0.360695\n",
            "[Iter 8001] Loss 0.360671\n",
            "[Iter 8101] Loss 0.364812\n",
            "[Iter 8201] Loss 0.348620\n",
            "[Iter 8301] Loss 0.355205\n",
            "[Iter 8401] Loss 0.357541\n",
            "[Iter 8501] Loss 0.350031\n",
            "[Iter 8601] Loss 0.362117\n",
            "[Iter 8701] Loss 0.353100\n",
            "[Iter 8801] Loss 0.351417\n",
            "[Iter 8901] Loss 0.353110\n",
            "[Iter 9001] Loss 0.357086\n",
            "[Iter 9101] Loss 0.351111\n",
            "[Iter 9201] Loss 0.347649\n",
            "[Iter 9301] Loss 0.351898\n",
            "[Iter 9401] Loss 0.353877\n",
            "[Iter 9501] Loss 0.351761\n",
            "[Iter 9601] Loss 0.348371\n",
            "[Iter 9701] Loss 0.349166\n",
            "[Iter 9801] Loss 0.342939\n",
            "[Iter 9901] Loss 0.353605\n",
            "[Iter 10001] Loss 0.350942\n",
            "[Iter 10101] Loss 0.348094\n",
            "[Iter 10201] Loss 0.341945\n",
            "[Iter 10301] Loss 0.349718\n",
            "Epoch :  4\n",
            "[Iter 10401] Loss 0.251180\n",
            "[Iter 10501] Loss 0.339947\n",
            "[Iter 10601] Loss 0.345073\n",
            "[Iter 10701] Loss 0.345270\n",
            "[Iter 10801] Loss 0.341236\n",
            "[Iter 10901] Loss 0.346670\n",
            "[Iter 11001] Loss 0.349130\n",
            "[Iter 11101] Loss 0.342536\n",
            "[Iter 11201] Loss 0.342225\n",
            "[Iter 11301] Loss 0.342593\n",
            "[Iter 11401] Loss 0.347510\n",
            "[Iter 11501] Loss 0.340025\n",
            "[Iter 11601] Loss 0.337841\n",
            "[Iter 11701] Loss 0.349676\n",
            "[Iter 11801] Loss 0.337839\n",
            "[Iter 11901] Loss 0.333917\n",
            "[Iter 12001] Loss 0.344994\n",
            "[Iter 12101] Loss 0.345445\n",
            "[Iter 12201] Loss 0.339519\n",
            "[Iter 12301] Loss 0.343005\n",
            "[Iter 12401] Loss 0.341121\n",
            "[Iter 12501] Loss 0.339749\n",
            "[Iter 12601] Loss 0.339582\n",
            "[Iter 12701] Loss 0.345238\n",
            "[Iter 12801] Loss 0.338759\n",
            "[Iter 12901] Loss 0.340622\n",
            "Epoch :  5\n",
            "[Iter 13001] Loss 0.303015\n",
            "[Iter 13101] Loss 0.338415\n",
            "[Iter 13201] Loss 0.341420\n",
            "[Iter 13301] Loss 0.334723\n",
            "[Iter 13401] Loss 0.340722\n",
            "[Iter 13501] Loss 0.338399\n",
            "[Iter 13601] Loss 0.336056\n",
            "[Iter 13701] Loss 0.340331\n",
            "[Iter 13801] Loss 0.339572\n",
            "[Iter 13901] Loss 0.334225\n",
            "[Iter 14001] Loss 0.338397\n",
            "[Iter 14101] Loss 0.336434\n",
            "[Iter 14201] Loss 0.336131\n",
            "[Iter 14301] Loss 0.331357\n",
            "[Iter 14401] Loss 0.339518\n",
            "[Iter 14501] Loss 0.338318\n",
            "[Iter 14601] Loss 0.337513\n",
            "[Iter 14701] Loss 0.333233\n",
            "[Iter 14801] Loss 0.337172\n",
            "[Iter 14901] Loss 0.337228\n",
            "[Iter 15001] Loss 0.336367\n",
            "[Iter 15101] Loss 0.338271\n",
            "[Iter 15201] Loss 0.334311\n",
            "[Iter 15301] Loss 0.331950\n",
            "[Iter 15401] Loss 0.335994\n",
            "Epoch :  6\n",
            "[Iter 15501] Loss 0.026822\n",
            "[Iter 15601] Loss 0.336429\n",
            "[Iter 15701] Loss 0.333325\n",
            "[Iter 15801] Loss 0.332343\n",
            "[Iter 15901] Loss 0.336615\n",
            "[Iter 16001] Loss 0.331170\n",
            "[Iter 16101] Loss 0.336939\n",
            "[Iter 16201] Loss 0.336623\n",
            "[Iter 16301] Loss 0.329498\n",
            "[Iter 16401] Loss 0.335361\n",
            "[Iter 16501] Loss 0.334024\n",
            "[Iter 16601] Loss 0.330425\n",
            "[Iter 16701] Loss 0.333691\n",
            "[Iter 16801] Loss 0.331731\n",
            "[Iter 16901] Loss 0.331206\n",
            "[Iter 17001] Loss 0.331760\n",
            "[Iter 17101] Loss 0.330734\n",
            "[Iter 17201] Loss 0.335535\n",
            "[Iter 17301] Loss 0.331613\n",
            "[Iter 17401] Loss 0.331883\n",
            "[Iter 17501] Loss 0.331737\n",
            "[Iter 17601] Loss 0.332335\n",
            "[Iter 17701] Loss 0.329764\n",
            "[Iter 17801] Loss 0.331164\n",
            "[Iter 17901] Loss 0.332337\n",
            "[Iter 18001] Loss 0.332832\n",
            "Epoch :  7\n",
            "[Iter 18101] Loss 0.085308\n",
            "[Iter 18201] Loss 0.329984\n",
            "[Iter 18301] Loss 0.334684\n",
            "[Iter 18401] Loss 0.328478\n",
            "[Iter 18501] Loss 0.334720\n",
            "[Iter 18601] Loss 0.322526\n",
            "[Iter 18701] Loss 0.336100\n",
            "[Iter 18801] Loss 0.326828\n",
            "[Iter 18901] Loss 0.329991\n",
            "[Iter 19001] Loss 0.330503\n",
            "[Iter 19101] Loss 0.334380\n",
            "[Iter 19201] Loss 0.328799\n",
            "[Iter 19301] Loss 0.323881\n",
            "[Iter 19401] Loss 0.336700\n",
            "[Iter 19501] Loss 0.326698\n",
            "[Iter 19601] Loss 0.331407\n",
            "[Iter 19701] Loss 0.325584\n",
            "[Iter 19801] Loss 0.337919\n",
            "[Iter 19901] Loss 0.327813\n",
            "[Iter 20001] Loss 0.328558\n",
            "[Iter 20101] Loss 0.332286\n",
            "[Iter 20201] Loss 0.331448\n",
            "[Iter 20301] Loss 0.328647\n",
            "[Iter 20401] Loss 0.326967\n",
            "[Iter 20501] Loss 0.332285\n",
            "[Iter 20601] Loss 0.325913\n",
            "Epoch :  8\n",
            "[Iter 20701] Loss 0.145454\n",
            "[Iter 20801] Loss 0.323456\n",
            "[Iter 20901] Loss 0.336386\n",
            "[Iter 21001] Loss 0.327187\n",
            "[Iter 21101] Loss 0.331575\n",
            "[Iter 21201] Loss 0.325685\n",
            "[Iter 21301] Loss 0.331042\n",
            "[Iter 21401] Loss 0.325552\n",
            "[Iter 21501] Loss 0.332107\n",
            "[Iter 21601] Loss 0.328525\n",
            "[Iter 21701] Loss 0.325958\n",
            "[Iter 21801] Loss 0.329820\n",
            "[Iter 21901] Loss 0.325533\n",
            "[Iter 22001] Loss 0.325710\n",
            "[Iter 22101] Loss 0.325369\n",
            "[Iter 22201] Loss 0.324627\n",
            "[Iter 22301] Loss 0.329826\n",
            "[Iter 22401] Loss 0.331985\n",
            "[Iter 22501] Loss 0.324544\n",
            "[Iter 22601] Loss 0.335592\n",
            "[Iter 22701] Loss 0.324253\n",
            "[Iter 22801] Loss 0.323434\n",
            "[Iter 22901] Loss 0.333274\n",
            "[Iter 23001] Loss 0.327426\n",
            "[Iter 23101] Loss 0.326761\n",
            "[Iter 23201] Loss 0.325118\n",
            "Epoch :  9\n",
            "[Iter 23301] Loss 0.202903\n",
            "[Iter 23401] Loss 0.329558\n",
            "[Iter 23501] Loss 0.330878\n",
            "[Iter 23601] Loss 0.322407\n",
            "[Iter 23701] Loss 0.328921\n",
            "[Iter 23801] Loss 0.328268\n",
            "[Iter 23901] Loss 0.328204\n",
            "[Iter 24001] Loss 0.324398\n",
            "[Iter 24101] Loss 0.324303\n",
            "[Iter 24201] Loss 0.329130\n",
            "[Iter 24301] Loss 0.328889\n",
            "[Iter 24401] Loss 0.324512\n",
            "[Iter 24501] Loss 0.325242\n",
            "[Iter 24601] Loss 0.327311\n",
            "[Iter 24701] Loss 0.327351\n",
            "[Iter 24801] Loss 0.327250\n",
            "[Iter 24901] Loss 0.319959\n",
            "[Iter 25001] Loss 0.330965\n",
            "[Iter 25101] Loss 0.319661\n",
            "[Iter 25201] Loss 0.331782\n",
            "[Iter 25301] Loss 0.324813\n",
            "[Iter 25401] Loss 0.324985\n",
            "[Iter 25501] Loss 0.326556\n",
            "[Iter 25601] Loss 0.324418\n",
            "[Iter 25701] Loss 0.329545\n",
            "[Iter 25801] Loss 0.324738\n",
            "Epoch :  10\n",
            "[Iter 25901] Loss 0.258883\n",
            "[Iter 26001] Loss 0.326272\n",
            "[Iter 26101] Loss 0.321154\n",
            "[Iter 26201] Loss 0.332191\n",
            "[Iter 26301] Loss 0.322945\n",
            "[Iter 26401] Loss 0.324533\n",
            "[Iter 26501] Loss 0.323534\n",
            "[Iter 26601] Loss 0.325896\n",
            "[Iter 26701] Loss 0.323548\n",
            "[Iter 26801] Loss 0.327451\n",
            "[Iter 26901] Loss 0.323913\n",
            "[Iter 27001] Loss 0.322877\n",
            "[Iter 27101] Loss 0.326327\n",
            "[Iter 27201] Loss 0.331776\n",
            "[Iter 27301] Loss 0.319692\n",
            "[Iter 27401] Loss 0.320993\n",
            "[Iter 27501] Loss 0.327839\n",
            "[Iter 27601] Loss 0.327018\n",
            "[Iter 27701] Loss 0.324014\n",
            "[Iter 27801] Loss 0.329479\n",
            "[Iter 27901] Loss 0.322804\n",
            "[Iter 28001] Loss 0.324618\n",
            "[Iter 28101] Loss 0.322540\n",
            "[Iter 28201] Loss 0.322246\n",
            "[Iter 28301] Loss 0.324330\n",
            "[Iter 28401] Loss 0.325646\n",
            "Epoch :  11\n",
            "[Iter 28501] Loss 0.320811\n",
            "[Iter 28601] Loss 0.321311\n",
            "[Iter 28701] Loss 0.321013\n",
            "[Iter 28801] Loss 0.327053\n",
            "[Iter 28901] Loss 0.324961\n",
            "[Iter 29001] Loss 0.324926\n",
            "[Iter 29101] Loss 0.322757\n",
            "[Iter 29201] Loss 0.326021\n",
            "[Iter 29301] Loss 0.323296\n",
            "[Iter 29401] Loss 0.321846\n",
            "[Iter 29501] Loss 0.323418\n",
            "[Iter 29601] Loss 0.320825\n",
            "[Iter 29701] Loss 0.323387\n",
            "[Iter 29801] Loss 0.325382\n",
            "[Iter 29901] Loss 0.322742\n",
            "[Iter 30001] Loss 0.324775\n",
            "[Iter 30101] Loss 0.323402\n",
            "[Iter 30201] Loss 0.319922\n",
            "[Iter 30301] Loss 0.322794\n",
            "[Iter 30401] Loss 0.324313\n",
            "[Iter 30501] Loss 0.322412\n",
            "[Iter 30601] Loss 0.323046\n",
            "[Iter 30701] Loss 0.323866\n",
            "[Iter 30801] Loss 0.323356\n",
            "[Iter 30901] Loss 0.324056\n",
            "Epoch :  12\n",
            "[Iter 31001] Loss 0.051785\n",
            "[Iter 31101] Loss 0.322465\n",
            "[Iter 31201] Loss 0.327362\n",
            "[Iter 31301] Loss 0.321081\n",
            "[Iter 31401] Loss 0.320018\n",
            "[Iter 31501] Loss 0.324485\n",
            "[Iter 31601] Loss 0.321608\n",
            "[Iter 31701] Loss 0.324433\n",
            "[Iter 31801] Loss 0.320952\n",
            "[Iter 31901] Loss 0.320662\n",
            "[Iter 32001] Loss 0.322357\n",
            "[Iter 32101] Loss 0.319798\n",
            "[Iter 32201] Loss 0.319305\n",
            "[Iter 32301] Loss 0.323598\n",
            "[Iter 32401] Loss 0.322714\n",
            "[Iter 32501] Loss 0.317912\n",
            "[Iter 32601] Loss 0.325392\n",
            "[Iter 32701] Loss 0.317546\n",
            "[Iter 32801] Loss 0.325115\n",
            "[Iter 32901] Loss 0.327086\n",
            "[Iter 33001] Loss 0.320003\n",
            "[Iter 33101] Loss 0.323381\n",
            "[Iter 33201] Loss 0.322592\n",
            "[Iter 33301] Loss 0.320561\n",
            "[Iter 33401] Loss 0.322451\n",
            "[Iter 33501] Loss 0.320896\n",
            "Epoch :  13\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-23-b8f44182dead>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_fcn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreply_vocab_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_epochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m50\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-a2ea79c7294e>\u001b[0m in \u001b[0;36mtrain_fcn\u001b[0;34m(model, data, reply_vocab_size, batch_size, num_epochs, lr, print_every)\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0mout_reshaped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mreply_vocab_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m             \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# no bos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m             \u001b[0mtarget\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     30\u001b[0m             \u001b[0;31m#print(batch.reply[:,1:])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_reshaped\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dIP5HE46Lrtq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loss = np.loadtxt(\"{}_train_loss.csv\".format('LSTM'))\n",
        "plt.title(\"Train Loss\");\n",
        "plt.plot(train_loss, label=\"Train\");\n",
        "plt.xlabel(\"Iters\");\n",
        "plt.ylabel(\"Loss\");\n",
        "plt.legend(loc='best');\n",
        "plt.show();"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRPK1Ni9_2Mw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sample_sequence(model, context, text_field, label_field, max_len=100, temperature=0.1):\n",
        "    generated_sequence = \"\"\n",
        "    \n",
        "    data_inp = [\"<BOS>\"] + list(context) + [\"<EOS>\"];\n",
        "    data_inp_indices = [text_field.vocab.stoi[ch] for ch in data_inp];\n",
        "    data_inp_tensor = torch.Tensor(data_inp_indices).long().unsqueeze(0);\n",
        "    \n",
        "    inp = torch.Tensor([label_field.vocab.stoi[\"<BOS>\"]]).long()\n",
        "    hidden = None;\n",
        "    for p in range(max_len):\n",
        "        #print(inp)\n",
        "        output, hidden = model(data_inp_tensor, inp.unsqueeze(0), hidden)\n",
        "        #print(output)\n",
        "        #output = F.softmax(output, dim=2)\n",
        "        #print(torch.argmax(output, dim=2))\n",
        "        # Sample from the network as a multinomial distribution\n",
        "        output_dist = output.data.view(-1).div(temperature).exp()\n",
        "        #print(output_dist)\n",
        "        top_i = int(torch.multinomial(output_dist, 1)[0])\n",
        "        # Add predicted character to string and use as next input\n",
        "        predicted_char = label_field.vocab.itos[top_i]\n",
        "        \n",
        "        if predicted_char == \"<EOS>\":\n",
        "            break\n",
        "        generated_sequence += predicted_char       \n",
        "        inp = torch.Tensor([top_i]).long()\n",
        "    return generated_sequence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NiHLuSMDDN0i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def sample_sequence2(model, context, text_field, label_field, max_len=100, temperature=0.1):\n",
        "    generated_sequence = \"\"\n",
        "    \n",
        "    data_inp = [\"<BOS>\"] + list(context) + [\"<EOS>\"];\n",
        "    data_inp_indices = [text_field.vocab.stoi[ch] for ch in data_inp];\n",
        "    data_inp_tensor = torch.Tensor(data_inp_indices).long().unsqueeze(0);\n",
        "    \n",
        "    inp = torch.Tensor([label_field.vocab.stoi[\"<BOS>\"]]).long()\n",
        "    hidden = None;\n",
        "    for p in range(max_len):\n",
        "        #print(inp)\n",
        "        output, hidden = model(data_inp_tensor, inp.unsqueeze(0), hidden)\n",
        "        #print(output)\n",
        "        #output = F.softmax(output, dim=2)\n",
        "        #print(torch.argmax(output, dim=2))\n",
        "        # Sample from the network as a multinomial distribution\n",
        "        output = torch.reshape(output, (1, -1));\n",
        "        output_dist = F.softmax(output, dim=1)\n",
        "        #print(output_dist.shape)\n",
        "        #print(output_dist)\n",
        "        top_i = int(torch.argmax(output_dist, dim=1))\n",
        "        # Add predicted character to string and use as next input\n",
        "        predicted_char = label_field.vocab.itos[top_i]\n",
        "        \n",
        "        if predicted_char == \"<EOS>\":\n",
        "            break\n",
        "        generated_sequence += predicted_char       \n",
        "        inp = torch.Tensor([top_i]).long()\n",
        "    return generated_sequence"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V_veH1BXzpSX",
        "colab_type": "code",
        "outputId": "25b52606-70ff-46a1-c32a-8be46b135471",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "test_model = ChatBot(input_vocab_size, reply_vocab_size, encoder_layers = 2, generator_layers = 2)\n",
        "test_model.load_state_dict(torch.load('/content/gdrive/My Drive/Chatbot/MM_WEIGHTS_11'))\n",
        "test_model.eval()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ChatBot(\n",
              "  (encode_rnn): LSTM(95, 100, num_layers=2, batch_first=True)\n",
              "  (decode_rnn): LSTM(95, 100, num_layers=2, batch_first=True)\n",
              "  (fcnn): Linear(in_features=100, out_features=95, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_EnG6FpIRkq",
        "colab_type": "code",
        "outputId": "7a1a7770-6190-453c-aa55-a6ed5d7179bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "output1 = sample_sequence(test_model, \"fcc or congress?\", text_field, label_field, temperature = 0.3)\n",
        "output2 = sample_sequence2(test_model, \"fcc or congress?\", text_field, label_field, temperature = 0)\n",
        "print(output1)\n",
        "print(output2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I dont know what they are able to start the same thing they dont have to be a service of the most pe\n",
            "I would have been a big part of the police of the police of the police of the police of the police o\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}