MMI_bidi.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import time\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import preprocess\n",
    "from DataLoader import DataLoader\n",
    "import model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "print(USE_CUDA)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data loading and processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of total words: 51805\n",
      "Dictionary cover ratio: 0.9796814762021151\n"
     ]
    }
   ],
   "source": [
    "# twitter dataset\n",
    "\n",
    "PAD_token = 0\n",
    "SOS_token = 1\n",
    "EOS_token = 2\n",
    "UNK_token = 3\n",
    "\n",
    "max_vocab_size = 20000\n",
    "max_sen, min_sen = 14, 3\n",
    "unk_most = 2\n",
    "reverse_flag = 1  # reverse the input sequence order Sutskever et al., 2014\n",
    "inverse_flag = 0  # MMI bidirection: train P(T|S) by inversing source and target\n",
    "\n",
    "dataStat = preprocess.dataPreProcess('dataset/twitter.txt', max_vocab_size, max_sen, min_sen, unk_most, reverse_flag, inverse_flag) \n",
    "print(\"Number of total words:\", dataStat.numOfWords)\n",
    "\n",
    "wordCount = sorted(dataStat.word2cnt.values(), reverse=True)\n",
    "print(\"Dictionary cover ratio:\", sum(wordCount[:max_vocab_size-4]) / sum(wordCount))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad or cut input and output sentence\n",
    "\n",
    "def padding(source, maxLen):\n",
    "    return np.pad(source[:maxLen],(0,max(0,maxLen-len(source))),'constant')\n",
    "\n",
    "input_max_len, output_max_len = 15, 15\n",
    "\n",
    "pairsNum = len(dataStat.pairsInd)\n",
    "pairsLength = np.array([[len(l[0]), len(l[1])] for l in dataStat.pairsInd])\n",
    "upperLength = np.concatenate((np.ones((pairsNum,1), dtype=int)*input_max_len, \n",
    "                              np.ones((pairsNum,1), dtype=int)*output_max_len), axis=1)\n",
    "pairsLength = np.minimum(pairsLength,upperLength)\n",
    "pairsAligned = np.array([np.concatenate((padding(l[0], input_max_len), \n",
    "                                              padding(l[1], output_max_len))) for l in dataStat.pairsInd])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training pairs:  104010\n",
      "Total develop pairs:  5778\n",
      "Total test pairs:  5779\n"
     ]
    }
   ],
   "source": [
    "train_type = 'resume'\n",
    "\n",
    "ratios = [0.90, 0.05, 0.05]\n",
    "pairsNumTrain, pairsNumDeve = int(ratios[0]*pairsNum), int(ratios[1]*pairsNum)\n",
    "pairsNumTest = pairsNum - pairsNumTrain - pairsNumDeve\n",
    "\n",
    "if train_type=='restart':\n",
    "    deve_idxes = np.random.choice(pairsNum, pairsNumDeve, replace=False)\n",
    "    test_idxes = np.random.choice(list(set(np.arange(pairsNum)).difference(set(deve_idxes))), pairsNumTest, replace=False)\n",
    "    train_idxes = np.array(list(set(np.arange(pairsNum)).difference(set(deve_idxes)).difference(set(test_idxes))))\n",
    "    np.save(\"parameter/pairsIdxesTriple.npy\", (train_idxes, deve_idxes, test_idxes))\n",
    "else:\n",
    "    train_idxes, deve_idxes, test_idxes = np.load( \"parameter/pairsIdxesTriple.npy\" )\n",
    "\n",
    "pairsAlignedTrain, pairsAlignedDeve, pairsAlignedTest = pairsAligned[train_idxes], pairsAligned[deve_idxes], pairsAligned[test_idxes]\n",
    "pairsLengthTrain, pairsLengthDeve, pairsLengthTest = pairsLength[train_idxes], pairsLength[deve_idxes], pairsLength[test_idxes]\n",
    "    \n",
    "print(\"Total training pairs: \",pairsNumTrain)\n",
    "print(\"Total develop pairs: \",pairsNumDeve)\n",
    "print(\"Total test pairs: \",pairsNumTest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct Seq2Seq model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load forward and backward model\n",
    "\n",
    "# environment setup\n",
    "vocab_size = min(dataStat.numOfWords, max_vocab_size)\n",
    "hidden_size = 1000\n",
    "\n",
    "embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "rnnEncoder = model.Encoder(embedding, vocab_size,20,hidden_size, n_layers=2, bidirectional=False, variable_lengths=True)\n",
    "rnnDecoder = model.Decoder(embedding, vocab_size,1,hidden_size, n_layers=2)\n",
    "\n",
    "embedding_IN = nn.Embedding(vocab_size, hidden_size)\n",
    "rnnEncoder_IN = model.Encoder(embedding_IN, vocab_size,20,hidden_size, n_layers=2, bidirectional=False, variable_lengths=True)\n",
    "rnnDecoder_IN = model.Decoder(embedding_IN, vocab_size,1,hidden_size, n_layers=2)\n",
    "\n",
    "if train_type.lower()=='restart': pass\n",
    "elif train_type.lower()=='resume':\n",
    "    para_name = 'twitter_0518_35'\n",
    "    embedding.load_state_dict(torch.load('parameter/embeding_'+para_name+'.pt'))\n",
    "    rnnEncoder.load_state_dict(torch.load('parameter/encoder_'+para_name+'.pt'))\n",
    "    rnnDecoder.load_state_dict(torch.load('parameter/decoder_'+para_name+'.pt'))\n",
    "    para_name = '1_twitter_0518_40'\n",
    "    embedding_IN.load_state_dict(torch.load('parameter/embeding_'+para_name+'.pt'))\n",
    "    rnnEncoder_IN.load_state_dict(torch.load('parameter/encoder_'+para_name+'.pt'))\n",
    "    rnnDecoder_IN.load_state_dict(torch.load('parameter/decoder_'+para_name+'.pt'))\n",
    "    \n",
    "else: print(\"Please enter valid training type !\")\n",
    "\n",
    "if USE_CUDA:\n",
    "    rnnEncoder.cuda()\n",
    "    rnnDecoder.cuda()\n",
    "    rnnEncoder_IN.cuda()\n",
    "    rnnDecoder_IN.cuda()\n",
    "\n",
    "criterion = nn.NLLLoss(size_average=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration per epoch: 3250\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.0002\n",
    "optimizer_encoder = optim.Adam(rnnEncoder.parameters(), learning_rate)\n",
    "optimizer_decoder = optim.Adam(rnnDecoder.parameters(), learning_rate)\n",
    "\n",
    "# initialize dataloader\n",
    "batch_size = 32\n",
    "trainLoader = DataLoader(pairsAlignedTrain, pairsLengthTrain, input_max_len, output_max_len)\n",
    "trainLoader.reset(batch_size)\n",
    "\n",
    "deveLoader = DataLoader(pairsAlignedDeve, pairsLengthDeve, input_max_len, output_max_len)\n",
    "deveLoader.reset(1)\n",
    "\n",
    "testLoader = DataLoader(pairsAlignedTest, pairsLengthTest, input_max_len, output_max_len)\n",
    "testLoader.reset(1)\n",
    "\n",
    "print(\"iteration per epoch:\", int(pairsNumTrain/batch_size))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def geneMask(outputs_record, lengths):\n",
    "    batch_size = lengths.size(0)\n",
    "    # prepare\n",
    "    comp = torch.arange(output_max_len).view(-1,1)\n",
    "    if USE_CUDA: comp = comp.cuda()\n",
    "    comp_ex = comp.repeat(1,vocab_size).repeat(batch_size,1,1)\n",
    "    # generate\n",
    "    l_ex = lengths[:,1].view(batch_size,-1).repeat(1,vocab_size).view(batch_size,1,-1)\n",
    "    if USE_CUDA: l_ex = l_ex.type(torch.cuda.FloatTensor)\n",
    "    else: l_ex = l_ex.type(torch.FloatTensor)\n",
    "    mask = comp_ex < l_ex\n",
    "    if USE_CUDA: mask = mask.type(torch.cuda.FloatTensor)\n",
    "    else: mask = mask.type(torch.FloatTensor)\n",
    "    return torch.mul(mask, outputs_record)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def oneEpoch():\n",
    "\n",
    "    running_loss = 0\n",
    "\n",
    "    for batch_ind in range(int(pairsNum/batch_size)+1):\n",
    "    \n",
    "        # prepare mini-batch data\n",
    "        try:\n",
    "            inputs, targets, lengths = trainLoader.getMiniBatch()\n",
    "        except Exception as e:\n",
    "            # print('GG...')\n",
    "            break\n",
    "        else:\n",
    "            # print('Good!')\n",
    "            # Zero gradients of both optimizers\n",
    "            optimizer_encoder.zero_grad()\n",
    "            optimizer_decoder.zero_grad()\n",
    "\n",
    "            # encoding and decoding\n",
    "            inputs, targets = Variable(inputs), Variable(targets)\n",
    "            hid_init = rnnEncoder.init_hidden(batch_size)\n",
    "            out_enc, hid_enc = rnnEncoder.forward(inputs,lengths[:,0],hid_init)\n",
    "            \n",
    "            # teacher forcing\n",
    "            hid_dec = hid_enc#[:rnnDecoder.n_layers]\n",
    "\n",
    "            # SOS_token\n",
    "            in_dec = Variable(torch.LongTensor([dataStat.word2ind['SOS']] * batch_size))\n",
    "            if USE_CUDA: in_dec = in_dec.cuda()\n",
    "            outputs_record, hid_dec = rnnDecoder.forward(in_dec.view(batch_size,-1),hid_dec)\n",
    "            # continue\n",
    "            for i in range(output_max_len-1):\n",
    "                out_dec, hid_dec = rnnDecoder.forward(targets[:,i].view(batch_size,-1),hid_dec)\n",
    "                outputs_record = torch.cat((outputs_record, out_dec), 1)\n",
    "\n",
    "            outputs_mask = geneMask(outputs_record, lengths)\n",
    "            loss = criterion(torch.transpose(outputs_mask,1,2), targets)\n",
    "            # print(loss)\n",
    "            \n",
    "            loss.backward()\n",
    "            optimizer_encoder.step()\n",
    "            optimizer_decoder.step()\n",
    "            \n",
    "            running_loss += float(loss)\n",
    "            \n",
    "            if (batch_ind+1)%1000 == 0:\n",
    "                print(\"iteration\", batch_ind+1, \" ---- running loss:\", running_loss/batch_ind)\n",
    "            \n",
    "    # print('running_loss:',running_loss)\n",
    "    return running_loss/batch_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def savePara(epoch):\n",
    "    para_name = 'twitter_0518_'+str(epoch)\n",
    "    torch.save(embedding.state_dict(),'parameter/embeding_'+para_name+'.pt')\n",
    "    torch.save(rnnEncoder.state_dict(),'parameter/encoder_'+para_name+'.pt')\n",
    "    torch.save(rnnDecoder.state_dict(),'parameter/decoder_'+para_name+'.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin training...\n",
      "Fri May 18 13:48:28 2018\n",
      "iteration 1000  ---- running loss: 0.7764374853015781\n",
      "iteration 2000  ---- running loss: 0.7936298665313377\n",
      "iteration 3000  ---- running loss: 0.8099324350636894\n",
      "Epoch: 1 \tLoss: 0.8131477450590867\n",
      "Fri May 18 13:53:55 2018\n",
      "iteration 1000  ---- running loss: 0.7073472350447982\n",
      "iteration 2000  ---- running loss: 0.7228136096315542\n",
      "iteration 3000  ---- running loss: 0.7357599485632021\n",
      "Epoch: 2 \tLoss: 0.7386495542526245\n",
      "Fri May 18 13:59:20 2018\n",
      "iteration 1000  ---- running loss: 0.6406484229547961\n",
      "iteration 2000  ---- running loss: 0.6553890522030844\n",
      "iteration 3000  ---- running loss: 0.6696273117869963\n",
      "Epoch: 3 \tLoss: 0.6734025310736436\n",
      "Fri May 18 14:04:46 2018\n",
      "iteration 1000  ---- running loss: 0.5867972803545428\n",
      "iteration 2000  ---- running loss: 0.597785772011243\n",
      "iteration 3000  ---- running loss: 0.6119136117962688\n",
      "Epoch: 4 \tLoss: 0.615221876676266\n",
      "Fri May 18 14:10:12 2018\n",
      "iteration 1000  ---- running loss: 0.5322195835657664\n",
      "iteration 2000  ---- running loss: 0.5470120981521998\n",
      "iteration 3000  ---- running loss: 0.5607763142158843\n",
      "Epoch: 5 \tLoss: 0.5638911216900899\n",
      "Fri May 18 14:15:38 2018\n",
      "iteration 1000  ---- running loss: 0.48991997022409217\n",
      "iteration 2000  ---- running loss: 0.5011361236420794\n",
      "iteration 3000  ---- running loss: 0.5149129932027692\n",
      "Epoch: 6 \tLoss: 0.517942050255262\n",
      "Fri May 18 14:21:05 2018\n",
      "iteration 1000  ---- running loss: 0.44768989658928493\n",
      "iteration 2000  ---- running loss: 0.4602323704626991\n",
      "iteration 3000  ---- running loss: 0.474189354016726\n",
      "Epoch: 7 \tLoss: 0.4770525205868941\n",
      "Fri May 18 14:26:32 2018\n",
      "iteration 1000  ---- running loss: 0.41324694787298477\n",
      "iteration 2000  ---- running loss: 0.42459340264583717\n",
      "iteration 3000  ---- running loss: 0.43749470045344757\n",
      "Epoch: 8 \tLoss: 0.4401566848663183\n",
      "Fri May 18 14:31:58 2018\n",
      "iteration 1000  ---- running loss: 0.38189873534876545\n",
      "iteration 2000  ---- running loss: 0.3933874129384324\n",
      "iteration 3000  ---- running loss: 0.4044909343178092\n",
      "Epoch: 9 \tLoss: 0.407024121761322\n",
      "Fri May 18 14:37:25 2018\n",
      "iteration 1000  ---- running loss: 0.3491632898708244\n",
      "iteration 2000  ---- running loss: 0.3612826618553699\n",
      "iteration 3000  ---- running loss: 0.37394261578791693\n",
      "Epoch: 10 \tLoss: 0.37677874757693364\n",
      "Fri May 18 14:42:52 2018\n",
      "iteration 1000  ---- running loss: 0.3265627006272057\n",
      "iteration 2000  ---- running loss: 0.33498411603633493\n",
      "iteration 3000  ---- running loss: 0.34613183506451756\n",
      "Epoch: 11 \tLoss: 0.34866021879819725\n",
      "Fri May 18 14:48:20 2018\n",
      "iteration 1000  ---- running loss: 0.3011653171525942\n",
      "iteration 2000  ---- running loss: 0.3102537785442905\n",
      "iteration 3000  ---- running loss: 0.3210819805958304\n",
      "Epoch: 12 \tLoss: 0.3237495224017363\n",
      "Fri May 18 14:53:46 2018\n",
      "iteration 1000  ---- running loss: 0.27587192127475507\n",
      "iteration 2000  ---- running loss: 0.28768844781755626\n",
      "iteration 3000  ---- running loss: 0.2977851469761612\n",
      "Epoch: 13 \tLoss: 0.300586718586775\n",
      "Fri May 18 14:59:13 2018\n",
      "iteration 1000  ---- running loss: 0.26020703335722406\n",
      "iteration 2000  ---- running loss: 0.2673546939551562\n",
      "iteration 3000  ---- running loss: 0.27730924378994826\n",
      "Epoch: 14 \tLoss: 0.27955965886666223\n",
      "Fri May 18 15:04:39 2018\n",
      "iteration 1000  ---- running loss: 0.23777974218696923\n",
      "iteration 2000  ---- running loss: 0.24809334893385251\n",
      "iteration 3000  ---- running loss: 0.25828283468696744\n",
      "Epoch: 15 \tLoss: 0.26072940016251345\n",
      "Fri May 18 15:10:10 2018\n"
     ]
    }
   ],
   "source": [
    "print(\"Begin training...\")\n",
    "print(time.asctime( time.localtime(time.time()) ))\n",
    "\n",
    "for i in range(10):\n",
    "    trainLoader.reset(batch_size)\n",
    "    loss = oneEpoch()\n",
    "    if (i+1)%1==0:\n",
    "        print('Epoch:', i+1, '\\tLoss:',loss)\n",
    "        print(time.asctime( time.localtime(time.time()) ))\n",
    "    if (i+1)%5==0:\n",
    "        savePara(i+1+30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute posterior probability: P(S|T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute MMI-bidi P(S|T)\n",
    "def postProb(inputs, targets, lengths):\n",
    "    \n",
    "    batch_size = lengths.size(0)\n",
    "    # encoding and decoding\n",
    "    hid_init = rnnEncoder_IN.init_hidden(batch_size)\n",
    "    out_enc, hid_enc = rnnEncoder_IN.forward(inputs,lengths[:,0],hid_init)\n",
    "\n",
    "    # teacher forcing\n",
    "    hid_dec = hid_enc#[:rnnDecoder.n_layers]\n",
    "\n",
    "    # SOS_token\n",
    "    in_dec = Variable(torch.LongTensor([dataStat.word2ind['SOS']] * batch_size))\n",
    "    if USE_CUDA: in_dec = in_dec.cuda()\n",
    "    outputs_record, hid_dec = rnnDecoder_IN.forward(in_dec.view(batch_size,-1),hid_dec)\n",
    "    # continue\n",
    "    for i in range(output_max_len-1):\n",
    "        out_dec, hid_dec = rnnDecoder_IN.forward(targets[:,i].view(batch_size,-1),hid_dec)\n",
    "        outputs_record = torch.cat((outputs_record, out_dec), 1)\n",
    "    \n",
    "    pp = []\n",
    "    for b in range(batch_size):\n",
    "        pp.append( sum([outputs_record[b][i][int(targets[b][i])] for i in range(lengths[b][1])]) )\n",
    "    return pp\n",
    "\n",
    "    \n",
    "# trainLoader.reset(2)\n",
    "# inputs, targets, lengths = trainLoader.getMiniBatch()\n",
    "# print('Good!')\n",
    "# prob = postProb(inputs, targets, lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def interChange(inputs, targets, lengths):\n",
    "    input_len, output_len = len(inputs[0]), len(targets[0])\n",
    "    np_inputs, np_targets, np_lengths = inputs.cpu().numpy(), targets.cpu().numpy(), lengths.cpu().numpy()\n",
    "    for i in range(len(np_inputs)):\n",
    "        np_inputs[i] = padding(np.append(np_inputs[i][:np_lengths[i][0]-1][::-1],np.array([EOS_token])), input_len)\n",
    "        np_targets[i] = padding(np.append(np_targets[i][:np_lengths[i][1]-1][::-1],np.array([EOS_token])), output_len)\n",
    "        np_lengths[i] = np_lengths[i][::-1]\n",
    "    inputs_tensor = torch.cuda.LongTensor(np_inputs) if USE_CUDA else torch.LongTensor(np_inputs)\n",
    "    targets_tensor = torch.cuda.LongTensor(np_targets) if USE_CUDA else torch.LongTensor(np_targets)\n",
    "    lengths_tensor = torch.cuda.LongTensor(np_lengths) if USE_CUDA else torch.LongTensor(np_lengths)\n",
    "    return targets_tensor, inputs_tensor, lengths_tensor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evalute by BLEU and distinct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# translate into natural language\n",
    "\n",
    "def showResult(ind_seq, reverse=False):\n",
    "    token_list = []\n",
    "    for i in ind_seq:\n",
    "        if i == dataStat.word2ind['EOS']: break\n",
    "        token_list.append(dataStat.ind2word[i])\n",
    "    return ' '.join(token_list[::-1]) if reverse else  ' '.join(token_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Sentence:\n",
    "    def __init__(self, decoder_hidden, last_idx=SOS_token, sentence_idxes=[], sentence_scores=[]):\n",
    "        if(len(sentence_idxes) != len(sentence_scores)):\n",
    "            raise ValueError(\"length of indexes and scores should be the same\")\n",
    "        self.decoder_hidden = decoder_hidden\n",
    "        self.last_idx = last_idx\n",
    "        self.sentence_idxes =  sentence_idxes\n",
    "        self.sentence_scores = sentence_scores\n",
    "\n",
    "    def getScore(self, mode='sum', gamma=-0.2):\n",
    "        if len(self.sentence_scores) == 0:\n",
    "            return torch.cuda.FloatTensor(-999) if USE_CUDA else torch.FloatTensor(-999)\n",
    "        if mode=='avg':\n",
    "            res = sum(self.sentence_scores) / len(self.sentence_scores)\n",
    "        else:\n",
    "            res = sum(self.sentence_scores) + gamma*len(self.sentence_scores)\n",
    "        return res\n",
    "\n",
    "    def addTopk(self, topi, topv, decoder_hidden, beam_size, voc):\n",
    "        terminates, sentences = [], []\n",
    "        \n",
    "        topi, topv = topi[0], topv[0]  # get data out of batch\n",
    "        \n",
    "        for i in range(beam_size):\n",
    "            if topi[0][i] == EOS_token:\n",
    "                terminates.append(([int(idx) for idx in self.sentence_idxes] + [EOS_token],\n",
    "                                   self.getScore())) # tuple(word_list, score_float)\n",
    "                continue\n",
    "            idxes = self.sentence_idxes[:] # pass by value\n",
    "            scores = self.sentence_scores[:] # pass by value\n",
    "            idxes.append(topi[0][i])\n",
    "            scores.append(topv[0][i])\n",
    "            sentences.append(Sentence(decoder_hidden, topi[0][i], idxes, scores))\n",
    "        return terminates, sentences\n",
    "\n",
    "    def toWordScore(self, voc):\n",
    "        words = []\n",
    "        for i in range(len(self.sentence_idxes)):\n",
    "            if self.sentence_idxes[i] == EOS_token:\n",
    "                words.append(EOS_token)\n",
    "            else:\n",
    "                words.append(int(self.sentence_idxes[i]))\n",
    "        if self.sentence_idxes[-1] != EOS_token:\n",
    "            words.append(EOS_token)\n",
    "        return (words, self.getScore())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topOneDecode(decoder, decoder_hidden, stat, max_length=output_max_len):\n",
    "    \n",
    "    decoder_input = torch.LongTensor([SOS_token]).view(1,-1)\n",
    "    if USE_CUDA: \n",
    "        decoder_input = decoder_input.cuda()\n",
    "        decoder_hidden = decoder_hidden.cuda()\n",
    "\n",
    "    decoded_words = []\n",
    "\n",
    "    for di in range(max_length):\n",
    "\n",
    "        decoder_output, decoder_hidden = decoder(decoder_input, decoder_hidden)\n",
    "        topv, topi = decoder_output.topk(1)\n",
    "        ni = topi[0][0]\n",
    "        if ni == EOS_token:\n",
    "            decoded_words.append(ni.item())\n",
    "            break\n",
    "        else:\n",
    "            decoded_words.append(ni.item())\n",
    "\n",
    "        decoder_input = torch.LongTensor([[ni]])\n",
    "        if USE_CUDA: decoder_input = decoder_input.cuda()\n",
    "\n",
    "    return decoded_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beamDecode(decoder, decoder_hidden, voc, beam_size, max_length=output_max_len):\n",
    "    terminal_sentences, prev_top_sentences, next_top_sentences = [], [], []\n",
    "    prev_top_sentences.append(Sentence(decoder_hidden))\n",
    "    for _ in range(max_length-1):\n",
    "        for sentence in prev_top_sentences:\n",
    "            decoder_input = torch.LongTensor([[sentence.last_idx]])\n",
    "            if USE_CUDA: decoder_input = decoder_input.cuda()\n",
    "            decoder_output, decoder_hidden = decoder(decoder_input, sentence.decoder_hidden)\n",
    "            topv, topi = decoder_output.topk(beam_size)\n",
    "            term, top = sentence.addTopk(topi, topv, decoder_hidden, beam_size, voc)\n",
    "            terminal_sentences.extend(term)\n",
    "            next_top_sentences.extend(top)\n",
    "            \n",
    "        next_top_sentences.sort(key=lambda s: s.getScore(), reverse=True)\n",
    "        prev_top_sentences = next_top_sentences[:beam_size]\n",
    "        next_top_sentences = []\n",
    "\n",
    "    terminal_sentences += [sentence.toWordScore(voc) for sentence in prev_top_sentences]\n",
    "    terminal_sentences.sort(key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    n = min(len(terminal_sentences), 128)  # N-best list\n",
    "    return terminal_sentences[:n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate(input, length, encoder, decoder, beam_size, lamda, verbose=False):\n",
    "    # encoding and decoding\n",
    "    hid_init = encoder.init_hidden(batch_size = 1)\n",
    "    out_enc, hid_enc = encoder.forward(input.view(1,-1),length.view(1),hid_init)\n",
    "    hid_dec = hid_enc#[:rnnDecoder.n_layers]\n",
    "    \n",
    "    if beam_size==1:\n",
    "        path = topOneDecode(decoder, hid_dec, dataStat, max_length=15)  # return path in list\n",
    "        return path\n",
    "    else:\n",
    "        path_beam = beamDecode(decoder, hid_dec, dataStat, beam_size)  # return list of tuples: (path, score)\n",
    "        path_beam.sort(key=lambda x:len(x[0]),reverse=True)\n",
    "        \n",
    "        # apply MMI-bidi\n",
    "        batch_size = len(path_beam)\n",
    "        targets_np = np.array([padding(path, output_max_len) for path,_ in path_beam])\n",
    "        lengths_np = np.array([[int(length), len(path)] for path,_ in path_beam])\n",
    "        \n",
    "        inputs_tensor = input.view(1,-1).repeat(batch_size,1)\n",
    "        targets_tensor = torch.cuda.LongTensor(targets_np) if USE_CUDA else torch.LongTensor(targets_np)\n",
    "        lengths_tensor = torch.cuda.LongTensor(lengths_np) if USE_CUDA else torch.LongTensor(lengths_np)\n",
    "        \n",
    "        inputs_flip, targets_flip, lengths_flip = interChange(inputs_tensor, targets_tensor, lengths_tensor)\n",
    "        post_score = postProb(inputs_flip, targets_flip, lengths_flip)\n",
    "        path_beam = [(path_beam[i][0],(1-lamda)*path_beam[i][1]+lamda*post_score[i]) for i in range(len(path_beam))]\n",
    "        path_beam.sort(key=lambda x:x[1],reverse=True)\n",
    "        \n",
    "        if verbose:\n",
    "            for p in path_beam: print(float(p[1]), '\\t', showResult(p[0]))\n",
    "        return path_beam[0][0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateSample(encoder, decoder, beam_size=5, lamda=0.0, verbose=True, myQuery=''):\n",
    "    \n",
    "    if myQuery == '':\n",
    "        print(\"Blank Input\")\n",
    "        return -1\n",
    "    \n",
    "    # feed in customized tokens\n",
    "    sample_query = myQuery.lower()\n",
    "    sample_query_ind, _ = preprocess.encodePair(dataStat, (sample_query,'.'),reverse=True)\n",
    "    sample_query_tensor = torch.LongTensor([padding(sample_query_ind, input_max_len)])\n",
    "    sample_query_length = torch.LongTensor([len(sample_query_ind)])\n",
    "    if USE_CUDA: input, length, target = sample_query_tensor.cuda(), sample_query_length.cuda(), None\n",
    "    \n",
    "    trace = generate(input, length, encoder, decoder, beam_size, lamda, verbose=True)\n",
    "    if verbose:\n",
    "        print(\"Message:\\t\", showResult(input.data[0].cpu().numpy(), reverse=True))\n",
    "        print(\"Response:\\t\", showResult(trace))\n",
    "        if target is not None:\n",
    "            print(\"Teaching:\\t\", showResult(target.data[0].cpu().numpy()))\n",
    "    return 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.translate.bleu_score import corpus_bleu\n",
    "from nltk.translate.bleu_score import SmoothingFunction\n",
    "chencherry = SmoothingFunction()\n",
    "\n",
    "def evaluateCorpus(encoder, decoder, beam_size=1, lamda=0.0, loader=testLoader, display=False):\n",
    "    \n",
    "    loader.reset(1)\n",
    "    data_length = loader.dataLength\n",
    "    responses = []\n",
    "    total_score = 0\n",
    "    sample_length = min(2000, data_length)\n",
    "    \n",
    "    for i in range(sample_length):\n",
    "        inputs, targets, lengths = loader.getMiniBatch()\n",
    "        input, length, target = inputs, lengths[0][0], targets\n",
    "\n",
    "        trace = generate(input, length, encoder, decoder, beam_size, lamda)\n",
    "        responses.append(trace)\n",
    "                \n",
    "        length_ref = lengths[0][1]\n",
    "        references = [[target.data[0].tolist()[:int(length_ref)]]]\n",
    "        candidates = [trace]\n",
    "        score = corpus_bleu(references, candidates, smoothing_function=chencherry.method1)\n",
    "        total_score += score\n",
    "        \n",
    "        if display and (i+1)%int(sample_length/10)==0: print(\"complete\",int(100*(i+1)/sample_length),\"%\")\n",
    "        \n",
    "    return total_score/i, responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# distinct evaluation\n",
    "\n",
    "import nltk\n",
    "\n",
    "def distinctEval(all_paths):\n",
    "\n",
    "    response_ugm = set([])\n",
    "    response_bgm = set([])        \n",
    "    response_len = sum([len(p) for p in all_paths])\n",
    "\n",
    "    for path in all_paths:\n",
    "        for u in path:\n",
    "            response_ugm.add(u)\n",
    "        for b in list(nltk.bigrams(path)):\n",
    "            response_bgm.add(b)\n",
    "\n",
    "    print(\"total length of response:\", response_len)\n",
    "    print(\"distinct unigrams:\", len(response_ugm)/response_len)\n",
    "    print(\"distinct bigrams:\", len(response_bgm)/response_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-28.299510955810547 \t love it ! can t wait for the next one . .\n",
      "-28.70806121826172 \t love it ! can t wait for the next one with new UNK .\n",
      "-28.76465606689453 \t love it ! can t wait for the next one .\n",
      "-29.720705032348633 \t love it ! can t wait for the next one . . .\n",
      "-30.940441131591797 \t love it ! can t wait for the next one with their message .\n",
      "-33.03714370727539 \t love it ! can t wait to hear your UNK UNK\n",
      "-33.1236686706543 \t agreed . it s over there !\n",
      "-33.641258239746094 \t love it ! can t wait for your UNK at !\n",
      "-34.11371994018555 \t love it ! can t wait for the next one with new message .\n",
      "-34.264404296875 \t love it ! can t wait for the next one with new UNK\n",
      "-34.78605651855469 \t love it ! can t wait\n",
      "-34.79853057861328 \t agreed . it s over there\n",
      "-34.92169952392578 \t agreed . it s over there .\n",
      "-36.388023376464844 \t can t wait for it\n",
      "-37.20137023925781 \t love it ! can t wait to hear your UNK UNK reporting\n",
      "-37.800838470458984 \t can t wait for it well\n",
      "-38.074317932128906 \t love it ! can t wait for the next one . with three about\n",
      "-38.931461334228516 \t love it ! can t wait to hear your UNK reporting\n",
      "-39.915828704833984 \t agreed . it s over\n",
      "-40.61457824707031 \t love it ! can t wait to hear your about\n",
      "-41.540653228759766 \t true ! i love that actually haha\n",
      "-46.52647399902344 \t love it ! can t wait for the next one with their message\n",
      "-46.62197494506836 \t love it ! can t wait for the next one with new message here\n",
      "Message:\t seriously one of the best shows ! ! !\n",
      "Response:\t love it ! can t wait for the next one . .\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluateSample(rnnEncoder, rnnDecoder, 5, 0.5, True, 'seriously one of the best shows ! ! !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-11.428078651428223 \t of course love it\n",
      "-13.738347053527832 \t of course i love you\n",
      "-15.266695022583008 \t of course boo\n",
      "-16.15329933166504 \t of course i love you !\n",
      "-16.477169036865234 \t now of it i m glad it was an look .\n",
      "-16.86753273010254 \t of course love UNK\n",
      "-16.93350601196289 \t of course love it !\n",
      "-17.79831886291504 \t now of it i m like it can look it up it\n",
      "-17.813858032226562 \t of course your welcome\n",
      "-17.907814025878906 \t of course i love you ! !\n",
      "-17.964780807495117 \t now of it i m like it it can you look it up it\n",
      "-18.291221618652344 \t now of it i m like it it can you look at it ?\n",
      "-18.43475914001465 \t now of it i m like it it can you look it up\n",
      "-18.67047119140625 \t now of it i m like it it can you look it up this\n",
      "-19.173110961914062 \t of course your welcome !\n",
      "-19.234413146972656 \t of course love UNK i m UNK of course\n",
      "-19.29511260986328 \t now of it i m like it it can you look at it again\n",
      "-19.543359756469727 \t now of it i m glad it was an look !\n",
      "-19.93739128112793 \t now of it i m glad it was an amazing\n",
      "-20.907649993896484 \t now of it i m glad it was an look\n",
      "-20.99257469177246 \t of course love UNK i m UNK\n",
      "-20.993276596069336 \t of course love UNK i m UNK of course !\n",
      "-22.354145050048828 \t now of it i m like it it can you look at it UNK\n",
      "-22.693950653076172 \t now of it i m like it can look it up it b\n",
      "-23.887598037719727 \t of course love UNK i m UNK of course ! !\n",
      "-26.48532485961914 \t now of it i m like it can look it up it UNK\n",
      "-27.958515167236328 \t now of it i m like it can look it up don t\n",
      "-28.372480392456055 \t now of it i m like it can look it up don\n",
      "Message:\t thank you for your retweet\n",
      "Response:\t of course love it\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluateSample(rnnEncoder, rnnDecoder, 5, 0.5, True, 'thank you for your retweet')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyper-parameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May 20 10:26:13 2018\n",
      "lambda:\t 0\n",
      "0.5362942283961266 0.03836904777509308 0.03646343053341346\n",
      "total length of response: 10962\n",
      "distinct unigrams: 0.12324393358876118\n",
      "distinct bigrams: 0.4154351395730706\n",
      "total length of response: 10856\n",
      "distinct unigrams: 0.12186809137803979\n",
      "distinct bigrams: 0.4106484893146647\n",
      "Sun May 20 10:43:18 2018\n",
      "lambda:\t 0.2\n",
      "0.8723004851154688 0.03933534291751297 0.039010927313406565\n",
      "total length of response: 13268\n",
      "distinct unigrams: 0.10604461863129334\n",
      "distinct bigrams: 0.40284895990352726\n",
      "total length of response: 13255\n",
      "distinct unigrams: 0.10818559034326669\n",
      "distinct bigrams: 0.4055827989437948\n",
      "Sun May 20 11:00:21 2018\n",
      "lambda:\t 0.4\n",
      "0.8546722297900166 0.039246627489394934 0.03635357186474585\n",
      "total length of response: 14644\n",
      "distinct unigrams: 0.09990439770554493\n",
      "distinct bigrams: 0.4164162797049986\n",
      "total length of response: 14685\n",
      "distinct unigrams: 0.10160027238678924\n",
      "distinct bigrams: 0.40360912495743956\n",
      "Sun May 20 11:16:40 2018\n",
      "lambda:\t 0.6\n",
      "0.8686487922366823 0.03791820405699015 0.037242495826267326\n",
      "total length of response: 15351\n",
      "distinct unigrams: 0.10051462445443293\n",
      "distinct bigrams: 0.4174972314507198\n",
      "total length of response: 15648\n",
      "distinct unigrams: 0.09579498977505113\n",
      "distinct bigrams: 0.41545245398773006\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0,0.2,0.4,0.6]  # gamma = 0\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,5, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May 20 16:52:18 2018\n",
      "lambda:\t 0.1\n",
      "0.852224614722001 0.040576123123919484 0.03989233536631668\n",
      "total length of response: 12413\n",
      "distinct unigrams: 0.10859582695561106\n",
      "distinct bigrams: 0.40248126963667125\n",
      "total length of response: 12237\n",
      "distinct unigrams: 0.10942224401405573\n",
      "distinct bigrams: 0.4067990520552423\n",
      "Sun May 20 17:10:19 2018\n",
      "lambda:\t 0.3\n",
      "0.8579391756165962 0.03616957871798036 0.03882489577941158\n",
      "total length of response: 14538\n",
      "distinct unigrams: 0.1008391800797909\n",
      "distinct bigrams: 0.4087907552620718\n",
      "total length of response: 14715\n",
      "distinct unigrams: 0.09901461094121644\n",
      "distinct bigrams: 0.40706761807679237\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0.1,0.3]  # gamma = 0.0\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,5, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May 20 14:08:33 2018\n",
      "lambda:\t 0\n",
      "0.5180059415845227 0.03863957783386995 0.03834643839072519\n",
      "total length of response: 10468\n",
      "distinct unigrams: 0.12227741688956821\n",
      "distinct bigrams: 0.40513947267863964\n",
      "total length of response: 10329\n",
      "distinct unigrams: 0.1140478265078904\n",
      "distinct bigrams: 0.3913253945202827\n",
      "Sun May 20 14:26:32 2018\n",
      "lambda:\t 0.2\n",
      "0.846535550333517 0.04082734135081596 0.04208997728009065\n",
      "total length of response: 12281\n",
      "distinct unigrams: 0.1079716635453139\n",
      "distinct bigrams: 0.40070026870775993\n",
      "total length of response: 12237\n",
      "distinct unigrams: 0.11048459589768735\n",
      "distinct bigrams: 0.40786140393887393\n",
      "Sun May 20 14:44:39 2018\n",
      "lambda:\t 0.4\n",
      "0.8461584355910389 0.0388449260833696 0.0401199617052644\n",
      "total length of response: 13552\n",
      "distinct unigrams: 0.1016086186540732\n",
      "distinct bigrams: 0.4042207792207792\n",
      "total length of response: 13798\n",
      "distinct unigrams: 0.09856500942165532\n",
      "distinct bigrams: 0.3992607624293376\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0,0.2,0.4] # gamma = -0.2\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,5, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May 20 19:07:30 2018\n",
      "lambda:\t 0\n",
      "0.4812283438774192 0.03668723277002475 0.03664434406796168\n",
      "total length of response: 10175\n",
      "distinct unigrams: 0.11921375921375921\n",
      "distinct bigrams: 0.4048157248157248\n",
      "total length of response: 10121\n",
      "distinct unigrams: 0.11915818595000494\n",
      "distinct bigrams: 0.3964035174389882\n",
      "Sun May 20 19:24:11 2018\n",
      "lambda:\t 0.1\n",
      "0.7663262347786298 0.04041385453447268 0.040433159341322335\n",
      "total length of response: 10946\n",
      "distinct unigrams: 0.10871551251598757\n",
      "distinct bigrams: 0.3868079663804129\n",
      "total length of response: 11000\n",
      "distinct unigrams: 0.10718181818181818\n",
      "distinct bigrams: 0.38736363636363635\n",
      "Sun May 20 19:40:52 2018\n",
      "lambda:\t 0.3\n",
      "0.8408575209564628 0.04122847407251038 0.04197858564967113\n",
      "total length of response: 13069\n",
      "distinct unigrams: 0.10306832963501415\n",
      "distinct bigrams: 0.40477465758665543\n",
      "total length of response: 13112\n",
      "distinct unigrams: 0.10295912141549725\n",
      "distinct bigrams: 0.4044386821232459\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0,0.1,0.3] # gamma = -0.4\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,5, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun May 20 21:06:41 2018\n",
      "lambda:\t 0.2\n",
      "0.8291092543628275 0.04064549432381752 0.04014092327730664\n",
      "total length of response: 12028\n",
      "distinct unigrams: 0.10583638177585633\n",
      "distinct bigrams: 0.3937479215164616\n",
      "total length of response: 12051\n",
      "distinct unigrams: 0.10970043979752718\n",
      "distinct bigrams: 0.4005476723923326\n",
      "Sun May 20 21:23:43 2018\n",
      "lambda:\t 0.4\n",
      "0.8664403552459563 0.036505805755905216 0.04140470640171562\n",
      "total length of response: 14277\n",
      "distinct unigrams: 0.098550115570498\n",
      "distinct bigrams: 0.40526721299992996\n",
      "total length of response: 14288\n",
      "distinct unigrams: 0.10029395296752519\n",
      "distinct bigrams: 0.41097424412094063\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0.2,0.4] # gamma = -0.4\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,5, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 5, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon May 21 00:27:26 2018\n",
      "lambda:\t 0\n",
      "0.8748437645375782 0.03542231429482811 0.041545325818199495\n",
      "total length of response: 15681\n",
      "distinct unigrams: 0.11453351189337414\n",
      "distinct bigrams: 0.41617243798227155\n",
      "total length of response: 15711\n",
      "distinct unigrams: 0.11297816816243396\n",
      "distinct bigrams: 0.41696900260963654\n",
      "Mon May 21 01:39:39 2018\n",
      "lambda:\t 0.1\n",
      "0.886314003481408 0.03865351265139616 0.03854682356666949\n",
      "total length of response: 16250\n",
      "distinct unigrams: 0.10504615384615384\n",
      "distinct bigrams: 0.4083076923076923\n",
      "total length of response: 15929\n",
      "distinct unigrams: 0.10408688555464875\n",
      "distinct bigrams: 0.416347542218595\n",
      "Mon May 21 02:51:34 2018\n",
      "lambda:\t 0.2\n"
     ]
    }
   ],
   "source": [
    "# hyper-prrameter tuning\n",
    "lamda_list = [0,0.1,0.2,0.4] # gamma = 0.4, N-best list = 128\n",
    "score_list = [[],[],[]]\n",
    "\n",
    "for lamda in lamda_list:\n",
    "    print(time.asctime( time.localtime(time.time()) ))\n",
    "    print('lambda:\\t', lamda)\n",
    "    score_train,_ = evaluateCorpus(rnnEncoder, rnnDecoder,10, lamda, loader=trainLoader, display=0)\n",
    "    score_deve,paths_deve = evaluateCorpus(rnnEncoder, rnnDecoder, 10, lamda, loader=deveLoader, display=0)\n",
    "    score_test,paths_test = evaluateCorpus(rnnEncoder, rnnDecoder, 10, lamda, loader=testLoader, display=0)\n",
    "    score_list[0].append(score_train)\n",
    "    score_list[1].append(score_deve)\n",
    "    score_list[2].append(score_test)\n",
    "    print(score_train, score_deve, score_test)\n",
    "    distinctEval(paths_deve)\n",
    "    distinctEval(paths_test)\n",
    "\n",
    "# score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}