End-to-end_memory_network-class.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pickle\n",
    "import preprocessing\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class EndToEndNetwork:\n",
    "    def __init__(self, vocabluary_size,  num_hops = 2, batch_size = 100, validation_size = 300, sentence_input_embed_dim = 20, sentence_output_embed_dim = 20):\n",
    "        self.num_hops = num_hops\n",
    "        self.question_embed_dim = sentence_input_embed_dim\n",
    "        self.sentence_input_embed_dim = sentence_input_embed_dim\n",
    "        self.sentence_ouput_embed_dim = sentence_output_embed_dim\n",
    "        self.vocabluary_size = vocabluary_size\n",
    "        self.memory_dim = sentence_output_embed_dim\n",
    "        self.batch_size = batch_size\n",
    "        self.validation_size = validation_size\n",
    "        self.session = tf.Session()\n",
    "        self.A = []\n",
    "        self.B = tf.Variable(tf.truncated_normal([self.vocabluary_size, self.question_embed_dim], stddev = 0.1))\n",
    "        self.C = []\n",
    "        self.W = tf.Variable(tf.truncated_normal([self.sentence_ouput_embed_dim, self.vocabluary_size], stddev = 0.1))\n",
    "        for i in range(self.num_hops):\n",
    "            self.A.append(tf.Variable(tf.truncated_normal([self.vocabluary_size, self.sentence_input_embed_dim], stddev = 0.1)))\n",
    "            self.C.append(tf.Variable(tf.truncated_normal([self.vocabluary_size, self.sentence_ouput_embed_dim], stddev = 0.1)))\n",
    "        self.init = tf.initialize_all_variables()\n",
    "\n",
    "    def init_batch_gen(self, filename):\n",
    "        self.batch_gen = preprocessing.BatchGenerator(filename, self.batch_size, self.validation_size)\n",
    "        \n",
    "    def init_variables(self):\n",
    "        self.session.run(self.init)\n",
    "        \n",
    "    def build_linear_graph(self):\n",
    "        A = self.A\n",
    "        B = self.B\n",
    "        C = self.C\n",
    "        W = self.W\n",
    "        \n",
    "        batch_question = tf.placeholder(tf.int32, [None, None])\n",
    "        batch_sentences = tf.placeholder(tf.int32, [None, None, None])\n",
    "        batch_y_ = tf.placeholder(tf.float32, [None, None])\n",
    "\n",
    "        cross_entropy = tf.zeros([])\n",
    "\n",
    "        for i in range(self.batch_size):\n",
    "            question = batch_question[i,:]\n",
    "            sentences = batch_sentences[i,:,:]\n",
    "            y_ = tf.reshape(batch_y_[i,:], [1, self.vocabluary_size])\n",
    "\n",
    "            question_embed = tf.reshape(\n",
    "                tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(B, question), 0), \n",
    "                [1, self.question_embed_dim])\n",
    "\n",
    "\n",
    "            u = question_embed\n",
    "\n",
    "            for j in range(self.num_hops):\n",
    "                sentences_output_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(C[j], sentences), 1)\n",
    "                sentences_input_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(A[j], sentences), 1)\n",
    "                \n",
    "                p = tf.matmul(u, sentences_input_embed, transpose_b = True)\n",
    "                u = u + tf.reduce_sum(\n",
    "                    tf.mul(tf.transpose(p), sentences_output_embed), 0)\n",
    "\n",
    "            y = tf.nn.softmax(\n",
    "                tf.matmul(u, W))\n",
    "\n",
    "            cross_entropy = cross_entropy - tf.reduce_sum(y_ * tf.log(y))\n",
    "\n",
    "        train_step = tf.train.GradientDescentOptimizer(0.000001).minimize(cross_entropy)\n",
    "        \n",
    "        self.batch_placeholder = [batch_sentences, batch_question, batch_y_]\n",
    "#        self.train_step = train_step\n",
    "        self.cross_entropy = cross_entropy\n",
    "    \n",
    "    \n",
    "    def build_graph(self):\n",
    "        #Variables\n",
    "        A = self.A\n",
    "        B = self.B\n",
    "        C = self.C\n",
    "        W = self.W\n",
    "\n",
    "        #Batch Graph\n",
    "        batch_question = tf.placeholder(tf.int32, [None, None])\n",
    "        batch_sentences = tf.placeholder(tf.int32, [None, None, None])\n",
    "        batch_y_ = tf.placeholder(tf.float32, [None, None])\n",
    "\n",
    "        cross_entropy = tf.zeros([])\n",
    "        \n",
    "        global_step = tf.Variable(0, trainable=False)\n",
    "        self.make_step = global_step.assign(global_step + 1)\n",
    "        starter_learning_rate = 0.01\n",
    "        learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 350, 0.5, staircase=False)\n",
    "\n",
    "        for i in range(self.batch_size):\n",
    "            question = batch_question[i,:]\n",
    "            sentences = batch_sentences[i,:,:]\n",
    "            y_ = tf.reshape(batch_y_[i,:], [1, self.vocabluary_size])\n",
    "\n",
    "            question_embed = tf.reshape(\n",
    "                tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(B, question), 0), \n",
    "                [1, self.question_embed_dim])\n",
    "\n",
    "\n",
    "            u = question_embed\n",
    "\n",
    "            for j in range(self.num_hops):\n",
    "                sentences_output_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(C[j], sentences), 1)\n",
    "                sentences_input_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(A[j], sentences), 1)\n",
    "                \n",
    "                p = tf.nn.softmax(\n",
    "                    tf.matmul(u, sentences_input_embed, transpose_b = True))\n",
    "                u = u + tf.reduce_sum(\n",
    "                    tf.mul(tf.transpose(p), sentences_output_embed), 0)\n",
    "\n",
    "            y = tf.nn.softmax(\n",
    "                tf.matmul(u, W))\n",
    "\n",
    "            cross_entropy = cross_entropy - tf.reduce_sum(y_ * tf.log(y))\n",
    "\n",
    "\n",
    "        self.learning_rate = learning_rate\n",
    "#         train_step = (\n",
    "#             tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy, global_step=global_step)\n",
    "#         )\n",
    "        \n",
    "        optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "        gvs = optimizer.compute_gradients(cross_entropy)\n",
    "        grads = tf.clip_by_global_norm([grad for grad, var in gvs], 40)\n",
    "        self.grad_before_clipping = grads[1]\n",
    "        self.grad_after_clipping = tf.clip_by_global_norm(grads[0], 1)[1]\n",
    "        capped_gvs = [(grads[0][i], gvs[i][1]) for i in range(len(gvs))]   # [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs]\n",
    "        self.app_grads = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "#        train_step = tf.train.AdagradOptimizer(0.01).minimize(cross_entropy)\n",
    "\n",
    "        #Validation Graph\n",
    "        validation_question = tf.placeholder(tf.int32, [None, None])\n",
    "        validation_sentences = tf.placeholder(tf.int32, [None, None, None])\n",
    "        validation_y_ = tf.placeholder(tf.float32, [None, None])\n",
    "\n",
    "        correct_rate = tf.zeros([])\n",
    "        predictions_list = []\n",
    "        support_list = []\n",
    "\n",
    "        for i in range(self.validation_size):\n",
    "            question = validation_question[i,:]\n",
    "            sentences = validation_sentences[i,:,:]\n",
    "            y_ = tf.reshape(validation_y_[i,:], [1, self.vocabluary_size])\n",
    "\n",
    "            question_embed = tf.reshape(\n",
    "                tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(B, question), 0), \n",
    "                [1, self.question_embed_dim])\n",
    "\n",
    "\n",
    "            u = question_embed\n",
    "\n",
    "            for j in range(self.num_hops):\n",
    "                sentences_output_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(C[j], sentences), 1)\n",
    "                sentences_input_embed = tf.reduce_sum(\n",
    "                    tf.nn.embedding_lookup(A[j], sentences), 1)\n",
    "                \n",
    "                p = tf.nn.softmax(\n",
    "                    tf.matmul(u, sentences_input_embed, transpose_b = True))\n",
    "                support_list.append(tf.argmax(p, 1))\n",
    "                u = u + tf.reduce_sum(\n",
    "                    tf.mul(tf.transpose(p), sentences_output_embed), 0)\n",
    "\n",
    "            y = tf.nn.softmax(\n",
    "                tf.matmul(u, W))\n",
    "\n",
    "            predictions_list.append(tf.argmax(y, 1))\n",
    "            \n",
    "            \n",
    "        support = tf.reshape(\n",
    "            tf.pack(support_list), [self.validation_size, self.num_hops])\n",
    "        predictions = tf.reshape(tf.pack(predictions_list), [-1])\n",
    "        correct_rate = tf.reduce_mean(\n",
    "            tf.cast(\n",
    "                tf.equal(\n",
    "                    predictions, tf.argmax(validation_y_, 1)),\n",
    "                tf.float32))\n",
    "\n",
    "        self.batch_placeholder = [batch_sentences, batch_question, batch_y_]\n",
    "        self.validation_placeholder = [validation_sentences, validation_question, validation_y_]\n",
    "#        self.train_step = train_step\n",
    "        self.correct_rate = correct_rate\n",
    "        self.predictions = predictions\n",
    "        self.support = support\n",
    "        self.cross_entropy = cross_entropy\n",
    "        self.init = tf.initialize_all_variables()\n",
    "\n",
    "        \n",
    "    def train(self, steps):\n",
    "        for i in range(steps):\n",
    "            batch = self.batch_gen.get_next_batch()\n",
    "            batch_dic = {self.batch_placeholder[0] : batch[0], self.batch_placeholder[1] : batch[1], self.batch_placeholder[2] : batch[2]}\n",
    "#            if i % 100 == 0:\n",
    "#                print(self.session.run(self.learning_rate, feed_dict = batch_dic), \" \", self.session.run(self.grad_before_clipping, feed_dict = batch_dic), self.session.run(self.grad_after_clipping, feed_dict = batch_dic))\n",
    "            self.session.run(self.app_grads, feed_dict = batch_dic)\n",
    "            self.session.run(self.make_step, feed_dict = batch_dic)\n",
    "\n",
    "                         \n",
    "    def validate(self, print_examples, print_score = True):\n",
    "        validation_set = self.batch_gen.get_validation()\n",
    "        validation_dict = {self.validation_placeholder[0] : validation_set[0], self.validation_placeholder[1] : validation_set[1], self.validation_placeholder[2] : validation_set[2]}\n",
    "        if print_score == True:    \n",
    "            print(\"Correct rate:  \", self.session.run(self.correct_rate, feed_dict = validation_dict))\n",
    "        ans = self.session.run(self.predictions, feed_dict = validation_dict)\n",
    "        sup = self.session.run(self.support, feed_dict = validation_dict)\n",
    "        for x in np.random.random_integers(0, self.validation_size - 1, print_examples):\n",
    "            preprocessing.print_task(validation_set[0][[x],:,:], validation_set[1][[x],:], validation_set[2][[x],:], self.batch_gen.words)\n",
    "            print(\"\\n\")\n",
    "            print(\"Model's answer: \", end = \" \")\n",
    "            print(preprocessing.num2word(self.batch_gen.words, ans[x]), \"\\n\")\n",
    "            print(\"Support sentences:\")\n",
    "            indicies = list(sup[x, :])\n",
    "            preprocessing.print_sentences(np.reshape(validation_set[0][x, indicies, :], [1, self.num_hops, -1]), self.batch_gen.words)\n",
    "                \n",
    "            print(\"-------------------------\")\n",
    "        return self.session.run(self.correct_rate, feed_dict = validation_dict)\n",
    "            \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train_babi_task(num):\n",
    "    filename = \"/home/savvai/Desktop/end-to-end_network/data/task_\" + str(num) + \".txt\"\n",
    "    vocabluary_size = len(preprocessing.build_dic(preprocessing.load_file(filename)))\n",
    "    n = EndToEndNetwork(vocabluary_size)\n",
    "    n.init_batch_gen(filename)\n",
    "#    n.build_linear_graph()\n",
    "#    n.train(300)\n",
    "    n.build_graph()\n",
    "    n.init_variables()\n",
    "    n.train(1000)\n",
    "    n.validate(10)\n",
    "    \n",
    "def best_of_ten(task):\n",
    "    filename = \"/home/savvai/Desktop/end-to-end_network/data/task_\" + str(task) + \".txt\"\n",
    "    vocabluary_size = len(preprocessing.build_dic(preprocessing.load_file(filename)))\n",
    "    n = EndToEndNetwork(vocabluary_size)\n",
    "    n.build_graph()\n",
    "    maxscore = 0\n",
    "    n.init_batch_gen(filename)\n",
    "    for i in range(10):\n",
    "        n.init_variables()\n",
    "        n.train(1000)\n",
    "        score = n.validate(0, print_score = False)\n",
    "#        print(score)\n",
    "        if score > maxscore:\n",
    "            maxscore = score\n",
    "    print(\"task #\", task, \"    \", maxscore)\n",
    "    return maxscore\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py:214: DeprecationWarning: This function is deprecated. Please call randint(0, 299 + 1) instead\n"
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "n = EndToEndNetwork(100)\n",
    "n.build_graph()\n",
    "for i in range(1, 20):\n",
    "    scores.append(best_of_ten(i))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}