2. Dataset Used in Training..ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "V6XaWhS8ChBy",
    "outputId": "105776dd-33d4-431d-b9b0-d2b54aa7a6a7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content\n"
     ]
    }
   ],
   "source": [
    "#https://drive.google.com/open?id=1NVfn_L9YoGJloB_rzKh06YCqQNe6DfJj\n",
    "%cd /content\n",
    "!pip install -U -q PyDrive\n",
    "\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "\n",
    "# 1. Authenticate and create the PyDrive client.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XZfzgrWf5nLg"
   },
   "outputs": [],
   "source": [
    "# https://drive.google.com/open?id=1iLrNPNQ3GMZquyE7QhB7l9Yt4mS46r_p\n",
    "downloaded = drive.CreateFile({'id': '1iLrNPNQ3GMZquyE7QhB7l9Yt4mS46r_p'})\n",
    "downloaded.GetContentFile('coco_vgg_id_map.txt')\n",
    "\n",
    "# https://drive.google.com/open?id=1qmDX_URx9rKqjBZczX0hINk5uA4kI9SA\n",
    "downloaded = drive.CreateFile({'id': '1qmDX_URx9rKqjBZczX0hINk5uA4kI9SA'})\n",
    "downloaded.GetContentFile('Training Data QA.pickle')\n",
    "\n",
    "# https://drive.google.com/open?id=1vBVqBh-caYYat7ZXBleJWHGV0al1I4nJ\n",
    "downloaded = drive.CreateFile({'id': '1vBVqBh-caYYat7ZXBleJWHGV0al1I4nJ'})\n",
    "downloaded.GetContentFile('Validation Data QA.pickle')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YS7XM7FFZy_C"
   },
   "outputs": [],
   "source": [
    "# https://drive.google.com/open?id=1dp5cFJGjCwzND2SXn22N5ON8dnuS2NT5\n",
    "downloaded = drive.CreateFile({'id': '1dp5cFJGjCwzND2SXn22N5ON8dnuS2NT5'})\n",
    "downloaded.GetContentFile('model_2_lstm_embed.h5')\n",
    "\n",
    "# https://drive.google.com/open?id=1OczoYYqeYjRGF8LOYu1wzitHzs40t-ox\n",
    "downloaded = drive.CreateFile({'id': '1OczoYYqeYjRGF8LOYu1wzitHzs40t-ox'})\n",
    "downloaded.GetContentFile('model_2_lstm_embed_tok.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 354
    },
    "colab_type": "code",
    "id": "rH1AUMAFPp16",
    "outputId": "8ea47f18-ad33-47e9-a325-50dd1def3c8d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-06-26 20:16:25--  http://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip\n",
      "Resolving cs.stanford.edu (cs.stanford.edu)... 171.64.64.64\n",
      "Connecting to cs.stanford.edu (cs.stanford.edu)|171.64.64.64|:80... connected.\n",
      "HTTP request sent, awaiting response... 301 Moved Permanently\n",
      "Location: https://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip [following]\n",
      "--2019-06-26 20:16:25--  https://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip\n",
      "Connecting to cs.stanford.edu (cs.stanford.edu)|171.64.64.64|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 764830639 (729M) [application/zip]\n",
      "Saving to: ‘coco.zip’\n",
      "\n",
      "coco.zip            100%[===================>] 729.40M  15.9MB/s    in 50s     \n",
      "\n",
      "2019-06-26 20:17:16 (14.6 MB/s) - ‘coco.zip’ saved [764830639/764830639]\n",
      "\n",
      "Archive:  coco.zip\n",
      "  inflating: coco/dataset.json       \n",
      "  inflating: coco/readme.txt         \n",
      "  inflating: coco/vgg_feats.mat      \n"
     ]
    }
   ],
   "source": [
    "!wget http://cs.stanford.edu/people/karpathy/deepimagesent/coco.zip\n",
    "!unzip coco.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Bm1kt55xc7Bb"
   },
   "outputs": [],
   "source": [
    "import scipy.io\n",
    "mat = scipy.io.loadmat('/content/coco/vgg_feats.mat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "rs5XQlB5dMfn",
    "outputId": "532d3a1a-1bae-4cc4-8bda-48d93d709f73"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4096, 123287)"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mat[\"feats\"].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uykpJmeYgLeg"
   },
   "source": [
    "vgg_feature matrix are the features of images which have been fed into vgg16 architecture, with 16 layers of convolutional neural network and Dense layer(with 4096 units), except the last dense layer(with 1000 units). So each of the image has beeen fed into the model and which gave us the feature vector of each image, in the shape of 4096 X 1. Concatenating each feature vector of every image, give us the matrix of shape 4096 X 123287.\n",
    "SInce, 82783 + 40504 = 1232287 (Training and valiation data).\n",
    "  \n",
    "From coco_vgg_id_map.txt file, we have image_id in order of the columns of vgg_feats matrix.\n",
    "Selecting top 10 image id from coco_vgg_id_map file, means selecting top 10 columns from vgg_feats matix. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7Y4XodX5igbH"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import scipy as sc\n",
    "import scipy.io\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def get_coco_features(split, types ):\n",
    "  %cd /content\n",
    "    if split == 'train':\n",
    "        data_path = 'Training Data QA.pickle'\n",
    "        if ( types == \"small\"):\n",
    "            num_data = 10000\n",
    "        elif (types == \"full\"):\n",
    "            num_data = 82783\n",
    "\n",
    "    elif split == 'val':\n",
    "        data_path = 'Validation Data QA.pickle'\n",
    "        if (types == \"small\"):\n",
    "            num_data = 2000\n",
    "        elif (types == \"full\"):\n",
    "            num_data = 40504\n",
    "    else:\n",
    "        print('Invalid split!')\n",
    "        sys.exit()\n",
    "  \n",
    "    id_map_path = 'coco_vgg_id_map.txt'\n",
    "    features_path = '/content/coco/vgg_feats.mat'\n",
    "    img_labels = pd.read_pickle(data_path)[['image_id']].drop_duplicates().values.tolist()\n",
    "    img_ids = open(id_map_path).read().splitlines()\n",
    "    features_struct = sc.io.loadmat(features_path)\n",
    "\n",
    "    id_map = {}\n",
    "    for ids in img_ids:\n",
    "        ids_split = ids.split()\n",
    "        id_map[int(ids_split[0])] = int(ids_split[1])\n",
    "\n",
    "    VGGfeatures = features_struct['feats']\n",
    "    nb_dimensions = VGGfeatures.shape[0]\n",
    "    nb_images = len(img_labels)\n",
    "    image_matrix = np.zeros((nb_images,nb_dimensions))\n",
    "\n",
    "    for i in range(nb_images):\n",
    "        image_matrix[i,:] = VGGfeatures[:,id_map[img_labels[i][0]]]  \n",
    "    image_matrix.astype('float32')\n",
    "    return image_matrix[0:num_data]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "colab_type": "code",
    "id": "eCkhwKMuqZXX",
    "outputId": "37f9e29a-e78e-492a-b816-bb66c45ebdbc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image features ...\n",
      "/content\n",
      "/content\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "print('Loading image features ...')\n",
    "small_img_features_train = get_coco_features('train', types = \"small\")\n",
    "small_img_features_val = get_coco_features('val', types = \"small\")\n",
    "\n",
    "#combined_image = np.concatenate((img_features_train, img_features_val), axis = 0) #(123287, 4096)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xPiYsyyGqcPy"
   },
   "outputs": [],
   "source": [
    "import h5py\n",
    "h5_feats = h5py.File('small_img_features_train.h5', 'w')\n",
    "h5_feats.create_dataset('small_img_features_train', data = small_img_features_train)\n",
    "h5_feats.close()\n",
    "\n",
    "h5_feats_val = h5py.File('small_img_features_val.h5', 'w')\n",
    "h5_feats_val.create_dataset('small_img_features_val', data = small_img_features_val)\n",
    "h5_feats_val.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "b2Mw3xokrp0D",
    "outputId": "156492e0-b3eb-4944-82b0-05357bf3d93a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 4096)\n",
      "(2000, 4096)\n"
     ]
    }
   ],
   "source": [
    "print(small_img_features_train.shape)\n",
    "print(small_img_features_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hdIffKMVuVJr"
   },
   "outputs": [],
   "source": [
    "uploaded = drive.CreateFile({'title': 'small_img_features_train.h5'})\n",
    "uploaded.SetContentFile('small_img_features_train.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'small_img_features_val.h5'})\n",
    "uploaded.SetContentFile('small_img_features_val.h5')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "colab_type": "code",
    "id": "o6q1TmilVyt5",
    "outputId": "0cb3faee-9911-43a9-dbec-f391187e0c8f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading image features ...\n",
      "/content\n",
      "/content\n",
      "(82783, 4096)\n",
      "(40504, 4096)\n"
     ]
    }
   ],
   "source": [
    "print('Loading image features ...')\n",
    "full_img_features_train = get_coco_features('train', types = \"full\")\n",
    "full_img_features_val = get_coco_features('val', types = \"full\")\n",
    "\n",
    "import h5py\n",
    "h5_feats = h5py.File('full_img_features_train.h5', 'w')\n",
    "h5_feats.create_dataset('full_img_features_train', data = full_img_features_train)\n",
    "h5_feats.close()\n",
    "\n",
    "h5_feats_val = h5py.File('full_img_features_val.h5', 'w')\n",
    "h5_feats_val.create_dataset('full_img_features_val', data = full_img_features_val)\n",
    "h5_feats_val.close()\n",
    "\n",
    "print(full_img_features_train.shape)\n",
    "print(full_img_features_val.shape)\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'full_img_features_train.h5'})\n",
    "uploaded.SetContentFile('full_img_features_train.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'full_img_features_val.h5'})\n",
    "uploaded.SetContentFile('full_img_features_val.h5')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "e5SnlbwsA7Xl"
   },
   "source": [
    "Now, Lets process the questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "ulfomy4OPs_V",
    "outputId": "5c1b4455-731c-4bb3-e020-ebd1efe33f83"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download(\"punkt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "8bZ4TC8lBUow",
    "outputId": "632f3d41-ee9e-474e-a3ff-c9e591ff05d4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from scipy import io\n",
    "import operator\n",
    "import sys\n",
    "import scipy as sc\n",
    "from collections import defaultdict\n",
    "from nltk import word_tokenize\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import h5py\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WHgvEdtMBMz8"
   },
   "outputs": [],
   "source": [
    "def get_question_tokenizer(types):\n",
    "    data_path = \"Training Data QA.pickle\"\n",
    "    data_path_val = \"Validation Data QA.pickle\"\n",
    "\n",
    "    if ( types == \"small\"):\n",
    "        num_data = 10000\n",
    "        num_data_val = 2000\n",
    "    elif (types == \"full\"):\n",
    "        num_data = 248349\n",
    "        num_data_val = 121512\n",
    "\n",
    "    df = pd.read_pickle(data_path) \n",
    "    df_val = pd.read_pickle(data_path_val)\n",
    "    questions = df['questions'].values.tolist()\n",
    "    questions_val = df_val['questions'].values.tolist()\n",
    "   \n",
    "    all_question = questions + questions_val\n",
    "  \n",
    "    tokenizer = Tokenizer(num_words = 10000)\n",
    "    tokenizer.fit_on_texts(all_question)\n",
    "\n",
    "    word_index = tokenizer.word_index\n",
    "\n",
    "    # Save the tokenizer, so that we can use this tokenizer whenever we need to predict any reviews.\n",
    "    with open('tokenizer.pickle', 'wb') as handle:\n",
    "        pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    #tokenising train data\n",
    "    train_question_tokenized = tokenizer.texts_to_sequences(questions)      \n",
    "    questions = pad_sequences(train_question_tokenized, maxlen = 25)          # len(X_train) x 25\n",
    "\n",
    "    #tokenising validation data\n",
    "    val_question_tokenized = tokenizer.texts_to_sequences(questions_val)\n",
    "    questions_val = pad_sequences(val_question_tokenized, maxlen = 25)               # len(X_val) X 25 \n",
    "\n",
    "    return questions[0:num_data], questions_val[0: num_data_val], word_index\n",
    "  \n",
    "def get_questions_matrix(split):\n",
    "  \n",
    "    if split == 'train':\n",
    "        data_path = 'data_train_qa.pickle'\n",
    "    elif split == 'val':\n",
    "        data_path = 'data_val_qa.pickle'\n",
    "    else:\n",
    "        print('Invalid split!')\n",
    "        sys.exit()\n",
    "\n",
    "    df = pd.read_pickle(data_path)\n",
    "    questions = df[['questions']].values.tolist()\n",
    "    word_idx = load_idx()\n",
    "    seq_list = []\n",
    "\n",
    "    for question in questions:\n",
    "        words = word_tokenize(question[0])\n",
    "        seq = []\n",
    "        for word in words:\n",
    "            seq.append(word_idx.get(word,0))\n",
    "        seq_list.append(seq)\n",
    "    question_matrix = pad_sequences(seq_list)\n",
    "  \n",
    "    question_matrix.astype('int32')\n",
    "    return question_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aa2MyUCMBcmU"
   },
   "outputs": [],
   "source": [
    "# create tokenized question data\n",
    "import h5py\n",
    "small_question_train_tokenize, small_question_val_tokenize, word_idx = get_question_tokenizer(types = \"small\")\n",
    "\n",
    "#combined_question_tokenize = np.concatenate((question_train_tokenize,  question_val_tokenize),axis = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zIUrTo68coja"
   },
   "outputs": [],
   "source": [
    "h5f = h5py.File('small_question_train_tokenize.h5', 'w')\n",
    "h5f.create_dataset('small_question_train_tokenize', data=small_question_train_tokenize)\n",
    "h5f.close()\n",
    "\n",
    "h5f_val = h5py.File('small_question_val_tokenize.h5', 'w')\n",
    "h5f_val.create_dataset('small_question_val_tokenize', data=small_question_val_tokenize)\n",
    "h5f_val.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'small_question_train_tokenize.h5'})\n",
    "uploaded.SetContentFile('small_question_train_tokenize.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'small_question_val_tokenize.h5'})\n",
    "uploaded.SetContentFile('small_question_val_tokenize.h5')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BxRQxuS9P9S3"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "file = open(\"word_idx.pickle\", \"wb\")\n",
    "pickle.dump(word_idx, file)\n",
    "file.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'word_idx.pickle'})\n",
    "uploaded.SetContentFile('word_idx.pickle')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pX8Zk4AWaVd9"
   },
   "outputs": [],
   "source": [
    "# create tokenized question data\n",
    "import h5py\n",
    "full_question_train_tokenize, full_question_val_tokenize, word_idx_full = get_question_tokenizer(types = \"full\")\n",
    "\n",
    "h5f = h5py.File('full_question_train_tokenize.h5', 'w')\n",
    "h5f.create_dataset('full_question_train_tokenize', data=full_question_train_tokenize)\n",
    "h5f.close()\n",
    "\n",
    "h5f_val = h5py.File('full_question_val_tokenize.h5', 'w')\n",
    "h5f_val.create_dataset('full_question_val_tokenize', data=full_question_val_tokenize)\n",
    "h5f_val.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'full_question_train_tokenize.h5'})\n",
    "uploaded.SetContentFile('full_question_train_tokenize.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'full_question_val_tokenize.h5'})\n",
    "uploaded.SetContentFile('full_question_val_tokenize.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "import pickle\n",
    "\n",
    "file = open(\"word_idx_full.pickle\", \"wb\")\n",
    "pickle.dump(word_idx_full, file)\n",
    "file.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'word_idx_full.pickle'})\n",
    "uploaded.SetContentFile('word_idx_full.pickle')\n",
    "uploaded.Upload()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 391
    },
    "colab_type": "code",
    "id": "Mcd6awTU-PSk",
    "outputId": "bdb80c67-175b-408c-a8c3-93ab6e62950e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-06-18 15:02:42--  http://nlp.stanford.edu/data/glove.840B.300d.zip\n",
      "Resolving nlp.stanford.edu (nlp.stanford.edu)... 171.64.67.140\n",
      "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:80... connected.\n",
      "HTTP request sent, awaiting response... 302 Found\n",
      "Location: https://nlp.stanford.edu/data/glove.840B.300d.zip [following]\n",
      "--2019-06-18 15:02:42--  https://nlp.stanford.edu/data/glove.840B.300d.zip\n",
      "Connecting to nlp.stanford.edu (nlp.stanford.edu)|171.64.67.140|:443... connected.\n",
      "HTTP request sent, awaiting response... 301 Moved Permanently\n",
      "Location: http://downloads.cs.stanford.edu/nlp/data/glove.840B.300d.zip [following]\n",
      "--2019-06-18 15:02:42--  http://downloads.cs.stanford.edu/nlp/data/glove.840B.300d.zip\n",
      "Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 171.64.64.22\n",
      "Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|171.64.64.22|:80... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 2176768927 (2.0G) [application/zip]\n",
      "Saving to: ‘glove.840B.300d.zip’\n",
      "\n",
      "glove.840B.300d.zip 100%[===================>]   2.03G  47.5MB/s    in 49s     \n",
      "\n",
      "2019-06-18 15:03:32 (42.1 MB/s) - ‘glove.840B.300d.zip’ saved [2176768927/2176768927]\n",
      "\n",
      "Archive:  glove.840B.300d.zip\n",
      "  inflating: glove.840B.300d.txt     \n"
     ]
    }
   ],
   "source": [
    "!wget http://nlp.stanford.edu/data/glove.840B.300d.zip\n",
    "!unzip glove.840B.300d.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lZ3L1Xl51JjQ"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def loadGloveModel(gloveFile, word_index):\n",
    "    print(\"Loading Glove Model\")\n",
    "    f = open(gloveFile,'r', encoding='utf8')\n",
    "    embedding_index = {}\n",
    "    print(\"Opened!\")\n",
    "    for j, line in enumerate(f):\n",
    "        splitLine = line.split(' ')\n",
    "        word = splitLine[0]\n",
    "        embedding = np.asarray(splitLine[1:], dtype='float32')\n",
    "        embedding_index[word] = embedding\n",
    "    \n",
    "    print(\"Done.\",len(embedding_index),\" words loaded!\")\n",
    "  \n",
    "    # Now, we need to create embedding matrix.\n",
    "    EMBEDDING_DIM = 300\n",
    "    embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "    print(embedding_matrix.shape)\n",
    "  \n",
    "    for word, i in word_index.items():\n",
    "        embedding_vector = embedding_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "            # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    \n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "uE25eJIxV8Tw",
    "outputId": "fc1c7480-c55c-46ab-a752-afd221395995"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Glove Model\n",
      "Opened!\n",
      "Done. 2196016  words loaded!\n",
      "(16110, 300)\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import pickle\n",
    "gloveFile = 'glove.840B.300d.txt'\n",
    "file = open(\"word_idx.pickle\", \"rb\")\n",
    "word_idx = pickle.load(file)\n",
    "file.close()\n",
    "embedding_matrix_tokenize = loadGloveModel(gloveFile, word_idx) # (16110, 300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q1aSaPg6eMTL"
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "file = open(\"embedding_matrix_tokenize.pickle\", \"wb\")\n",
    "pickle.dump(embedding_matrix_tokenize, file)\n",
    "file.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'embedding_matrix_tokenize.pickle'})\n",
    "uploaded.SetContentFile('embedding_matrix_tokenize.pickle')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "js32TQEbTn-d"
   },
   "outputs": [],
   "source": [
    "import h5py\n",
    "h5_feats = h5py.File('embedding_matrix_tokenize.h5', 'w')\n",
    "h5_feats.create_dataset('embedding_matrix_tokenize', data = embedding_matrix_tokenize)\n",
    "h5_feats.close()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'embedding_matrix_tokenize.h5'})\n",
    "uploaded.SetContentFile('embedding_matrix_tokenize.h5')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zDkYsSW401m_"
   },
   "outputs": [],
   "source": [
    "uploaded = drive.CreateFile({'title': 'tokenizer.pickle'})\n",
    "uploaded.SetContentFile('tokenizer.pickle')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y5358TBWbbn4"
   },
   "outputs": [],
   "source": [
    "uploaded = drive.CreateFile({'title': 'tokenizer_full.pickle'})\n",
    "uploaded.SetContentFile('tokenizer.pickle')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "ZZ2QCad-LycF",
    "outputId": "1c7c8158-1c69-4f01-8a61-07b50e714793"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "          0,   0,   0,   0,   0,   3,   2,   1, 932,   7,   1,  44]],\n",
       "      dtype=int32)"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions = [\"what is the stripe on the train\"]\n",
    "with open('tokenizer.pickle', 'rb') as handle:\n",
    "    tokenizer = pickle.load(handle)\n",
    "\n",
    "\n",
    "train_question_tokenized = tokenizer.texts_to_sequences(questions)      \n",
    "questions = pad_sequences(train_question_tokenized, maxlen = 25)          # len(X_train) x 25\n",
    "questions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ez4G0vFXxOfT"
   },
   "source": [
    "Let's process answers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "DuLTGJJdj07S",
    "outputId": "76d7605f-18e5-475d-f168-e737f4f67e80"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content\n",
      "/content\n"
     ]
    }
   ],
   "source": [
    "def int_to_answers():\n",
    "    %cd /content\n",
    "    data_path = 'Training Data QA.pickle'\n",
    "    df = pd.read_pickle(data_path)\n",
    "    answers = df['multiple_choice_answer'].values.tolist()\n",
    "    freq = defaultdict(int)\n",
    "    for answer in answers:\n",
    "        freq[answer[0].lower()] += 1\n",
    "    int_to_answer = sorted(freq.items(),key=operator.itemgetter(1),reverse=True)[0:1000]\n",
    "    int_to_answer = [answer[0] for answer in int_to_answer]\n",
    "    return int_to_answer\n",
    "\n",
    "top_answers = int_to_answers()\t\n",
    "\n",
    "def answers_to_onehot():\n",
    "\ttop_answers = int_to_answers()\n",
    "\tanswer_to_onehot = {}\n",
    "\tfor i, word in enumerate(top_answers):\n",
    "\t\tonehot = np.zeros(1001)\n",
    "\t\tonehot[i] = 1.0\n",
    "\t\tanswer_to_onehot[word] = onehot\n",
    "\treturn answer_to_onehot\n",
    "\t\n",
    "answer_to_onehot_dict = answers_to_onehot()\n",
    "\n",
    "def get_answers_matrix(split, types):\n",
    "  \n",
    "    if split == 'train':\n",
    "        data_path = 'Training Data QA.pickle'\n",
    "        if ( types == \"small\"):\n",
    "            num_data = 30000\n",
    "        elif (types == \"full\"):\n",
    "            num_data = 2483490 \n",
    "\n",
    "    elif split == 'val':\n",
    "        data_path = 'Validation Data QA.pickle'\n",
    "        if (types == \"small\"):\n",
    "            num_data = 6000\n",
    "        elif (types == \"full\"):\n",
    "            num_data = 1215120 \n",
    "    else:\n",
    "        print('Invalid split!')\n",
    "        sys.exit()\n",
    "     \n",
    "    df = pd.read_pickle(data_path)\n",
    "    answers = df['multiple_choice_answer'].values.tolist()\n",
    "    answer_matrix = np.zeros((len(answers),1001))\n",
    "    default_onehot = np.zeros(1001)\n",
    "    default_onehot[1000] = 1.0\n",
    "\t\n",
    "    for i, answer in enumerate(answers):\n",
    "        answer_matrix[i] = answer_to_onehot_dict.get(answer[0].lower(),default_onehot)\n",
    "\t\n",
    "    answer_matrix.astype('int32')\n",
    "    return answer_matrix[0:num_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "gbo6M_v_MTtZ",
    "outputId": "b3bbb4cf-f142-4c29-a540-50ff4af19975"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading answers ...\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "\n",
    "print('Loading answers ...')\n",
    "small_answers_train = get_answers_matrix('train', types = \"small\") # float64\n",
    "small_answers_val = get_answers_matrix('val', types = \"small\")\n",
    "\n",
    "h5_ans = h5py.File('small_answers_train.h5', 'w')\n",
    "h5_ans.create_dataset('small_answers_train', data = small_answers_train)\n",
    "h5_ans.close()\n",
    "\n",
    "h5_ans_val = h5py.File('small_answers_val.h5', 'w')\n",
    "h5_ans_val.create_dataset('small_answers_val', data = small_answers_val)\n",
    "h5_ans_val.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "k6L6eYZLMZAA",
    "outputId": "6ce574e2-2577-4bed-e866-20f577c77407"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 1001)\n",
      "(6000, 1001)\n"
     ]
    }
   ],
   "source": [
    "print(small_answers_train.shape)\n",
    "print(small_answers_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hAb2APpX5PCc"
   },
   "outputs": [],
   "source": [
    "uploaded = drive.CreateFile({'title': 'small_answers_train.h5'})\n",
    "uploaded.SetContentFile('small_answers_train.h5')\n",
    "uploaded.Upload()\n",
    "\n",
    "uploaded = drive.CreateFile({'title': 'small_answers_val.h5'})\n",
    "uploaded.SetContentFile('small_answers_val.h5')\n",
    "uploaded.Upload()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D4RZ1kgXzNsj"
   },
   "source": [
    "With all the data in our drive, let's start training VQA. Continue with Notebook#3"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "anaconda-cloud": {},
  "colab": {
   "collapsed_sections": [],
   "name": "Testing Phase.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}