dialogue_denoiser_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ishalyminov/.virtualenvs/dialogue_denoiser/local/lib/python2.7/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from random import shuffle\n",
    "import random\n",
    "from operator import itemgetter\n",
    "from collections import defaultdict\n",
    "\n",
    "import keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENCODER_DATA = 'swda_parallel_corpus/encoder.txt'\n",
    "DECODER_DATA = 'swda_parallel_corpus/decoder.txt'\n",
    "\n",
    "TRAINSET_RATIO = 0.8\n",
    "VOCABULARY_SIZE = 10000\n",
    "MAX_INPUT_LENGTH = 80\n",
    "\n",
    "PAD_ID = 0\n",
    "UNK_ID = 1\n",
    "PAD = '_PAD'\n",
    "UNK = '_UNK'\n",
    "\n",
    "random.seed(273)\n",
    "np.random.seed(273)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_vocabulary(in_lines, max_vocabulary_size):\n",
    "    freqdict = defaultdict(lambda: 0)\n",
    "    for line in in_lines:\n",
    "        for token in line:\n",
    "            freqdict[token] += 1\n",
    "    vocab = sorted(freqdict.items(), key=itemgetter(1), reverse=True)\n",
    "    rev_vocab = ([PAD, UNK] + map(itemgetter(0), vocab))[:max_vocabulary_size]\n",
    "    vocab = {word: idx for idx, word in enumerate(rev_vocab)}\n",
    "    return vocab, rev_vocab\n",
    "\n",
    "\n",
    "def vectorize_sequences(in_sequences, in_vocab, max_input_length):\n",
    "    sequences_vectorized = []\n",
    "    for sequence in in_sequences:\n",
    "        sequences_vectorized.append([in_vocab.get(token, UNK_ID) for token in sequence])\n",
    "    return keras.preprocessing.sequence.pad_sequences(sequences_vectorized, value=PAD_ID, maxlen=max_input_length)\n",
    "\n",
    "\n",
    "def load_dataset(in_encoder_input, in_decoder_input):\n",
    "    with open(in_encoder_input) as encoder_in:\n",
    "        with open(in_decoder_input) as decoder_in:\n",
    "            encoder_lines, decoder_lines = [map(lambda x: x.strip(), encoder_in.readlines()),\n",
    "                                            map(lambda x: x.strip(), decoder_in.readlines())]\n",
    "    return encoder_lines, decoder_lines\n",
    "\n",
    "\n",
    "def make_tagger_data_point(in_src, in_tgt):\n",
    "    source, target = in_src.lower().split(), in_tgt.lower().split()\n",
    "    tags = []\n",
    "    src_index, tgt_index = 0, 0\n",
    "    while src_index < len(source):\n",
    "        if tgt_index < len(target) and source[src_index] == target[tgt_index]:\n",
    "            tags.append(1)\n",
    "            tgt_index += 1\n",
    "        else:\n",
    "            tags.append(0)\n",
    "        src_index += 1\n",
    "    assert len(tags) == len(source)\n",
    "    return source, tags\n",
    "\n",
    "\n",
    "def make_tagger_data_points(in_encoder_lines, in_decoder_lines):\n",
    "    result = []\n",
    "    for src_line, tgt_line in zip(in_encoder_lines, in_decoder_lines):\n",
    "        result.append(make_tagger_data_point(src_line, tgt_line))\n",
    "    return result\n",
    "\n",
    "\n",
    "def to_one_hot(in_sequence, in_classes_number):\n",
    "    result = np.zeros((len(in_sequence), in_classes_number))\n",
    "    for idx, element in enumerate(in_sequence):\n",
    "        result[idx][element] = 1\n",
    "    return result\n",
    "\n",
    "\n",
    "def make_dataset(in_encoder_lines, in_decoder_lines, vocab=None):\n",
    "    data_points = make_tagger_data_points(encoder_lines, decoder_lines)\n",
    "    shuffle(data_points)\n",
    "    trainset_size = int(TRAINSET_RATIO * len(data_points))\n",
    "    devset_size = int((len(data_points) - trainset_size) / 2.0)\n",
    "    train, dev, test = (data_points[:trainset_size],\n",
    "                        data_points[trainset_size: trainset_size + devset_size],\n",
    "                        data_points[trainset_size + devset_size:])\n",
    "    if not vocab:\n",
    "        vocab, _ = make_vocabulary(map(itemgetter(0), train), VOCABULARY_SIZE)\n",
    "    X_train = vectorize_sequences(map(itemgetter(0), train), vocab, MAX_INPUT_LENGTH)\n",
    "    y_train = np.asarray([to_one_hot(tags, 2)\n",
    "                          for tags in keras.preprocessing.sequence.pad_sequences(map(itemgetter(1), train), value=0, maxlen=MAX_INPUT_LENGTH)])\n",
    "    X_dev = vectorize_sequences(map(itemgetter(0), dev), vocab, MAX_INPUT_LENGTH)\n",
    "    y_dev = np.asarray([to_one_hot(tags, 2)\n",
    "                        for tags in keras.preprocessing.sequence.pad_sequences(map(itemgetter(1), dev), value=0, maxlen=MAX_INPUT_LENGTH)])\n",
    "    X_test = vectorize_sequences(map(itemgetter(0), test), vocab, MAX_INPUT_LENGTH)\n",
    "    y_test = np.asarray([to_one_hot(tags, 2)\n",
    "                         for tags in keras.preprocessing.sequence.pad_sequences(map(itemgetter(1), test), value=0, maxlen=MAX_INPUT_LENGTH)])\n",
    "    return vocab, (X_train, y_train), (X_dev, y_dev), (X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model(in_vocab_size, in_cell_size, in_max_input_length, in_classes_number, lr):\n",
    "    input_sequence = keras.layers.Input(shape=(in_max_input_length,))\n",
    "    embedding = keras.layers.Embedding(in_vocab_size, in_cell_size)(input_sequence)\n",
    "    lstm = keras.layers.LSTM(in_cell_size, return_sequences=True)(embedding)\n",
    "    output = keras.layers.Dense(in_classes_number, activation='softmax')(lstm)\n",
    "    model = keras.Model(inputs=[input_sequence], outputs=[output])\n",
    "\n",
    "    # mean absolute error, accuracy\n",
    "    opt = keras.optimizers.Adam(lr=lr)\n",
    "    model.compile(optimizer=opt, loss='binary_crossentropy')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(in_model,\n",
    "          train_data,\n",
    "          dev_data,\n",
    "          test_data,\n",
    "          in_checkpoint_filepath,\n",
    "          epochs=100,\n",
    "          batch_size=32,\n",
    "          **kwargs):\n",
    "    X_train, y_train = train_data\n",
    "    X_dev, y_dev = dev_data\n",
    "    X_test, y_test = test_data\n",
    "\n",
    "    in_model.fit(X_train,\n",
    "                 y_train,\n",
    "                epochs=epochs,\n",
    "                shuffle=True,\n",
    "                validation_data=(X_dev, y_dev),\n",
    "                callbacks=[keras.callbacks.ModelCheckpoint(in_checkpoint_filepath,\n",
    "                                                           monitor='val_loss',\n",
    "                                                           verbose=1,\n",
    "                                                           save_best_only=True,\n",
    "                                                           save_weights_only=False,\n",
    "                                                           mode='auto',\n",
    "                                                           period=1),\n",
    "                           keras.callbacks.EarlyStopping(monitor='val_loss',\n",
    "                                                         min_delta=0,\n",
    "                                                         patience=10,\n",
    "                                                         verbose=1,\n",
    "                                                        mode='auto')])\n",
    "    test_loss = in_model.evaluate(x=X_test, y=y_test)\n",
    "    print 'Testset loss after {} epochs: {:.3f}'.format(epochs, test_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(in_model, X):\n",
    "    return np.argmax(model.predict(np.asarray([X])), axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(in_model, X, y):\n",
    "    y_pred = np.argmax(model.predict(X), axis=-1)\n",
    "    y_gold = np.argmax(y, axis=-1)\n",
    "    return sum([int(np.array_equal(y_pred_i, y_gold_i))\n",
    "                for y_pred_i, y_gold_i in zip(y_pred, y_gold)]) / float(y.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uh well what would you say your opinion is on gun control \n",
      "what would you say your opinion is on gun control\n"
     ]
    }
   ],
   "source": [
    "encoder_lines, decoder_lines = load_dataset(ENCODER_DATA, DECODER_DATA)\n",
    "print encoder_lines[0], '\\n', decoder_lines[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab, train_data, dev_data, test_data = make_dataset(encoder_lines, decoder_lines)\n",
    "model = create_model(len(vocab), 128, MAX_INPUT_LENGTH, 2, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 109600 samples, validate on 13700 samples\n",
      "Epoch 1/100\n",
      "109600/109600 [==============================] - 494s 5ms/step - loss: 0.0206 - val_loss: 0.0192\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.01920, saving model to model.h5\n",
      "Epoch 2/100\n",
      "109600/109600 [==============================] - 524s 5ms/step - loss: 0.0182 - val_loss: 0.0190\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.01920 to 0.01902, saving model to model.h5\n",
      "Epoch 3/100\n",
      "109600/109600 [==============================] - 554s 5ms/step - loss: 0.0181 - val_loss: 0.0192\n",
      "\n",
      "Epoch 00003: val_loss did not improve\n",
      "Epoch 4/100\n",
      "109600/109600 [==============================] - 533s 5ms/step - loss: 0.0181 - val_loss: 0.0195\n",
      "\n",
      "Epoch 00004: val_loss did not improve\n",
      "Epoch 5/100\n",
      "109600/109600 [==============================] - 477s 4ms/step - loss: 0.0182 - val_loss: 0.0193\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/100\n",
      "109600/109600 [==============================] - 454s 4ms/step - loss: 0.0182 - val_loss: 0.0196\n",
      "\n",
      "Epoch 00006: val_loss did not improve\n",
      "Epoch 7/100\n",
      "109600/109600 [==============================] - 513s 5ms/step - loss: 0.0182 - val_loss: 0.0195\n",
      "\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 8/100\n",
      "109600/109600 [==============================] - 557s 5ms/step - loss: 0.0182 - val_loss: 0.0196\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/100\n",
      "109600/109600 [==============================] - 440s 4ms/step - loss: 0.0184 - val_loss: 0.0198\n",
      "\n",
      "Epoch 00009: val_loss did not improve\n",
      "Epoch 10/100\n",
      "109600/109600 [==============================] - 402s 4ms/step - loss: 0.0186 - val_loss: 0.0199\n",
      "\n",
      "Epoch 00010: val_loss did not improve\n",
      "Epoch 11/100\n",
      "109600/109600 [==============================] - 402s 4ms/step - loss: 0.0186 - val_loss: 0.0202\n",
      "\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 12/100\n",
      "109600/109600 [==============================] - 428s 4ms/step - loss: 0.0186 - val_loss: 0.0202\n",
      "\n",
      "Epoch 00012: val_loss did not improve\n",
      "Epoch 00012: early stopping\n",
      "13700/13700 [==============================] - 14s 1ms/step\n",
      "Testset loss after 100 epochs: 0.020\n"
     ]
    }
   ],
   "source": [
    "train(model, train_data, dev_data, test_data, 'model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7242335766423358"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, *test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "BABI_ENCODER_DATA = 'dialogue_denoiser_data/dialog-babi-task1-API-calls-tst.txt/encoder.txt'\n",
    "BABI_DECODER_DATA = 'dialogue_denoiser_data/dialog-babi-task1-API-calls-tst.txt/decoder.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "good uhm yeah good morning \n",
      "good morning\n"
     ]
    }
   ],
   "source": [
    "babi_encoder_lines, babi_decoder_lines = load_dataset(BABI_ENCODER_DATA, BABI_DECODER_DATA)\n",
    "print babi_encoder_lines[0], '\\n', babi_decoder_lines[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model('model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "_, babi_train_data, babi_dev_data, babi_test_data = make_dataset(babi_encoder_lines, babi_decoder_lines, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7462773722627737"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(model, *babi_test_data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}