CNN_for_text_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8tQJd2YSCfWR"
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D7tqLMoKF6uq"
   },
   "source": [
    "CNN for text classification\n",
    "=============\n",
    "<span style=\"color: lightsteelblue;\">Deep Learning</span>\n",
    "\n",
    "The goal of this notebook is to train convolutional neural network over text extracted from public articles at [Lenta.ru](https://lenta.ru) for text classification purposes.\n",
    "\n",
    "**We will use**\n",
    "- embeddings from scratch (we will train them as well)\n",
    "- OR (as an excercise) pretrained embeddings from previous word2vec CBoW model\n",
    "- 3 different filter sizes for convolutions\n",
    "- relu activations\n",
    "- max-pooling\n",
    "- dropout\n",
    "- multinomial logistic classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MvEblsgEXxrd"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5982,
     "status": "ok",
     "timestamp": 1445965582916,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "Mvf09fjugFU_",
    "outputId": "8f75db58-3862-404b-a0c3-799380597390"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input size: 372 articles\n",
      "\n",
      "(10, 35)\n",
      "[['индийский' 'слон' 'кофейный' 'рынок' 'лихорадит' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['к' 'плохим' 'новостям' 'из' 'южной' 'америки' ',' 'гуляющим' 'уже'\n",
      "  'несколько' 'недель' ',' 'на' 'днях' 'добавились' 'тревожные' 'звонки'\n",
      "  'из' 'индии' ',' 'которая' 'является' 'ведущим' 'мировым' 'поставщиком'\n",
      "  'кофе' 'сорта' 'робуста' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>']\n",
      " ['так' ',' 'выяснилось' ',' 'что' 'выпуск' 'кофе' 'в' 'стране' 'в'\n",
      "  'следующем' 'сезоне' 'может' 'упасть' 'до' 'минимума' 'за' 'последние'\n",
      "  'восемь' 'лет' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['причина' 'продолжительная' 'засуха' 'на' 'основных' 'территориях'\n",
      "  'произрастания' 'кофейных' 'деревьев' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['аналитики' 'прогнозируют' ',' 'что' 'производство' 'робусты' 'в' 'новом'\n",
      "  'сельхозгоду' 'может' 'сократиться' 'до' '300' 'тысяч' 'тонн' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>']\n",
      " ['это' 'станет' 'самым' 'низким' 'показателем' 'с' '2010' 'года' ','\n",
      "  'когда' 'урожай' 'индийского' 'кофе' 'составил' '289' ',' '6' 'тысячи'\n",
      "  'тонн' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['по' 'данным' 'индийских' 'метеорологов' ',' 'объем' 'осадков' 'летом'\n",
      "  '2017' 'года' 'в' 'стране' 'был' 'на' '18' 'процентов' 'ниже' 'среднего'\n",
      "  'показателя' ',' 'а' 'в' 'штате' 'карнатака' ',' 'где' 'производится'\n",
      "  '70' 'процентов' 'всего' 'индийского' 'кофе' ',' 'количество' 'осадков']\n",
      " ['учитывая' ',' 'что' '70' 'процентов' 'кофе' 'индия' 'экспортирует' ','\n",
      "  'становится' 'понятна' 'нервная' 'реакция' 'биржевых' 'игроков' 'на'\n",
      "  'информацию' 'за' 'день' 'торгов' 'цены' 'на' 'арабику' 'на' 'бирже' 'в'\n",
      "  'нью' 'йорке' 'подскочили' 'на' '24' 'процента' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['а' 'на' 'лондонской' 'бирже' 'робуста' 'подорожал' 'на' '16' 'процентов'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>']\n",
      " ['при' 'этом' 'участники' 'рынка' 'уже' 'давно' 'играют' 'на' 'повышение'\n",
      "  'и' 'не' 'прекращают' 'это' 'делать' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>']]\n",
      "Number of business articles is 3. For example: \n",
      "Индийский слон Кофейный рынок лихорадит. К плохим новостям из Южной Америки, гуляющим уже несколько недель, на днях добавились тревожные звонки из Индии, которая является веду...\n",
      "Number of culture articles is 6. For example: \n",
      "Если погуглить рекомендательные подборки семейных саг, то на разных литературных сайтах в списке из условных 10 позиций непременно окажется «Сага о Форсайтах» и, надо думать, ...\n",
      "Number of sport articles is 4. For example: \n",
      "Сами себе чемпионы Среди всех участников чемпионата мира на этот раз было лишь 19 обладателей паспорта в бордовой корочке. Хотя формально ни одного россиянина на турнире не бы...\n"
     ]
    }
   ],
   "source": [
    "sequence_length = 35 # the length of a truncated (padded) sentence\n",
    "\n",
    "def clean(string):\n",
    "  string = re.sub(r\"[^А-Яа-я0-9(),!?]\", \" \", string)\n",
    "  string = re.sub(r\",\", \" , \", string)\n",
    "  string = re.sub(r\"!\", \" ! \", string)\n",
    "  string = re.sub(r\"\\(\", \" \\( \", string)\n",
    "  string = re.sub(r\"\\)\", \" \\) \", string)\n",
    "  string = re.sub(r\"\\?\", \" \\? \", string)\n",
    "  string = re.sub(r\"\\s{2,}\", \" \", string)\n",
    "  return string.strip().lower()\n",
    "\n",
    "def read_data(folder):\n",
    "  data = {}\n",
    "  data_files = os.listdir(folder)\n",
    "  for data_file in data_files:\n",
    "    with open(os.path.join(folder,data_file)) as f:\n",
    "      articles = []\n",
    "      for line in f:\n",
    "        articles.append(line)\n",
    "    data[data_file] = articles\n",
    "\n",
    "  return data\n",
    "\n",
    "data = read_data('articles')\n",
    "num_classes = len(data)\n",
    "\n",
    "total_articles_count = 0\n",
    "labels = []\n",
    "data_x = []\n",
    "data_y = []\n",
    "\n",
    "for label_index, label in enumerate(data):\n",
    "  labels.append(label)\n",
    "\n",
    "  for article in data[label]:\n",
    "    sentences = re.split('[\\.]', article)\n",
    "    for sentence in sentences:\n",
    "      words = clean(sentence).split(' ')\n",
    "\n",
    "      if len(words) >= sequence_length:\n",
    "        words = words[:sequence_length]\n",
    "      else:\n",
    "        padding = ['<PAD>'] * (sequence_length - len(words))\n",
    "        words = words + padding\n",
    "\n",
    "      data_x.append(words)\n",
    "      data_y.append(label_index)\n",
    "\n",
    "total_articles_count = len(data_x)\n",
    "\n",
    "print('Total input size: %d articles\\n' % total_articles_count)\n",
    "x_train = np.array(data_x[:10])\n",
    "y_train = np.array(data_y[:10])\n",
    "x_valid = np.array(data_x[11:])\n",
    "y_valid = np.array(data_x[11:])\n",
    "\n",
    "print(x_train.shape)\n",
    "print(x_train)\n",
    "\n",
    "for label in data:\n",
    "  excerpt = data[label][0][:175] + '...'\n",
    "  print('Number of %s articles is %d. For example: \\n%s' % (label, len(data[label]), excerpt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ga2CYACE-ghb"
   },
   "source": [
    "## The model\n",
    "Let's now prepare Convolutional Neural network. We will wrap it up in `ArticlesClassifier` class to make it reuseable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 6184,
     "status": "ok",
     "timestamp": 1445965583138,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "w-oBpfFG-j43",
    "outputId": "bdb96002-d021-4379-f6de-a977924f0d02"
   },
   "outputs": [],
   "source": [
    "class ArticlesClassifier(object):\n",
    "  def __init__(self, sequence_length, num_labels, vocab_size,\n",
    "               embedding_size, filter_sizes, filter_out_channels):\n",
    "    # Placeholders for input, output and dropout\n",
    "    self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name=\"input_x\")\n",
    "    self.input_y = tf.placeholder(tf.float32, [None, num_classes], name=\"input_y\")\n",
    "    self.dropout_keep_prob = tf.placeholder(tf.float32, name=\"dropout_keep_prob\")\n",
    "\n",
    "    # Embeddings layer\n",
    "    with tf.device('/cpu:0'), tf.name_scope(\"embedding\"):\n",
    "      self.embeddings = tf.Variable(\n",
    "          tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0), name=\"embeddings\")\n",
    "      self.embedded_chars = tf.nn.embedding_lookup(self.embeddings, self.input_x)\n",
    "      self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1)\n",
    "\n",
    "    # Convolution and pooling\n",
    "    pooled = []\n",
    "    for i, filter_size in enumerate(filter_sizes):\n",
    "      with tf.name_scope(\"conv-maxpool-%s\" % filter_size):\n",
    "        # Convolution layer\n",
    "        filter_shape = [filter_size, embedding_size, 1, filter_out_channels]\n",
    "        strides = [1, 1, 1, 1]\n",
    "        W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name='W')\n",
    "        b = tf.Variable(tf.constant(0.1, [filter_out_channels]), name='b')\n",
    "        conv = tf.nn.conv2d(self.embedded_chars_expanded, W, strides, 'VALID', name='conv')\n",
    "        # Bias and non-linearity\n",
    "        net = tf.nn.bias_add(conv, b)\n",
    "        h = tf.nn.relu(net, name=\"h\")\n",
    "        # Pooling\n",
    "        h_pooled = tf.nn.max_pool(h, [1, sequence_length - filter_size + 1, 1, 1], 'VALID', 'pool')\n",
    "        pooled.append(h_pooled)\n",
    "\n",
    "    total_channels = len(filter_sizes) * filter_out_channels\n",
    "    self.h_pool = tf.concat(3, pooled)\n",
    "    self.h_pool_flat = tf.reshape(self.h_pool, [-1, total_channels])\n",
    " \n",
    "    # Dropout layer\n",
    "    with tf.name_scope(\"dropout\"):\n",
    "      self.h_drop = tf.nn.dropout(self.h_pool_flat, dropout_keep_prob)\n",
    "\n",
    "    # Output layer (Scores and predictions)\n",
    "    W = tf.Variable(tf.truncated_normal([total_channels, num_classes], stddev=0.1), name='W')\n",
    "    b = tf.Variable(tf.constant(0.1, [num_classes]), name='b')\n",
    "    self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name=\"scores\")\n",
    "    self.predictions = tf.argmax(self.scores, 1, name=\"predictions\")\n",
    "\n",
    "    # Loss and accuracy\n",
    "    with tf.name_scope(\"loss\"):\n",
    "      losses = tf.nn.softmax_cross_entropy_with_logits(logits=self.scores, labels=self.input_y)\n",
    "      self.loss = tf.reduce_mean(losses, name='loss')\n",
    "    with tf.name_scope('accuracy'):\n",
    "      bool_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1))\n",
    "      self.accuracy = tf.reduce_mean(tf.cast(bool_predictions, 'float'), name='accuracy')\n",
    "\n",
    "  def train(self, input_x, input_y):\n",
    "    feed_dict = {\n",
    "      input_x: input_x,\n",
    "      input_y: input_y,\n",
    "      dropout_keep_prob: self.dropout_keep_prob      \n",
    "    }\n",
    "    optimizer = tf.AdamOptimizer(1e-4)\n",
    "    optimizer.train(self.loss, feed_dict=feed_dict)\n",
    "\n",
    "  def predict(self, input_x):\n",
    "    feed_dict = {\n",
    "      input_x: input_x,\n",
    "      dropout_keep_prob: 1.0\n",
    "    }\n",
    "    session.run([self.predictions], feed_dict=feed_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "tuple index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-dbf9654d4d69>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     model = ArticlesClassifier(\n\u001b[0;32m----> 9\u001b[0;31m       \u001b[0msequence_length\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m       \u001b[0mnum_classes\u001b[0m     \u001b[0;34m=\u001b[0m \u001b[0mnum_classes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m       \u001b[0mvocab_size\u001b[0m      \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvocabulary\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: tuple index out of range"
     ]
    }
   ],
   "source": [
    "with tf.Graph().as_default():\n",
    "  session_conf = tf.ConfigProto(\n",
    "    allow_soft_placement=True,\n",
    "    log_device_placement=True)\n",
    "  sess = tf.Session(config=session_conf)\n",
    "  with sess.as_default():\n",
    "\n",
    "    model = ArticlesClassifier(\n",
    "      sequence_length = x_train.shape[1],\n",
    "      num_classes     = num_classes,\n",
    "      vocab_size      = len(vocabulary),\n",
    "      embedding_size  = 128,\n",
    "      filter_sizes    = [3, 4, 5],\n",
    "      num_filters     = 2\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "6_lstm.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}