seminar_02/embeddings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intro\n",
    "\n",
    "\n",
    "_Word embedding_ is a mapping of words (or phrases) from the vocabulary to vectors of real numbers.\n",
    "\n",
    "### Suggested readings:\n",
    "* To understand better Skip-Gram Model the following tutorial is suggested\n",
    " - [Word2Vec Tutorial - The Skip-Gram Model](http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/)\n",
    "\n",
    "* To know about more complex and effective implementations of word2vec models see\n",
    "\n",
    " - [Distributed Representations of Words and Phrases and their Compositionality](http://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Everything</th>\n",
       "      <th>She</th>\n",
       "      <th>Sky</th>\n",
       "      <th>better</th>\n",
       "      <th>blue</th>\n",
       "      <th>getting</th>\n",
       "      <th>is</th>\n",
       "      <th>possible</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Everything</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>She</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sky</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>better</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>blue</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getting</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>is</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>possible</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Everything  She  Sky  better  blue  getting  is  possible\n",
       "Everything         1.0  0.0  0.0     0.0   0.0      0.0   1       1.0\n",
       "She                0.0  1.0  0.0     1.0   0.0      1.0   1       0.0\n",
       "Sky                0.0  0.0  1.0     0.0   1.0      0.0   1       0.0\n",
       "better             0.0  1.0  0.0     1.0   0.0      1.0   1       0.0\n",
       "blue               0.0  0.0  1.0     0.0   1.0      0.0   1       0.0\n",
       "getting            0.0  1.0  0.0     1.0   0.0      1.0   1       0.0\n",
       "is                 1.0  1.0  1.0     1.0   1.0      1.0   1       1.0\n",
       "possible           1.0  0.0  0.0     0.0   0.0      0.0   1       1.0"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from collections import defaultdict\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "# Lets for example consider a simple way to map words from sentences into dense vectors.\n",
    "# Lets make a table with words coocurrencies and then project vectors of all words into 2D using PCA.\n",
    "\n",
    "s = ['Sky is blue', 'She is getting better', 'Everything is possible']\n",
    "dic = defaultdict(dict)\n",
    "for sent in s:\n",
    "    words = sent.split()\n",
    "    for w in words:\n",
    "        for w2 in words:\n",
    "            dic[w][w2]=1\n",
    "\n",
    "df = pd.DataFrame(dic)\n",
    "df.fillna(0, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdQAAAHFCAYAAABLrQhSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHNpJREFUeJzt3XuUnXV97/H3J4mAeIFDAoKoIBaWVYutVRS1HlhFsV5A\n8dZSW0KrVAt4VuVAq1UBtVqtS2vFGxUNllK5VEHFIwoVLaUWAhZQKyICSiwkgHIPkOR7/nieqXs2\nM8nM5LczM+H9WmuvzH6uv8li8c5z2c9OVSFJkjbOgtkegCRJmwODKklSAwZVkqQGDKokSQ0YVEmS\nGjCokiQ1YFCnKUklOWLg/QVJztzAOrv2671k9COUJM2GRbM9gHlob+Da2R6EJGluMajTVFXfnu0x\nSJLmnnlzyjfJsiTLk7wsyQ+SrE5yYZInDSyzdZK/S3JjP/+SJC8Y2s5zk/xrktv7138medXA/AOS\nXJrkriQ/T/IfSf73wPxxp3wHph+W5Lok9yQ5J8nOU/idXpfke0nuTXJ9kmNm/jckSZpN8yaovV2A\nDwLvAg4GtgHOTbJVP//vgUOBvwJeDvwUOCfJcwGSPBL4MvBj4BXAK4F/ALbt5z8BOBP4F+ClwO/3\ny2+3gXHtDRwJvBn4Y2BP4Kz1rZDkaODj/XIv6X9+10SxliTNffPtlO8S4MCqugggyaXANcDSJN8E\nfg84tKpO7uefC1wBvB3YH9iDLsJHVNUd/Ta/NrD93wDuqKqjB6Z9ZQrj2gHYu6p+0u/3euDCJC+s\nqq8OL9yH/Vjg3VV1fD/560m2Bt6W5ONVtXYK+5UkzRHz7Qh15VhMAarqeuBSYC/gGUCAMwbmr+vf\nP7efdA1wJ3BqkgOTbDu0/SuBbZKcnOQFSR42xXFdNhbTfr//BqzsxzWRvYGHAWckWTT2ojsyfhTw\nmCnuV5I0R8y7oE4ybaf+dWdV3T00/yZg6yRbVtXPgecDDwFOB1b11zt3A6iqq4ADgd3ojkxvTnJq\nku03YlwTWdL/+T3g/oHXN/rpj93A/iRJc8x8C+oOk0z77/718P606aBHAXdX1b3Q3aVbVS+ku256\nEN1p4FPHFq6qc6rqt4DFdNdD9wM+shHjmsit/Z8voTuyHn5dvoH9SZLmmHkX1CTPHnuT5HHA04CL\ngUuAorvRaGx++vcXDm+oqu6pqi8BnwaeNMH826rqVOALE80f8rR+LGP7fQ5dUC+eZPl/B+4BHl1V\nyyd43THJepKkOWq+3ZR0M3BKkrfRBel4ulOry6pqdZJ/Ak5I8gi666WvB54IvBEgyYuBP6K7s/Yn\nwM7An9BduyTJn9Bd3/wq8DNgd+BVwGc3MK5VdHcTHwtsBbyP7rrqA25IAqiqXyQ5Dvhwkl2Ab9H9\n42YPYN+qevk0/14kSbNsvgX1euA9wF/TfYRmOXBwVa3u57+eLmbvoDuleyXwkqoaO0L9Ed1R7Hvo\njiBX0X0s5q39/CuAA+g+mrMd3Snbv++3tz4XAecBfwtsD1wAHLa+Farq/Ul+BvwZcBSwGvghcNoG\n9iVJmoNSVbM9hilJsgx4SlU9fbbHIknSsPl2DVWSpDnJoEqS1MC8OeUrSdJc5hGqJEkNGFRJkhow\nqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBB\nlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyq\nJEkNGFRJkhqYUlCTHJekJnm9dtSDnKokxyTZZ4LpleSIDay7tF/u4SMboCRps7VoGsveBrxwguk/\najSWFo4BTgAumMG65wB7A3e3HJAk6cFhOkFdU1XfHtlIJpHkoVV1z6j3U1WrgFWj3o8kafPU5Bpq\nkmuT/M0E089IcuHA++2SnJjkpiSrk1yU5JlD61SSNyf52ySrgCuT/GmSO4dPxybZp1/+qUmuAxYD\nxw6cjt5nYPGFSd6TZFWSlUk+mmTLgW2NO+WbZNf+/auTfDLJbUluSHJ8kgVD43hVkquT3JPkG0l+\no1936Yz/UiVJ88q0gppk0fCrn3U68KqhZR8OvBj4XP9+S+A8YD/gaOBldEeE5yXZcWhXRwM7AX8A\nvAk4FVgIvHJouUOBy6rqcuDldKelT6I7dbs3cNnAskcBjwZeC/wN8CfA/5nCr/1+4M5+36cA7xgc\nR5Kn97/jZf0YvgicNoXtSpI2I9M55bsYuH94YpLH0wXlmCTPGjgt/FJgC+CM/v1rgacAT66qq/t1\nzwOuoovd0QOb/e+qes3Qfv6ZLqDL+vcPB14B/AVAVX0nyRrghklOTV9XVUv7n89N8hzgILpgrs+3\nquqo/uevJ3lhv97p/bQ/B/4L+N2qKuCrSR4CvG8D25UkbUamc4R6G/CMCV4/q6rvAD8EBiP4GuCb\nVXVT/34/4FLg2qGj228CTx/a11cm2P9JwG8l2a1//2q6fxCcOsXxf23o/feBxzRY7xnAl/qYjvni\nFMckSdpMTPempOXrmX8a8EdJ3gw8gu6O4CMH5i8BnsUER7nANUPvb5pgmQuAHwNL6U67HgqcXVW3\nTmXwwC+G3t8HbNVgvR154M1M3twkSQ8y0wnqhpwGvB14LvB4uqPfzw/MvxVYDrxxgnXvHXpfwwtU\nVSX5NHBYklP6/fxOg3FvrBuB7YemDb+XJG3mmgW1qr6X5Lt0p3ofD5xXVbcMLHI+8ALgJ1W1coa7\nWQa8k+707wrg60Pzp3rU2dIlwEuTvHXgtO8Bm3gMkqRZNp2gLkryrAmm/7SqVvQ/n0Z35+w2wOuH\nlvss8AbggiQfoDt9uxjYC7ixqj60oQFU1c+SfJXu7uH3VtXaoUV+ALy4X+ZO4KqqumNqv96MvQ/4\nD+BzST4D/Cq//N3XjXjfkqQ5Yjo3JW0D/PsEr0MHlvkc3bXSdcBZgytX1WpgX7qjyuPpbvb5MLA7\ncPE0xjG23c9MMO9o4C66px5dAvzmNLY7I/115d/r93UW3Z3HY6e1bx/1/iVJc0PG35w69yU5Hdip\nqn5rtscymf75xv/wTyc9hu13eQiL7oXrrt/9ikPeeO5TZ3tskqTRmDffNpPk15IcSvcZ0A/P9ngG\nJfl4koOS7JvkqK22WrjsWc98KNvvugUkrNkqPO5Xrt7z5I/vf/lsj1WSNBot7/IdtS/RnU7+WFWd\nOduDGbIY+Fj/5y3P33frha974+JxC6xbGHbd5eo9Z2NwkqTRmzdBrapdZ3sMk6mqVw++P//83Yrk\nAcut2fIBkyRJm4l5c8p3Plk0/KnaDUyXJM1/BnUErrt+9ysWrB1/s9eCtcV11+9+xSwNSZI0YvPu\nLt/54uSP73/5rrtcveeaLfEuX0l6EDCokiQ14ClfSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJ\nUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmS\nGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLU\ngEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQG\nDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVg\nUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGD\nKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhU\nSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBK\nktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWS\npAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIk\nNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSp\nAYMqSVIDBlWSpAYMqiRJDRhUSdImleSwJC+bYPoxSfaZYHolOWKTDG4jGFRJ0qZ2GPCAoALHAPtM\nMH1v4IxRDqiFRbM9AEmS1qeqvj3bY5gKj1AlSVOW5IgkP01yV5Kzkvx2f0p2n37+giR/keRHSe5N\n8sMkhwysfwHwm8Ah/XqVZGmS64DFwLED08e2Oe6Ub5ILkpyZ5OB+P7cn+X9JHjM01sf10+9Jcm2/\nnzP7MTTnEaokaUqSvBz4CPAx4GzgucBJQ4t9BDgEeCdwGfB84NNJbqmqLwN/Cvwz8GPgXf061wCX\nA98AzgQ+1U///nqG80zg0cBRwEOBDwMnAi/qxxrgi8C2wB8Bq4G3A9v3+2vOoEqSpuqtwFeq6vD+\n/deSLAHeCJDkV/qfD62qk/tlzkuyE3As8OWq+n6Su4BVQ6dyVyVZA9wwxVO8jwReXFU/7/e9I/Ch\nJA+tqnvowvpUYK+quqRf5mLgOkYUVE/5SpI2KMki4DfojvoGDb7/bWAd8IUki8ZewPnArydZ2HBI\nl4zFtDd2NLtz/+czgBvHYgpQVSuASxuOYRyPUCVJU7EEWAisGpq+aoJlbptkGzsBNzQazy+G3t/X\n/7lV/+eOPHCs9NMe0WgM4xhUSdJU3AyspbsGOWjw/a3AGuA5dEeqw1aOZmgTupEHjpV+2upR7NBT\nvpKkDaqqNcB3gAOHZh0w8PO/0B2hblNVyyd4jR1F3scvjyQHTTZ9Ji4Bdkyy19iEJDvT3WE8Eh6h\nSpKm6r3APyc5ge7a6XOAF/fz1lXVVUk+AXwuyfuB5XSBfDKwR1W9rl/2B8D+SfYHbgGurapb+ukv\nTvJV4E7gqqq6Y4Zj/QrdncOnJ3kLcA/djVE3MfHR80bzCFWSNCVV9XngTXRPOTqL7saf/9vPvr3/\n83C6j8P8IV3UltFF91sDm3o38F/A6XRHki/tpx8N3AWc00+f8dFkVRXd0fQPgM/Qfazm43Q3L92+\nnlVnLN0+JUmauiTLgKfQhfUvge36j6tMdzuHASur6qyh6ccAF1fVBRs/2v/Z5jZ0n389oaqObbXd\nMZ7ylSRNSZLtgbfQPYBhJ7oHK/wlcNJMYto7DPguXZgHHQOcAFwww+2S5A10p3evprsZ6c3AlsCn\nZ7rN9TGokqSpug94It3p3O2A+4EP0T2BaC5aDfw5sAtQwMXAflV1/Sh25jVUSdKUVNVtVfWiqloC\nfBa4Evg2cGWS1UkuTPKkseVH9Fzf9W5zbLtJzgS26F9bAk+sqn1H+aB9j1AlSTO1C/BBuiPUe4Dj\ngXOT7F5VqxnNc303tM0xzwGeQHeEejeTP2yiGYMqSZqpJcCBVXURQJJL6YK4NMl5NH6u71SeFTyw\njW2BX6+qm1r/0pPxlK8kaaZWjsUUoL82eSmwF6N5ru90tnnppowpeIQqSZq5iR4luJLuDuBRPNd3\nOtvcpDEFgypJmrkdJpn2PUbzXN/pbHOTP2TBoEqSZmqHJM8euIb6OOBpdE8m+ga/fK7v19ezjek8\n13fwWcHr2+asMKiSpJm6GTglydv45V2+K4FlVbV6BM/1neqzgmeFQZUkzdT1wHuAv6b7CM1y4OD+\nIzPQPdf3h8Dr6T7mcjvdx19OGtjGu4HH0T3X95HAoXTP/z0a+Cjdc323Bvale2rSVLY5K3yWryRp\nk0qyFDgS2IPumuh1wDeq6s39/F2Ba4GXDn22dE7zYzOSpizJsiTLZ3scmr/6r1L7FHAucBDdYwzP\nZvz3qs5LHqFKmrIkTwAeWlXfne2xaH5KsgI4q6oOH5qe/ivXPEKVtPmrqmuMqTbStsCNwxNr4qO7\nrZN8MsltSW5IcnyScd1K8pQk5yS5o3+dkWTHUQ1+fQyqpCkbPOWbZNskn0rys/7B6D9J8vezPUbN\neZcBRyY5JMniDSz7fro7fF8JnAK8o/8Z+J9HEf4b3Z2+rwWW0t3x+6UkaT/09fMuX0kz9UHg2cCf\n0R1xPBZ43qyOSPPB4XTffboMqCT/RfeA/A9U1e1Dy36rqo7qf/56khfSXXc9vZ92LN1/e79TVfcB\nJLmC7iM3L6K7Q3iT8QhV0kztBXy0qk6rqm9W1SlVddhsD0pzW1VdAfwq3U1IHwNC9201y5M8fGjx\nrw29/z7wmIH3+wFfANYNPNf3Wrq7hp/efvTrZ1AlzdR/Akcn+dMke8z2YDR/VNW9VfWlqjqiqp4E\nvA7YHfjjoUV/MfR++OlJS+i+nu3+oddudGdMNimDKmmmjqA7dfcO4KokVyf53Vkek+ahqjqJ7jm9\nT5zmqrcCnwSeMcHr3S3HOBVeQ5U0I1X1C+BNwJuS7AkcA/xjkiuq6vvrX1sPVkl2qKqVQ9O2B7Zh\n+t8Qcz7dTUiXTnKX8CblEaqkjdZfFzua7v8p0z3K0IPLlUlOTPLKJM9L8gfAecDdwMkbWHfYccCv\nAef029snye/3d6Pv03bYG+YRqqQZSXIh3Q0h36X7qqzXA3cBF8/muDTnvRM4EPg7YDu6u3QvAl5T\nVddOZ0NV9cMkz6I7vXsi8FBgBd2R649aDnoqfFKSpClLsgx4SlU9PcnfAC8EdgXWAt8B3lFV/zp7\nI5Rmj0GVJKkBr6FKktSAQZUkqQGDKklSAwZVkqQG/NiMpGYOftsJJ+y88LY3bM39C+/mIWtXrN3m\nE6e++4gjZntc2vy97y3/uOrAtTsveVgWcFet4+yFK27+8/f+/vabcgze5SupiYPfdsIJuy285fBF\n+eX/U9ZU+PHaxR81qhql973lH1e9Zt1jlywa+Ma2NVWctuCnmzSqnvKV1MTOC297w2BMARal2Hnh\nbW+YpSHpQeLAtTuPiynAooQD1+68ZFOOw6BKamJr7l84nelSKw/LxCmbbPqoGFRJTdzNQ9ZOZ7rU\nyl21blrTR8WgSmpixdptPrGmxp92W1NhxdptPjFLQ9KDxNkLV9y8Zuh+oDVVnL1wxc2bchzelCSp\nGe/y1WzxLl9JkjYTnvKVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUk\nqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJ\nDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlq\nwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVID\nBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhow\nqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBB\nlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJUgMGVZKkBgyq\nJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCokiQ1YFAl\nSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGVJKkBgypJ\nUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmS\nGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLU\ngEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQe0mWJVm+gWUqyRGbakySpPnDoEqS1IBB\nlSSpAYM6JMnLkvwgyeokFyZ50nqWvS7JB4amLe1PDT98YNp2SU5MclO/3YuSPHOUv4ckadMyqOPt\nAnwQeBdwMLANcG6SrWa6wSRbAucB+wFHAy8DVgHnJdlxo0csSZoTFs32AOaYJcCBVXURQJJLgWuA\npcAnZrjN1wJPAZ5cVVf32z0PuAo4ii6ykqR5ziPU8VaOxRSgqq4HLgX22oht7tdv49oki5KM/SPm\nm8DTN2K7kqQ5xCPU8VZOMm2njdjmEuBZwP0TzLtmI7YrSZpDDOp4O0wy7XuTLL8a2GJo2v8aen8r\nsBx44wTr3zut0UmS5iyDOt4OSZ49cA31ccDTgM9MsvwNwK8OTXvB0Pvz+2k/qaqJjoAlSZsBgzre\nzcApSd4G3AMcT3fKd9kky38B+EiStwKXAK8Anjy0zGeBNwAX9B+x+TGwmO667I1V9aHWv4QkadMz\nqONdD7wH+Gu6j9AsBw6uqtWTLH8i8ATgTcCWdPF8N/DJsQWqanWSfYF30gX6UXSRvhj44mh+DUnS\nppaqmu0xzClJlgJHAnsAa4DrgG9U1Zv7+bsC1wIvraovz8ogJUlzjh+bGZDkLcCngHOBg4A/BM4G\nDpjNcUmS5j6PUAckWQGcVVWHD01P9X9RHqFKkibiEep42wI3Dk+sDfyrI8m+Se5I8p7+ub2r+1PH\ng8skyY+TeBOSJG2GDOp4lwFHJjkkyeKprJBkf+Ac4P1V9daqupXu7t+lQ4vuAzwe+HS74UqS5gqD\nOt7hwJ10H5NZleR7Sd6Z5JETLZzkALprrO+oqncNzDoJeF6S3QamHQpcWlVXjmbokqTZZFAHVNUV\ndA9qOAD4GBDg7cDywa9j670COAM4qqo+MDTvfLqP4BwCkOQR/fKTPSBCkjTPGdQhVXVvVX2pqo6o\nqicBrwN2B/54aNED6B4r+IUJtlF08TwkSYBXAwuBU0c6eEnSrDGoG1BVJ9GF84lDs44EVgBfm+R6\n62eAxwL70l1PPauqfj7CoUqSZpFBHZDkAQ/HT7I93ReN3zQ063Zg//7nc4evs1bVT4Gv0T0d6bl4\nuleSNmsGdbwrk5yY5JVJnpfkD4DzgLuBk4cXrqpbgOfTfdzmy0m2HlrkJLqY3gB8fbRDlyTNJoM6\n3juBXYG/ozu6fBfdV7ftVVXXTrRCVf038Nv9ep9PMvh1bl+me3zhyVW1bnTDliTNNp+UNEJJXkQX\n1T2q6kezPR5J0ugY1BFI8mi6O4M/Qvc9qC+Z5SFJkkbMU76jcRjdZ1FX090NLEnazHmEKklSAx6h\nSpLUgEGVJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqYNFsD2BzddxBSy9fzE173rdo\nAVusWcctPOqK4z6/7KmzPS5J0mj4pKQROO6gpZdvs3DlnusW/PIEwIJ167ht7Q5GVZI2U57yHYHF\n3DQupgDrFixgMTftOUtDkiSNmEEdgfsWTfzXOtl0SdL85//hR2CLNRN/l/hk0yVJ859BHYFbeNQV\nC9aNj+eCdd2NSbM0JEnSiBnUETju88ueetvaHa7Y4v61UMUW96/1hiRJ2sx5l68kSQ14hCpJUgMG\nVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKokSQ0YVEmSGjCo\nkiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIaMKiSJDVgUCVJasCgSpLUgEGV\nJKkBgypJUgMGVZKkBgyqJEkNGFRJkhowqJIkNWBQJUlqwKBKktSAQZUkqQGDKklSAwZVkqQGDKok\nSQ0YVEmSGjCokiQ1YFAlSWrAoEqS1IBBlSSpAYMqSVIDBlWSpAYMqiRJDRhUSZIa+P/E709Dx0KT\niAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7f78d4bb70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "res = PCA().fit_transform(df)\n",
    "\n",
    "font = {'size'   : 15}\n",
    "plt.rc('font', **font)\n",
    "plt.figure(figsize=(7,7))\n",
    "plt.scatter(res[:,0], res[:,1])\n",
    "plt.axis('off')\n",
    "for i, label in enumerate(df.columns):\n",
    "    x, y = res[i,0], res[i,1]\n",
    "    plt.scatter(x, y)\n",
    "    annot = {'has': (1, 50), 'is': (1, 5)}\n",
    "    plt.annotate(label, xy=(x, y),\n",
    "                 xytext=annot.get(label,(1+i*2, 6*i)), \n",
    "                 textcoords='offset points',\n",
    "                   ha='right', va='bottom', )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# word2vec: skip gram & cbow\n",
    "\n",
    "Models __CBOW (Continuous Bag of Words)__ and __Skip gram__ were invented in the now distant 2013,\n",
    "*article*:\n",
    "[*Tomas Mikolov et al.*](https://arxiv.org/pdf/1301.3781v3.pdf)\n",
    "\n",
    "* __CBOW__ model predict missing word (focus word) using context (surrounding words).\n",
    "* __skip gram__ model is reverse to _CBOW_. It predicts context based on the word in focus.\n",
    "\n",
    "* **Context** is a fixed number of words to the left and right of the word in focus (see picture below). The length of the context is defined by the \"window\" parameter.\n",
    "\n",
    "![context](pics/context.png)\n",
    "\n",
    "Two models comparision\n",
    "\n",
    "![architecture](pics/architecture.png)\n",
    "\n",
    "___\n",
    "\n",
    "There are a lot of implementations of word2vec e.g.[gensim](https://github.com/RaRe-Technologies/gensim/blob/develop/docs/notebooks/word2vec.ipynb).\n",
    "And there are a lot of trained word-vectors which are already ready to use.\n",
    "___\n",
    "\n",
    "\n",
    "### Skip_gram\n",
    "\n",
    "Consider a corpus with a sequence of words $ w_1, w_2, .., w_T $.\n",
    "\n",
    "Objective function (we would like to maximize it) for _skip gram_ is defined as follow:\n",
    "\n",
    "\n",
    "$$ AverageLogProbability = \\frac{1}{T} \\sum_{t=1}^{T} \\sum_{-c \\leqslant j\\leqslant c, j \\neq 0} log\\ p (w_{t+j} | w_t) $$\n",
    "\n",
    "* where $ c $ is a context length.\n",
    "* $w_t$ -- focus word\n",
    "\n",
    "The basic formulation for probability $ p (w_{t+j} | w_t) $ is calculated using __Softmax__ -\n",
    "\n",
    "$$ p (w_h | w_i) = \\frac{exp(s(v_i, v_h))}{ \\sum^{W}_{w=1}  exp(s(v_{w}, v_{i} )) } $$\n",
    "\n",
    "where\n",
    "* $w_i$ -- input focus word\n",
    "* $w_h$ -- hypothetically context word for a given focus word $w_i$\n",
    "* $v_i$ and $v_h$ input-word and hypothesis-word vector representations (for $w_i$, $w_h$)\n",
    "* $s(v_i, v_h) = v^{T} _{h} \\cdot v_{i}$\n",
    "* $W$ is the number of words in vocabulary\n",
    "\n",
    "___\n",
    "\n",
    "### CBOW\n",
    "\n",
    "Predict word using context.\n",
    "\n",
    "$$ E = -log\\ p(w_h\\ |\\ w_{1},\\ w_{2},\\ \\dots,\\ w_{c}) $$\n",
    "\n",
    "\n",
    "The **probability** is the same as in the *skip gram* model, but now $v_i$ is a sum of context-word vectors.\n",
    "\n",
    "$$ p(w_h\\ |\\ w_{1},\\ w_{2},\\ \\dots,\\ w_{c})  = \\frac{exp(s(v_i, v_h))}{\\sum^{W}_{w=1}  exp(s(v_{w}, v_{i}))} $$\n",
    "\n",
    "\n",
    "* $\\ w_{1},\\ w_{2},\\ \\dots,\\ w_{c}$ -- input context words\n",
    "* $w_h$ -- hypothetically focus word for a given context words\n",
    "* $ v_i = \\sum^{c}_{k=1} w_{k}$\n",
    "* $ v_h$ = vector of hypothesis word\n",
    "* $s(v_i, v_h) = v^{T} _{h} \\cdot v_{i}$\n",
    "* $W$ is the number of words in vocabulary\n",
    "\n",
    "___\n",
    "\n",
    "Lets implement __`CBOW`__ using tf framework.\n",
    "\n",
    "And then implement __`skip gram`__ using CBOW implementation as an example.\n",
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using [text8 dataset](http://mattmahoney.net/dc/textdata),\n",
    "\n",
    "It's a 100 Mb dump of English Wiki at the time of March 3, 2006.\n",
    "\n",
    "# Working with data\n",
    "\n",
    "It's not so interesting to explore this code.\n",
    "\n",
    "Truly, all the dirty work is done for you.:3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data at /home/cerdio/OneDrive/Courses/NLP_224/deep-nlp-seminars/seminar_02/data/text8.zip.\n"
     ]
    }
   ],
   "source": [
    "# WARNING! if this file \"./data/text8.zip\" doesn't exist\n",
    "# it will be downloaded right now.\n",
    "\n",
    "import os, urllib.request\n",
    "def fetch_data(url):\n",
    "    \n",
    "    filename = url.split(\"/\")[-1]\n",
    "    datadir = os.path.join(os.getcwd(), \"data\")\n",
    "    filepath = os.path.join(datadir, filename)\n",
    "    \n",
    "    if not os.path.exists(datadir):\n",
    "        os.makedirs(datadir)\n",
    "    if not os.path.exists(filepath):\n",
    "        urllib.request.urlretrieve(url, filepath)\n",
    "    \n",
    "    return filepath\n",
    "\n",
    "url = \"http://mattmahoney.net/dc/text8.zip\"\n",
    "filepath = fetch_data(url)\n",
    "print (\"Data at {0}.\".format(filepath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_size = 17005207\n"
     ]
    }
   ],
   "source": [
    "# Unzip and read data\n",
    "\n",
    "import os, zipfile\n",
    "\n",
    "def read_data(filename):\n",
    "    with zipfile.ZipFile(filename) as f:\n",
    "        data = tf.compat.as_str(f.read(f.namelist()[0])).split()\n",
    "    return data\n",
    "\n",
    "\n",
    "words = read_data(filepath)\n",
    "print(\"data_size = {0}\".format(len(words)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Only N = 50000 the most frequent words is considered\n",
    "# The other marked with token `UNK` (unknown)\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "def build_dataset(words, vocabulary_size):\n",
    "    count = [[ \"UNK\", -1 ]]\n",
    "    count.extend(Counter(words).most_common(vocabulary_size-1))\n",
    "    print(\"Least frequent word: \", count[-1])\n",
    "    word_to_index = { word: i for i, (word, _) in enumerate(count) }\n",
    "    data = [word_to_index.get(word, 0) for word in words] # map unknown words to 0\n",
    "    unk_count = data.count(0) # Number of unknown words\n",
    "    count[0][1] = unk_count\n",
    "    index_to_word= dict(zip(word_to_index.values(), word_to_index.keys()))\n",
    "    \n",
    "    return data, count, word_to_index, index_to_word\n",
    "\n",
    "vocabulary_size = 50000\n",
    "data, count, word_to_index, index_to_word = build_dataset(words, vocabulary_size)\n",
    "\n",
    "# Everything you need to know about the dataset\n",
    "\n",
    "print(\"data: {0}\".format(data[:10]))\n",
    "print(\"count: {0}\".format(count[:10]))\n",
    "print(\"index_to_word: {0}\".format(list(index_to_word.items())[:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import deque\n",
    "\n",
    "def generate_batch(data_index, data_size, batch_size, bag_window):\n",
    "    span = 2 * bag_window + 1 # [ bag_window, target, bag_window ]\n",
    "    batch = np.ndarray(shape = (batch_size, span - 1), dtype = np.int32)\n",
    "    labels = np.ndarray(shape = (batch_size, 1), dtype = np.int32)\n",
    "    \n",
    "    data_buffer = deque(maxlen = span)\n",
    "    \n",
    "    for _ in range(span):\n",
    "        data_buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % data_size\n",
    "        \n",
    "    for i in range(batch_size):\n",
    "        data_list = list(data_buffer)\n",
    "        labels[i, 0] = data_list.pop(bag_window)\n",
    "        batch[i] = data_list\n",
    "        \n",
    "        data_buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % data_size\n",
    "    return data_index, batch, labels\n",
    "\n",
    "\n",
    "print(\"data = {0}\".format([index_to_word[each] for each in data[:16]]))\n",
    "data_index, data_size, batch_size = 0, len(data), 4\n",
    "for bag_window in [1, 2]:\n",
    "    data_index, batch, labels = generate_batch(data_index, data_size, batch_size, bag_window)\n",
    "    print(\"bag_window = {0}\".format(bag_window))\n",
    "    print(\"batch = {0}\".format([[index_to_word[index] for index in each] for each in batch]))\n",
    "    print(\"labels = {0}\\n\".format([index_to_word[each] for each in labels.reshape(4)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now just take a close look at the output.\n",
    "\n",
    "* We just want to implement _CBOW_, and therefore missed words are considered as the `labels`.\n",
    "\n",
    "* Remember about the __window__ parameter discussed above, here it is __`bag_window`__.\n",
    "\n",
    "* Each sample in the batch has a number of words equal to __`bag_window * 2`__\n",
    "___\n",
    "\n",
    "# CBOW architecture\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "# define constants\n",
    "batch_size = 128\n",
    "embedding_size = 64\n",
    "\n",
    "# How many words to consider from each side\n",
    "bag_window = 2\n",
    "\n",
    "tf.reset_default_graph()\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    #\n",
    "    # Take the vectors for the context words, which are all bag_window * 2\n",
    "    train_data = tf.placeholder(tf.int32, [batch_size, bag_window * 2])\n",
    "    # Label -- is a word in focus\n",
    "    train_labels = tf.placeholder(tf.int32, [batch_size, 1])\n",
    "    \n",
    "    # Create an embedding matrix\n",
    "    # and initialize it by sampling from the uniform distribution [-1, 1]\n",
    "    \n",
    "    embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    \n",
    "    # Get vectors corresponding to the indices of context words\n",
    "    # embed is a matrix with shape [batch_size, bag_window * 2, embedding_size]\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_data)\n",
    "    \n",
    "    # Sum up all the context vectors to the one vector with the same dimension\n",
    "    # Here we got a matrix of such vectors with the shape [batch_size, embedding_size]\n",
    "    context_sum = <your code here>\n",
    "    \n",
    "    # s finction from theory above\n",
    "    scores = <use tf.matmul to compute scores for context_sum and embeddings>\n",
    "    \n",
    "    one_hot_labels = <make labels one-hot; use tf functions>\n",
    "    loss_tenosor = <implement softmax loss with cross entropy>\n",
    "    loss = tf.reduce_mean(<your loss here>)\n",
    "\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(0.01).minimize(loss)\n",
    "    \n",
    "    # We need to normalize word embeddings for dot product to be a cosine distance \n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims = True))\n",
    "    normalized_embeddings = embeddings / norm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optional\n",
    "\n",
    "You may use [NCE(Noise Contrastive Estimation)](https://www.tensorflow.org/api_docs/python/tf/nn/nce_loss) loss instead of softmax with cross entropy.\n",
    "\n",
    "Just try to make some experiments.\n",
    "\n",
    "Using NCE will accelerate training at times.\n",
    "\n",
    "For more details see the [original article](http://papers.nips.cc/paper/5165-learning-word-embeddings-efficiently-with-noise-contrastive-estimation.pdf)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Let's train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "num_steps = 50000\n",
    "loss_every_nsteps = 200\n",
    "from IPython.display import clear_output\n",
    "\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    try:\n",
    "        tf.global_variables_initializer().run()\n",
    "        print('Initialized')\n",
    "        average_loss = 0\n",
    "\n",
    "        for step in range(num_steps):\n",
    "            data_index, batch, labels = generate_batch(data_index, data_size, batch_size, bag_window)\n",
    "            feed_dict = { train_data: batch, train_labels: labels }\n",
    "            _, current_loss = sess.run([optimizer, loss], feed_dict = feed_dict)\n",
    "            average_loss += current_loss\n",
    "            if step % loss_every_nsteps == 0:\n",
    "                if step > 0:\n",
    "                    average_loss = average_loss / loss_every_nsteps\n",
    "                    clear_output(True)\n",
    "                    print (\"step = {0}, average_loss = {1}\".format(step, average_loss))\n",
    "                    average_loss = 0\n",
    "    except KeyboardInterrupt:\n",
    "        final_embeddings = normalized_embeddings.eval()\n",
    "    final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization\n",
    "\n",
    "[Use projector](http://projector.tensorflow.org/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use these files for Projector\n",
    "\n",
    "with open('embeddings.txt', 'w') as f:\n",
    "    for n in range(vocabulary_size):\n",
    "        s = '\\t'.join([index_to_word[n]] + [str(num) for num in final_embeddings[n]])\n",
    "        f.write(s + '\\n')\n",
    "with open('mentadata.txt', 'w') as f:\n",
    "    for n in range(vocabulary_size):\n",
    "        f.write(index_to_word[n] + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Or see TSNE here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "num_points = 250\n",
    "\n",
    "tsne = TSNE(perplexity=10, n_components=2, init=\"pca\", n_iter=5000)\n",
    "two_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])\n",
    "\n",
    "plt.figure(figsize=(15,15))\n",
    "words = [index_to_word[i] for i in range(1, num_points+1)]\n",
    "\n",
    "for i, label in enumerate(words):\n",
    "    x, y = two_d_embeddings[i, :]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords=\"offset points\",\n",
    "                   ha=\"right\", va=\"bottom\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test similarities of our embeddings with gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This file for Gensim .c (.vec) format\n",
    "\n",
    "def create_vec_file(final_emb_mtx, vocab_size, vec_size,filename):\n",
    "    with open(filename, 'w') as f:\n",
    "        f.write(str(vocab_size)+' '+str(vec_size) + '\\n')\n",
    "        for n in range(vocab_size):\n",
    "            s = ' '.join([index_to_word[n]] + [str(num) for num in final_emb_mtx[n]])\n",
    "            f.write(s + '\\n')\n",
    "            \n",
    "create_vec_file(final_embeddings, vocab_size=vocabulary_size, vec_size=embedding_size, filename='simple_cbow.vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import KeyedVectors\n",
    "\n",
    "simple_cbow = KeyedVectors.load_word2vec_format('simple_cbow.vec', binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Pick a word \n",
    "find_similar_to = 'car'\n",
    "\n",
    "# Finding out similar words [default= top 10]\n",
    "for similar_word in simple_cbow.similar_by_word(find_similar_to):\n",
    "    print(\"Word: {0}, Similarity: {1:.2f}\".format(\n",
    "        similar_word[0], similar_word[1]\n",
    "    ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Task\n",
    "\n",
    "Your task is to implement `skip-gram` model, using code above.\n",
    "\n",
    "This approach is nicely illustrated with this figure:\n",
    "\n",
    "![skip_gram](pics/training_data.png)\n",
    "As you can see on the picture, the training set consists of pairs (`central word`, `context word`).\n",
    "\n",
    "I.e. our model takes `central word` and should produce class in softmax, which corresponds to `context word`.\n",
    "\n",
    "The difference between two models is not that big after all, so good luck with coding!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "def generate_batch_2(data_index, data_size, batch_size, num_skips, skip_window):\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    \n",
    "    batch = np.ndarray(shape = batch_size, dtype = np.int32)\n",
    "    labels = np.ndarray(shape = (batch_size, 1), dtype = np.int32)\n",
    "    span = 2 * skip_window + 1\n",
    "    data_buffer = deque(maxlen = span)\n",
    "    for _ in range(span):\n",
    "        data_buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % data_size\n",
    "    \n",
    "    for i in range(batch_size // num_skips):\n",
    "        target, targets_to_avoid = skip_window, [skip_window]\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid: \n",
    "                target = random.randint(0, span - 1)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = data_buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = data_buffer[target]\n",
    "        data_buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % data_size\n",
    "    return data_index, batch, labels\n",
    "\n",
    "\n",
    "print (\"data = {0}\\n\".format([index_to_word[each] for each in data[:32]]))\n",
    "data_index, data_size = 0, len(data)\n",
    "for num_skips, skip_window in [(2, 1), (4, 2)]:\n",
    "    data_index = 0\n",
    "    data_index, batch, labels = generate_batch_2(data_index=data_index, \n",
    "                                               data_size=data_size, \n",
    "                                               batch_size=16, \n",
    "                                               num_skips=num_skips, \n",
    "                                               skip_window=skip_window)\n",
    "    print (\"data_index = {0}, num_skips = {1}, skip_window = {2}\".format( data_index, num_skips, skip_window))\n",
    "    print (\"batch = {0}\".format([index_to_word[each] for each in batch]))\n",
    "    print (\"labels = {0}\\n\".format([index_to_word[each] for each in labels.reshape(16)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "< Tensorflow code for skip-gram model itself >"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "< Tensorflow code for training of skip-gram model >"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "< tSNE visualization of embeddings >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# fastText"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets look at the [fastText](https://github.com/facebookresearch/fastText) library from Facebook Research.\n",
    "\n",
    "If you're interested, this is an article about _unsupervised learning_ with fasttext:  [link](https://arxiv.org/pdf/1607.04606v1.pdf).\n",
    "\n",
    "* It was announced in 2016 and allows you to train embeddings and solve classification tasks. It is able to work with the Out-of-vocabulary words through the n-gram training.\n",
    "\n",
    "* The library is implemented in C ++. [Python interface](https://pypi.python.org/pypi/fasttext) exists for it, but it is not officially supported. Thus it is not recommended to use it. Instead we will use another python wrapper from Gensim.\n",
    "\n",
    "\n",
    "* fastText allows you to train embeddings very quickly even for fairly large texts compared to other methods.\n",
    "* fastText works better than gensim's word2vec on small corpora.\n",
    "* fastText is not a standalone NLP library; it uses other libraries for preprocessing.\n",
    "\n",
    "\n",
    "The approach to learning attachments from fastText is slightly different than what was discussed above.\n",
    "\n",
    "The basis is the same as in [skip gram](#Skip_gram) and [CBOW](#CBOW), but the $s$ function in probability definition in _Softmax_ is slightly different.\n",
    "\n",
    "Let $ G $ be the set of n-grams that can be obtained from the word $ w $ by selecting n nearest letters. And by the way, lets add special symbols \"<\" and \">\" to the left and right to denote prefixes and suffixes.\n",
    "\n",
    "Then the partitioning will look like this for n = 3 and w = \"< where >\" we get the set: \"< wh\", \"whe\", \"her\", \"ere\", \"re>\" + the word \"where\" also included in this set.\n",
    "\n",
    "Comparing each element $ g \\in G $ with its own vector $ z_g $, we obtain the scoring function $ s $, which we will substitute into the _Softmax_ expression (as in [skip gram](# Skip_gram) or [CBOW](# CBOW))\n",
    "\n",
    "$$ s = \\sum_ {g \\in G_w} z_g ^ {T} \\cdot v_ {c} $$\n",
    "\n",
    "* $v_c$ -- is a vector of word to compare with\n",
    "\n",
    "Vectors for out-of-vocabulary words could be obtained as a sum of n-gram vectors.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are two ways of fastText usage.\n",
    "First way --\n",
    "# Training fastText just from cmd\n",
    "\n",
    "The first thing to do fastText installation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!git clone https://github.com/facebookresearch/fastText.git\n",
    "!cd fastText; make"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see how long it takes to train embbeddings on fastText for CBOW & skip gram models.\n",
    "\n",
    "**! WARNING!: ** * a model with a weight of 800 MB approximately will be saved to the disk*\n",
    "\n",
    "The parameters are as follows:\n",
    "\n",
    "    **For Skipgram, CBOW**\n",
    "    \n",
    "    input          training file path\n",
    "    output         output file path\n",
    "    lr             learning rate [0.05]\n",
    "    lr_update_rate change the rate of updates for the learning rate [100]\n",
    "    dim            size of word vectors [100]\n",
    "    ws             size of the context window [5]\n",
    "    epoch          number of epochs [5]\n",
    "    min_count      minimal number of word occurences [1]\n",
    "    neg            number of negatives sampled [5]\n",
    "    word_ngrams    max length of word ngram [1]\n",
    "    loss           loss function {ns, hs, softmax} [ns]\n",
    "    bucket         number of buckets [2000000]\n",
    "    minn           min length of char ngram [3]\n",
    "    maxn           max length of char ngram [6]\n",
    "    thread         number of threads [12]\n",
    "    t              sampling threshold [0.0001]\n",
    "    silent         disable the log output from the C++ extension [1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# where we a going to save our model\n",
    "MODELS_DIR = 'models/'\n",
    "!mkdir -p {MODELS_DIR}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = 0.05\n",
    "dim = 128\n",
    "ws = 5\n",
    "epoch = 5\n",
    "minCount = 11\n",
    "corpus_file = './data/text8'\n",
    "output_file = MODELS_DIR + 'text8_cbow'\n",
    "FS_HOME = '~/fastText/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!unzip ./data/text8.zip -d ./data\n",
    "%time !./fastText/fasttext cbow -input {corpus_file} -output {output_file} -lr {lr} -dim {dim} -ws {ws} -epoch {epoch} -minCount {minCount} -verbose 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ls -lh models/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Gensim\n",
    "Now we could load the model. The most easiest may to do it [via Gensim package](https://radimrehurek.com/gensim/models/wrappers/fasttext.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import gensim\n",
    "print(gensim.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models.wrappers import FastText\n",
    "\n",
    "model = FastText.load_fasttext_format(output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We could know quite a lot about this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(model.__doc__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pprint\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "pp.pprint(model.__dict__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Similar approach is also works with pretrained embeddings which you may download from the internet.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training fastText directly from Gensim\n",
    "\n",
    "Obviously, this way is quite easier and appropriate when you just need to train your own embeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# uncomment if you would like to try it\n",
    "# model = FastText.train('./fastText/fasttext', corpus_file='./data/text8')\n",
    "# print(model['forests'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Getting the tokens \n",
    "model_words = []\n",
    "for word in model.wv.vocab:\n",
    "    model_words.append(word)\n",
    "\n",
    "\n",
    "# Printing out number of tokens available\n",
    "print(\"Number of Tokens: {}\".format(len(model_words)))\n",
    "\n",
    "# Printing out the dimension of a word vector \n",
    "print(\"Dimension of a word vector: {}\\n\".format(\n",
    "    len(model[words[0]])\n",
    "))\n",
    "\n",
    "\n",
    "# Pick a word \n",
    "find_similar_to = 'car'\n",
    "\n",
    "# Finding out similar words [default= top 10]\n",
    "for similar_word in model.similar_by_word(find_similar_to):\n",
    "    print(\"Word: {0}, Similarity: {1:.2f}\".format(\n",
    "        similar_word[0], similar_word[1]\n",
    "    ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TSNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "tsne = TSNE(perplexity=10, n_components=2, init=\"pca\", n_iter=5000)\n",
    "num_points = 100\n",
    "\n",
    "random_points = random.sample(model_words,num_points)\n",
    "random_points_mtx = np.array([model[word] for word in random_points])\n",
    "\n",
    "two_d_embeddings = tsne.fit_transform(random_points_mtx)\n",
    "\n",
    "plt.figure(figsize=(15,15))\n",
    "\n",
    "for i, label in enumerate(random_points):\n",
    "    x, y = two_d_embeddings[i, :]\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords=\"offset points\",\n",
    "                   ha=\"right\", va=\"bottom\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}