entropy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab size:  50257\n",
      "sentences number:  41\n"
     ]
    }
   ],
   "source": [
    "with open(\"./samples/sample_sents_prob.npy\", 'rb') as f:\n",
    "    prob_set = np.load(f)\n",
    "with open(\"./samples/sample_sents_idxs.npy\", 'rb') as f:\n",
    "    idx_set = np.load(f)\n",
    "with open(\"./samples/sample_sents_tokens.npy\", 'rb') as f:\n",
    "    tk_set = np.load(f)\n",
    "\n",
    "print(\"vocab size: \", len(prob_set[0,0]))\n",
    "print(\"sentences number: \", len(idx_set[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. calculate entropy at each step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_ls = []\n",
    "for i in range(len(prob_set)):\n",
    "    for j in range(len(prob_set[i])):\n",
    "        \n",
    "        entropy = -prob_set[i,j] @ np.log2(prob_set[i,j])\n",
    "        entropy_ls.append(entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'entropy')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAEmlJREFUeJzt3X2sZHV9x/H3B1ai+ATK7RZY7VKlWmJU7MagNMaITVGsSwxSjQ+rkm6bWMVnV21i08RmjY+ojXYLwmqpgqiFqlXJKrVGRXcVFaEoRcSli7sqig8xiH77x5xlL+tvYe7DzJmZ+34lN/fMmTP3fM/uvfM533PO/E6qCkmS9ndQ3wVIkiaTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElS06q+C1iKI444otauXdt3GZI0VXbs2PHDqpq7q+WmOiDWrl3L9u3b+y5DkqZKku8Ns5yHmCRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU1T/UnqpVi76eO3T1+/+ZQeK5GkyWQHIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1jSwgkrw3ye4kV86bd78klyb5Tvf98G5+krwjybVJvpHkUaOqS5I0nFF2EOcBJ+83bxOwraqOBbZ1jwGeBBzbfW0E3j3CuiRJQxhZQFTV54Af7zd7PbC1m94KnDpv/vtq4EvAYUmOHFVtkqS7Nu5zEKuralc3fROwups+Gvj+vOV2dvMkST3p7SR1VRVQC31dko1JtifZvmfPnhFUJkmC8QfED/YeOuq+7+7m3wg8YN5ya7p5v6OqtlTVuqpaNzc3N9JiJWklG3dAXAJs6KY3ABfPm//c7mqmE4CfzjsUJUnqwcjuSZ3kA8DjgSOS7AReD2wGLkxyBvA94PRu8U8ATwauBX4JPH9UdUmShjOygKiqZx7gqZMayxbwwlHVIklaOD9JLUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkpl4CIslLk3wryZVJPpDk7kmOSXJ5kmuTXJDkkD5qkyQNjD0gkhwNvBhYV1UPAw4GngG8EXhbVT0YuBk4Y9y1SZL26esQ0yrgHklWAYcCu4AnABd1z28FTu2pNkkSPQREVd0IvBm4gUEw/BTYAfykqm7rFtsJHD3u2iRJ+/RxiOlwYD1wDHAUcE/g5AW8fmOS7Um279mzZ0RVSpL6OMT0ROC7VbWnqn4NfAQ4ETisO+QEsAa4sfXiqtpSVeuqat3c3Nx4KpakFaiPgLgBOCHJoUkCnARcBXwWOK1bZgNwcQ+1SZI6fZyDuJzByeivAt/satgCvBp4WZJrgfsD54y7NknSPqvuepHlV1WvB16/3+zrgEf3UI4kqcFPUkuSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNfUy3LckjdvaTR+/ffr6zaf0WMn0sIOQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUNFRAJNk2zDxJ0uy408H6ktwdOBQ4IsnhQLqn7gMcPeLaJEk9uqvRXP8aeAlwFLCDfQFxC/CuEdYlSerZnQZEVZ0FnJXkRVX1zuVaaZLDgLOBhwEFvAC4BrgAWAtcD5xeVTcv1zolSQsz1P0gquqdSR7L4M171bz571vkes8CPllVpyU5hMFhrNcC26pqc5JNwCbg1Yv8+ZKkJRoqIJK8H3gQcAXwm252AQsOiCT3BR4HPA+gqm4Fbk2yHnh8t9hW4DIMCEnqzbB3lFsHHFdVtQzrPAbYA5yb5BEMzm2cCayuql3dMjcBq5dhXZKkRRr2cxBXAr+/TOtcBTwKeHdVHQ/8gsHhpNt1QdQMoyQbk2xPsn3Pnj3LVJIkaX/DBsQRwFVJPpXkkr1fi1znTmBnVV3ePb6IQWD8IMmRAN333a0XV9WWqlpXVevm5uYWWYIk6a4Me4jp75drhVV1U5LvJ3lIVV0DnARc1X1tADZ33y9ernVKkhZu2KuY/muZ1/si4PzuCqbrgOcz6GYuTHIG8D3g9GVepyRpAYa9iuln7DsncAhwN+AXVXWfxay0qq5gcOJ7fyct5udJkpbfsB3EvfdOJwmwHjhhVEVJkvq34NFca+DfgT8fQT2SpAkx7CGmp817eBCDw0O/GklFkqSJMOxVTH8xb/o2BmMlrV/2aiRJE2PYcxDPH3UhkqTJMuwNg9Yk+WiS3d3Xh5OsGXVxkqT+DHuS+lzgEgb3hTgK+I9uniRpRg0bEHNVdW5V3dZ9nQc4zoUkzbBhA+JHSZ6d5ODu69nAj0ZZmCSpX8MGxAsYDH1xE7ALOI3ufg6SpNk07GWu/wBs2HsL0CT3A97MIDgkSTNo2A7i4fPvD11VPwaOH01JkqRJMGxAHJTk8L0Pug5i2O5DkjSFhn2TfwvwxSQf6h4/HXjDaEqSJE2CYT9J/b4k24EndLOeVlVXja4sSVLfhj5M1AWCoSBJK8SCh/uWJK0MBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1NRbQCQ5OMnXknyse3xMksuTXJvkgiSH9FWbJKnfDuJM4Op5j98IvK2qHgzcDJzRS1WSJKCngEiyBjgFOLt7HAY3I7qoW2QrcGoftUmSBvrqIN4OvAr4bff4/sBPquq27vFO4Og+CpMkDYw9IJI8BdhdVTsW+fqNSbYn2b5nz55lrk6StFcfHcSJwFOTXA98kMGhpbOAw5LsvQXqGuDG1ouraktVrauqdXNzc+OoV5JWpKHvSb1cquo1wGsAkjweeEVVPSvJh4DTGITGBuDicdcmTau1mz5++/T1m0/psRLNkkn6HMSrgZcluZbBOYlzeq5Hkla0sXcQ81XVZcBl3fR1wKP7rEeStM8kdRCSpAliQEiSmgwISVKTASFJaur1JLWk8VnopbAHWn4pl9TOf+1iXq/xsoOQJDUZEJKkJgNCktTkOQjNnOUadmLSh6/Y/3i+tNzsICRJTXYQ0hhMejcitdhBSJKa7CA08WZ573uWt03Tzw5CktRkByFNkWm8cmkaa9aAHYQkqckOQivGNB3vX67xjpYy5tK4TUod2scOQpLUZAehXvW117jS91an6bzASv+/6pMdhCSpyQ5CWoID7Ynf2Z7uNO29L4R7+rPHgJDmmbU371nbHo2Xh5gkSU12EJLu0kI7kaV2LnY+k8EOQpLUZAchTbhJ3pu2U5htdhCSpCY7iBVkmi5DHGbPcrn2Pt2L3cd/C81nByFJahp7B5HkAcD7gNVAAVuq6qwk9wMuANYC1wOnV9XN465Py2eaOhZJv6uPDuI24OVVdRxwAvDCJMcBm4BtVXUssK17LEnqydg7iKraBezqpn+W5GrgaGA98Phusa3AZcCrx12f9pmFDmAU5ykWOoS2NK16PQeRZC1wPHA5sLoLD4CbGByCkiT1pLermJLcC/gw8JKquiXJ7c9VVSWpA7xuI7AR4IEPfOA4Sp15k9IpTMte96jqnJbt79Ok/K6uFL10EEnuxiAczq+qj3Szf5DkyO75I4HdrddW1ZaqWldV6+bm5sZTsCStQH1cxRTgHODqqnrrvKcuATYAm7vvF4+7Ng1nqXtx7ilL06GPQ0wnAs8Bvpnkim7eaxkEw4VJzgC+B5zeQ22SpE4fVzF9HsgBnj5pnLVIkg7MT1JLkpociwmvjJgmnr/QXou53etd/Rz//u/IgNBY+MauSeLv43A8xCRJarKDmGLDtMaj3lPa/+fbomuS2CksjR2EJKnJDmJIs7an7HARku6KHYQkqckOYj8r/ZI3OwBNO3+Hl48dhCSpyQ6iZ8v1YR9JS7fSjyDszw5CktRkB7EMRr3X4V6N1K+V+jdoByFJajIgJElNBoQkqcmAkCQ1GRCSpCavYroTy/mJzIVeBeGnQSX1zQ5CktRkBzFl+uos7GikgZU0+oEBscwm8SY+ksZrVj5Y5yEmSVKTHcQiTdpe+qTVI027hf5NLWX5Se0y7CAkSU12ECPkXr2k+abtPcEOQpLUZAchSSM0bV3DfHYQkqSmieogkpwMnAUcDJxdVZt7LkmSJsa4r3yamA4iycHAPwFPAo4DnpnkuH6rkqSVa5I6iEcD11bVdQBJPgisB67qtSpJGrNJOW8xMR0EcDTw/XmPd3bzJEk9mKQOYihJNgIbu4c/T3LNIn/UEcAPl6eq3s3KtszKdsDsbMusbAdM8LbkjQta/Ajghwt8zf7+YJiFJikgbgQeMO/xmm7eHVTVFmDLUleWZHtVrVvqz5kEs7Its7IdMDvbMivbAbOzLePcjkk6xPQV4NgkxyQ5BHgGcEnPNUnSijUxHURV3Zbkb4FPMbjM9b1V9a2ey5KkFWtiAgKgqj4BfGJMq1vyYaoJMivbMivbAbOzLbOyHTA72zK27UhVjWtdkqQpMknnICRJE2RFBkSSk5Nck+TaJJv6rmcxkjwgyWeTXJXkW0nO7LumpUpycJKvJflY37UsVpLDklyU5H+SXJ3kMX3XtFhJXtr9bl2Z5ANJ7t53TcNI8t4ku5NcOW/e/ZJcmuQ73ffD+6xxWAfYljd1v1/fSPLRJIeNav0rLiBmaEiP24CXV9VxwAnAC6d0O+Y7E7i67yKW6Czgk1X1UOARTOn2JDkaeDGwrqoexuDCkWf0W9XQzgNO3m/eJmBbVR0LbOseT4Pz+N1tuRR4WFU9HPg28JpRrXzFBQTzhvSoqluBvUN6TJWq2lVVX+2mf8bgjWhqP3meZA1wCnB237UsVpL7Ao8DzgGoqlur6if9VrUkq4B7JFkFHAr8X8/1DKWqPgf8eL/Z64Gt3fRW4NSxFrVIrW2pqk9X1W3dwy8x+MzYSKzEgJi5IT2SrAWOBy7vt5IleTvwKuC3fReyBMcAe4Bzu0NlZye5Z99FLUZV3Qi8GbgB2AX8tKo+3W9VS7K6qnZ10zcBq/ssZhm9APjPUf3wlRgQMyXJvYAPAy+pqlv6rmcxkjwF2F1VO/quZYlWAY8C3l1VxwO/YHoOZdxBd4x+PYPQOwq4Z5Jn91vV8qjBpZtTf/lmktcxONR8/qjWsRIDYqghPaZBkrsxCIfzq+ojfdezBCcCT01yPYNDfk9I8q/9lrQoO4GdVbW3k7uIQWBMoycC362qPVX1a+AjwGN7rmkpfpDkSIDu++6e61mSJM8DngI8q0b4WYWVGBAzMaRHkjA41n11Vb2173qWoqpeU1Vrqmotg/+Pz1TV1O2tVtVNwPeTPKSbdRLTO1z9DcAJSQ7tftdOYkpPuHcuATZ00xuAi3usZUm6G6u9CnhqVf1ylOtacQHRndzZO6TH1cCFUzqkx4nAcxjsbV/RfT2576LEi4Dzk3wDeCTwjz3XsyhdF3QR8FXgmwzeK6bik8hJPgB8EXhIkp1JzgA2A3+W5DsMuqOpuFvlAbblXcC9gUu7v/v3jGz9fpJaktSy4joISdJwDAhJUpMBIUlqMiAkSU0GhCSpyYCQliDJqTMwSKLUZEBIS3Mqg1GBf0c3yJ00tQwIaT9Jnp3ky92HkP65u0/Fz5O8IcnXk3wpyeokjwWeCrypW/ZBSS5L8vYk24Ezk6xN8plu7P5tSR7YreO8JO9Jsj3Jt7vxqEjyuSSPnFfL55M8opd/CK14BoQ0T5I/Bv4SOLGqHgn8BngWcE/gS1X1COBzwF9V1RcYDOHwyqp6ZFX9b/djDqmqdVX1FuCdwNZu7P7zgXfMW91aBsPPnwK8p7shzznA87pa/gi4e1V9fZTbLB2IASHd0UnAnwBfSXJF9/gPgVuBvXe628Hgzf1ALpg3/Rjg37rp9wN/Ou+5C6vqt1X1HeA64KHAh4CndAMxvoDBDWOkXniMVLqjMNjjv8NdupK8Yt6omb/hzv92fjHkuvYf56aq6pdJLmUw1PbpDMJK6oUdhHRH24DTkvwe3H4v4z+4k+V/xmDgtAP5Avtu1fks4L/nPff0JAcleRCDLuWabv7ZDA5FfaWqbl7ENkjLwg5Cmqeqrkryd8CnkxwE/Bp44Z285IPAvyR5MXBa4/kXMbjD3CsZ3G3u+fOeuwH4MnAf4G+q6lddDTuS3AKcu+QNkpbA0VylHiQ5D/hYVV3UeO4o4DLgoVU1zbdg1ZTzEJM0QZI8l8G9xV9nOKhvdhCSpCY7CElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqSm/wcL6OEE33qUfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "x = entropy_ls\n",
    "plt.hist(x, bins=100)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('entropy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. check a sample sentence entropy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_ls = []\n",
    "i = 48\n",
    "for j in range(len(prob_set[i])):\n",
    "\n",
    "    entropy = np.sum(-prob_set[i,j] * np.log2(prob_set[i,j]))\n",
    "    entropy_ls.append(entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'entropy')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEKCAYAAAAB0GKPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE4lJREFUeJzt3X2wXXV97/H3h6daKVZtjiiEGOuleKkj0Z6hPt0OSmsRqFgHLYxaVGy0o1bv9LYX2zva8U47dFrbWulIU4mol8YHFMstUclgLTr1gYQGjTwU5EZJRBLBAmodG/zeP/aKbg57hx8nZ+91cs77NbNnr/Vbv73Wd+XhfM5ae63fSlUhSdKDOajvAiRJBwYDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSk0P6LmAhrVixolavXt13GZJ0wNiyZcu3qmqmpe+SCozVq1ezefPmvsuQpANGkq+19vWUlCSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKnJkrrTe3+sPu+KH01vP/+0HiuRpMXJIwxJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEwJAkNZnYWFJJ1gOnA7uq6sld2weB47oujwT+varWjPjsduBe4D5gT1XNTqpOSVKbSQ4+eDFwAfC+vQ1V9Rt7p5O8Hbh7H59/TlV9a2LVSZIekokFRlVdnWT1qGVJArwEeO6kti9JWlh9fYfx34A7qurmMcsLuDLJliRrp1iXJGmMvp6HcTawYR/Ln11VO5M8BtiU5MaqunpUxy5Q1gKsWrVq4SuVJAE9HGEkOQR4EfDBcX2qamf3vgu4DDhxH33XVdVsVc3OzMwsdLmSpE4fp6R+GbixqnaMWpjk8CRH7J0Gngdsm2J9kqQRJhYYSTYAnwOOS7IjybndorOYczoqyVFJNnazRwKfTXId8EXgiqr6xKTqlCS1meRVUmePaX/FiLZvAKd207cCJ0yqLknS/HintySpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkppMLDCSrE+yK8m2obY/SrIzydbudeqYz56S5KYktyQ5b1I1SpLaTfII42LglBHtf1lVa7rXxrkLkxwM/A3wfOB44Owkx0+wTklSg4kFRlVdDdw1j4+eCNxSVbdW1Q+ADwBnLGhxkqSHrI/vMF6f5EvdKatHjVh+NHDb0PyOrm2kJGuTbE6yeffu3QtdqySpM+3AeBfwRGANcDvw9v1dYVWtq6rZqpqdmZnZ39VJksaYamBU1R1VdV9V/RD4Owann+baCRwzNL+ya5Mk9WiqgZHkcUOzvw5sG9HtGuDYJE9IchhwFnD5NOqTJI13yKRWnGQDcBKwIskO4K3ASUnWAAVsB17T9T0KeHdVnVpVe5K8HvgkcDCwvqq+Mqk6JUltJhYYVXX2iOaLxvT9BnDq0PxG4AGX3EqS+uOd3pKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpycQCI8n6JLuSbBtq+7MkNyb5UpLLkjxyzGe3J/lykq1JNk+qRklSu0keYVwMnDKnbRPw5Kp6CvBvwJv38fnnVNWaqpqdUH2SpIdgYoFRVVcDd81pu7Kq9nSznwdWTmr7kqSF1ed3GK8CPj5mWQFXJtmSZO0Ua5IkjXFIHxtN8ofAHuCSMV2eXVU7kzwG2JTkxu6IZdS61gJrAVatWjWReiVJPRxhJHkFcDrw0qqqUX2qamf3vgu4DDhx3Pqqal1VzVbV7MzMzAQqliTBlAMjySnA7wMvqKrvjelzeJIj9k4DzwO2jeorSZqeSV5WuwH4HHBckh1JzgUuAI5gcJppa5ILu75HJdnYffRI4LNJrgO+CFxRVZ+YVJ2SpDYT+w6jqs4e0XzRmL7fAE7tpm8FTphUXZKk+fFOb0lSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1KQpMJJc1dImSVq69jmWVJKHAQ8HViR5FJBu0SOAoydcmyRpEXmwwQdfA7wJOArYwo8D4x4GI89KkpaJfQZGVb0DeEeSN1TVO6dUkyRpEWoa3ryq3pnkmcDq4c9U1fsmVJckaZFpCowk7weeCGwF7uuaCzAwJGmZaH2A0ixw/LhncEuSlr7W+zC2AY+dZCGSpMWtNTBWANcn+WSSy/e+HuxDSdYn2ZVk21Dbo5NsSnJz9/6oMZ89p+tzc5JzGuuUJE1I6ympP5rn+i9mcPnt8Hcd5wFXVdX5Sc7r5v/n8IeSPBp4K4NTYQVsSXJ5VX17nnVIkvZT61VS/zyflVfV1UlWz2k+Azipm34v8GnmBAbwq8CmqroLIMkm4BRgw3zqkCTtv9arpO5l8Js+wGHAocB3q+oR89jmkVV1ezf9TeDIEX2OBm4bmt+Bd5ZLUq9ajzCO2DudJAyOEp6+vxuvqkqyX1deJVkLrAVYtWrV/pYkSRrjIY9WWwMfY3DaaD7uSPI4gO5914g+O4FjhuZXdm2j6llXVbNVNTszMzPPkiRJD6b1lNSLhmYPYvBl9Pfnuc3LgXOA87v3fxjR55PAnwxdQfU84M3z3J4kaQG0XiX1a0PTe4DtDE5L7VOSDQy+4F6RZAeDK5/OBz6U5Fzga8BLur6zwGur6tVVdVeS/w1c063qbXu/AJck9aP1O4xXzmflVXX2mEUnj+i7GXj10Px6YP18titJWnitD1BameSy7ia8XUk+kmTlpIuTJC0erV96v4fBdw9Hda//27VJkpaJ1sCYqar3VNWe7nUx4CVJkrSMtAbGnUleluTg7vUy4M5JFiZJWlxaA+NVDK5m+iZwO3Am8IoJ1SRJWoRaL6t9G3DO3sH/usEB/5xBkEiSloHWI4ynDI8U290T8dTJlCRJWoxaA+Og4edWdEcYrUcnkqQloPWH/tuBzyX5cDf/YuCPJ1OSJGkxar3T+31JNgPP7ZpeVFXXT64sSdJi03xaqQsIQ0KSlqmHPLy5JGl5MjAkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUpOpB0aS45JsHXrdk+RNc/qclOTuoT5vmXadkqT7m/p4UFV1E7AGIMnBwE7gshFdP1NVp0+zNknSeH2fkjoZ+GpVfa3nOiRJD6LvwDgL2DBm2TOSXJfk40l+fppFSZIeqLfASHIY8ALgwyMWXws8vqpOAN4JfGwf61mbZHOSzbt3755MsZKkXo8wng9cW1V3zF1QVfdU1Xe66Y3AoUlWjFpJVa2rqtmqmp2ZmZlsxZK0jPUZGGcz5nRUkscmSTd9IoM675xibZKkOXp5al6Sw4FfAV4z1PZagKq6EDgT+O0ke4D/AM6qquqjVknSQC+BUVXfBX5mTtuFQ9MXABdMuy5J0nh9XyUlSTpAGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqUlvgZFke5IvJ9maZPOI5Uny10luSfKlJE/ro05J0sAhPW//OVX1rTHLng8c271+EXhX9y5J6sFiPiV1BvC+Gvg88Mgkj+u7KElarvoMjAKuTLIlydoRy48Gbhua39G13U+StUk2J9m8e/fuCZUqSeozMJ5dVU9jcOrpdUl+aT4rqap1VTVbVbMzMzMLW6Ek6Ud6C4yq2tm97wIuA06c02UncMzQ/MquTZLUg14CI8nhSY7YOw08D9g2p9vlwG92V0s9Hbi7qm6fcqmSpE5fV0kdCVyWZG8Nf19Vn0jyWoCquhDYCJwK3AJ8D3hlT7VKkugpMKrqVuCEEe0XDk0X8Lpp1iVJGm8xX1YrSVpEDAxJUhMDQ5LUxMCQJDUxMCRJTQwMSVITA0OS1MTAkCQ1MTAkSU0MDElSEwNDktTEwJAkNTEwJElNDAxJUhMDQ5LUxMCQJDXp64l7B6TV513xo+nt55/WYyWSNH0eYUiSmkw9MJIck+Sfklyf5CtJ3jiiz0lJ7k6ytXu9Zdp1SpLur49TUnuA362qa5McAWxJsqmqrp/T7zNVdXoP9UmSRpj6EUZV3V5V13bT9wI3AEdPuw5J0kPT63cYSVYDTwW+MGLxM5Jcl+TjSX5+qoVJkh6gt6ukkvwU8BHgTVV1z5zF1wKPr6rvJDkV+Bhw7Jj1rAXWAqxatWqCFUvS8tbLEUaSQxmExSVV9dG5y6vqnqr6Tje9ETg0yYpR66qqdVU1W1WzMzMzE61bkpazPq6SCnARcENV/cWYPo/t+pHkRAZ13jm9KiVJc/VxSupZwMuBLyfZ2rX9AbAKoKouBM4EfjvJHuA/gLOqqnqoVZLUmXpgVNVngTxInwuAC6ZTkSSphXd6S5KaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmBoYkqYmBIUlqYmBIkpoYGJKkJgaGJKlJb8/DWMxWn3fFgq9n+/mnLYp1tnx+XJ9J7M9C1fZQ++yvxfzn2FLDUtneYjbu58hi+VkwHx5hSJKaGBiSpCYGhiSpiYEhSWpiYEiSmhgYkqQmvQRGklOS3JTkliTnjVj+E0k+2C3/QpLV069SkjRs6oGR5GDgb4DnA8cDZyc5fk63c4FvV9V/Af4S+NPpVilJmquPI4wTgVuq6taq+gHwAeCMOX3OAN7bTV8KnJwkU6xRkjRHH4FxNHDb0PyOrm1kn6raA9wN/MxUqpMkjZSqmu4GkzOBU6rq1d38y4FfrKrXD/XZ1vXZ0c1/tevzrRHrWwus7WaPA26aZ2krgAesfwlzf5e25ba/sPz2eaH29/FVNdPSsY+xpHYCxwzNr+zaRvXZkeQQ4KeBO0etrKrWAev2t6gkm6tqdn/Xc6Bwf5e25ba/sPz2uY/97eOU1DXAsUmekOQw4Czg8jl9LgfO6abPBD5V0z4UkiTdz9SPMKpqT5LXA58EDgbWV9VXkrwN2FxVlwMXAe9PcgtwF4NQkST1qJfhzatqI7BxTttbhqa/D7x4ymXt92mtA4z7u7Qtt/2F5bfPU9/fqX/pLUk6MDk0iCSpybIPjAcbpmSpSXJMkn9Kcn2SryR5Y981TVqSg5P8a5J/7LuWaUjyyCSXJrkxyQ1JntF3TZOU5L93/5a3JdmQ5GF917TQkqxPsqu75WBv26OTbEpyc/f+qEnXsawDo3GYkqVmD/C7VXU88HTgdctgn98I3NB3EVP0DuATVfUk4ASW8L4nORr4HWC2qp7M4EKapXiRzMXAKXPazgOuqqpjgau6+Yla1oFB2zAlS0pV3V5V13bT9zL4YTL3TvslI8lK4DTg3X3XMg1Jfhr4JQZXGlJVP6iqf++3qok7BPjJ7p6thwPf6LmeBVdVVzO4YnTY8BBK7wVeOOk6lntgtAxTsmR1owA/FfhCv5VM1F8Bvw/8sO9CpuQJwG7gPd1puHcnObzvoialqnYCfw58HbgduLuqruy3qqk5sqpu76a/CRw56Q0u98BYtpL8FPAR4E1VdU/f9UxCktOBXVW1pe9apugQ4GnAu6rqqcB3mcKpir505+3PYBCURwGHJ3lZv1VNX3dj88QveV3ugdEyTMmSk+RQBmFxSVV9tO96JuhZwAuSbGdwuvG5Sf5PvyVN3A5gR1XtPWq8lEGALFW/DPy/qtpdVf8JfBR4Zs81TcsdSR4H0L3vmvQGl3tgtAxTsqR0w8RfBNxQVX/Rdz2TVFVvrqqVVbWawd/tp6pqSf/2WVXfBG5LclzXdDJwfY8lTdrXgacneXj3b/tklvCX/HMMD6F0DvAPk95gL3d6LxbjhinpuaxJexbwcuDLSbZ2bX/Q3X2vpeENwCXdL0G3Aq/suZ6JqaovJLkUuJbBFYD/yhK84zvJBuAkYEWSHcBbgfOBDyU5F/ga8JKJ1+Gd3pKkFsv9lJQkqZGBIUlqYmBIkpoYGJKkJgaGJKmJgSEtkCQvXAYDOWoZMzCkhfNCBqMeP0A3MJ50QDMwpH1I8rIkX0yyNcnfds/W+E6SP05yXZLPJzkyyTOBFwB/1vV9YpJPJ/mrJJuBNyZZneRTSb6U5Kokq7ptXJzkwiSbk/xbNwYWSa5Osmaols8mOaGXPwgJA0MaK8l/BX4DeFZVrQHuA14KHA58vqpOAK4Gfquq/oXBUA2/V1Vrquqr3WoOq6rZqno78E7gvVX1FOAS4K+HNreawXD7pwEXdg8Bugh4RVfLzwEPq6rrJrnP0r4YGNJ4JwO/AFzTDaNyMvCzwA+AvU/v28Lgh/04Hxyafgbw9930+4FnDy37UFX9sKpuZjCcx5OADwOnd4NFvorBQ3Sk3nheVRovDI4I3ny/xuR/1I/H1LmPff8/+m7jtuaO0VNV9b0kmxgM3/0SBuEl9cYjDGm8q4AzkzwGfvQM5cfvo/+9wBH7WP4v/PjxoS8FPjO07MVJDkryRAZHMTd17e9mcOrqmqr69jz2QVowHmFIY1TV9Un+F3BlkoOA/wRet4+PfAD4uyS/A5w5YvkbGDwJ7/cYPBVveBTZrwNfBB4BvLaqvt/VsCXJPcB79nuHpP3kaLVSz5JcDPxjVV06YtlRwKeBJ1XVcnnMrBYpT0lJi1SS32TwvPU/NCy0GHiEIUlq4hGGJKmJgSFJamJgSJKaGBiSpCYGhiSpiYEhSWry/wFNEn0sD+zP9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "x = entropy_ls\n",
    "plt.hist(x,bins=100)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('entropy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. bin embedding method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_buckets = 8\n",
    "N_vocab = 50257\n",
    "bucket_size = N_vocab // N_buckets\n",
    "entropy_ls = []\n",
    "for i in range(len(prob_set)):\n",
    "    for j in range(len(prob_set[i])):\n",
    "        whole_p = prob_set[i,j]\n",
    "        bins_p_ls = [np.sum(whole_p[k*bucket_size: (k+1)*bucket_size]) for k in range(N_buckets-1)]\n",
    "        bins_p_ls.append(np.sum(whole_p[(N_buckets-1)*bucket_size:]))\n",
    "        bins_p = np.array(bins_p_ls)\n",
    "        entropy = -bins_p @ np.log2(bins_p)\n",
    "        entropy_ls.append(entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'entropy')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAE41JREFUeJzt3XvQZHV95/H3h5skGoM4Twgy4MMaEpdYXtgpCsNWigq7FVZcoFyWkNI4KLuTZF0vu5tETFJhk6xVuNlc0OzGnRVhsFiBoAkT1N1QoyxJGdBBURG8TAjqUIMzUQQviWbwu3/0Gex5ZuY3/Vy6T/fzvF9VXXP69Onu72/OTH/69zvn/DpVhSRJh3JE3wVIkqabQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklS01F9F7Ac69atq/n5+b7LkKSZcs899/xtVc2Nuv1MB8X8/Dzbt2/vuwxJmilJvrCY7R16kiQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNc30ldnLMX/F+55cfuiq83usRJKmmz0KSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkprGFhRJ3plkd5L7htb9TpLPJPlkkj9JctzQY29KsiPJZ5P89LjqkiQtzjh7FNcB5y1YdzvwvKp6PvA54E0ASU4HLgV+vHvO/0hy5BhrkySNaGxBUVV3Al9dsO7Pq2pvd/cuYH23fCFwY1V9u6r+BtgBnDmu2iRJo+vzGMWrgQ90yycBXxp6bGe3TpLUs16CIsmvAXuBG5bw3E1JtifZvmfPnpUvTpK0n4kHRZLLgJcCL6+q6lY/DJw8tNn6bt0BqmpzVW2oqg1zc3NjrVWSNOGgSHIe8CvABVX1raGHtgKXJnlKklOB04CPTLI2SdLBje33KJK8GzgHWJdkJ3Alg7OcngLcngTgrqr6har6dJKbgfsZDEm9pqqeGFdtkqTRjS0oqupnD7L6msb2bwbePK56JElL45XZkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKlpbEGR5J1Jdie5b2jd8UluT/L57s9ndOuT5K1JdiT5ZJIzxlWXJGlxxtmjuA44b8G6K4BtVXUasK27D/AvgNO62ybgj8ZYlyRpEcYWFFV1J/DVBasvBLZ0y1uAi4bWX18DdwHHJTlxXLVJkkY36WMUJ1TVrm75EeCEbvkk4EtD2+3s1kmSetbbweyqKqAW+7wkm5JsT7J9z549Y6hMkjRs0kHx5X1DSt2fu7v1DwMnD223vlt3gKraXFUbqmrD3NzcWIuVJE0+KLYCG7vljcCtQ+tf2Z39dBbw2NAQlSSpR0eN64WTvBs4B1iXZCdwJXAVcHOSy4EvAJd0m78feAmwA/gW8Kpx1SVJWpyxBUVV/ewhHjr3INsW8Jpx1SJJWjqvzJYkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlqMigkSU0GhSSpyaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKmpl6BI8h+SfDrJfUneneTYJKcmuTvJjiQ3JTmmj9okSfubeFAkOQl4HbChqp4HHAlcCrwF+P2q+hHgUeDySdcmSTpQX0NPRwHfl+Qo4PuBXcBPAbd0j28BLuqpNknSkIkHRVU9DPw34IsMAuIx4B7ga1W1t9tsJ3DSwZ6fZFOS7Um279mzZxIlS9Ka1sfQ0zOAC4FTgWcBTwXOG/X5VbW5qjZU1Ya5ubkxVSlJ2qePoad/BvxNVe2pqn8A3gucDRzXDUUBrAce7qE2SdICfQTFF4Gzknx/kgDnAvcDHwIu7rbZCNzaQ22SpAX6OEZxN4OD1h8DPtXVsBl4I/Afk+wAnglcM+naJEkHOurwm6y8qroSuHLB6geBM3soR5LU4JXZkqQmg0KS1GRQSJKaRgqKJNtGWSdJWn2aB7OTHMtgio113YVy6R56Ooe4clqStLoc7qynnwfewOAK6nv4XlA8DvzhGOuSJE2JZlBU1dXA1UleW1Vvm1BNkqQpMtJ1FFX1tiQ/AcwPP6eqrh9TXZKkKTFSUCR5F/Ac4F7giW51AQaFJK1yo16ZvQE4vapqnMVIkqbPqNdR3Af88DgLkSRNp1F7FOuA+5N8BPj2vpVVdcFYqpIkTY1Rg+I/j7MIScszf8X7nlx+6Krze6xEq9GoZz39v3EXIkmaTqOe9fR1Bmc5ARwDHA18s6qePq7CJEnTYdQexQ/sW+5+le5C4KxxFSVJmh6Lnj22Bv4U+Okx1CNJmjKjDj29bOjuEQyuq/j7sVQkSZoqo5719C+HlvcCDzEYfpIkrXKjHqN41bgLkSRNp1F/uGh9kj9Jsru7vSfJ+nEXJ0nq36gHs68FtjL4XYpnAX/WrZMkrXKjBsVcVV1bVXu723XA3BjrkiRNiVGD4itJXpHkyO72CuArS33TJMcluSXJZ5I8kOTFSY5PcnuSz3d/PmOpry9JWjmjBsWrgUuAR4BdwMXAZct436uB/1NVzwVeADwAXAFsq6rTgG3dfUlSz0YNit8CNlbVXFX9EIPg+M2lvGGSHwR+ErgGoKq+U1VfY3C67ZZusy3ARUt5fUnSyho1KJ5fVY/uu1NVXwVetMT3PBXYA1yb5ONJ3pHkqcAJVbWr2+YR4IQlvr4kaQWNGhRHDB8zSHI8o1+st9BRwBnAH1XVi4BvsmCYqfslvYP+ml6STUm2J9m+Z8+eJZYgSRrVqEHxu8BfJfntJL8NfBj4r0t8z53Azqq6u7t/C4Pg+HKSEwG6P3cf7MlVtbmqNlTVhrk5T7ySpHEbKSiq6nrgZcCXu9vLqupdS3nDqnoE+FKSH+tWnQvcz+A6jY3duo3ArUt5fUnSyhp5+Kiq7mfwgb4SXgvckOQY4EHgVQxC6+YklwNfYHCWlSSpZ0s9zrAsVXUvgxloFzp30rVIktoW/XsUkqS1xaCQJDUZFJKkJoNCktRkUEiSmgwKSVKTQSFJajIoJElNBoUkqcmgkCQ1GRSSpCaDQpLUZFBIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNRkUkqQmg0KS1GRQSJKaDApJUpNBIUlq6i0okhyZ5ONJbuvun5rk7iQ7ktyU5Ji+apMkfU+fPYrXAw8M3X8L8PtV9SPAo8DlvVQlSdpPL0GRZD1wPvCO7n6AnwJu6TbZAlzUR22SpP311aP4A+BXgO92958JfK2q9nb3dwIn9VGYJGl/R036DZO8FNhdVfckOWcJz98EbAI45ZRTVri6tvkr3vfk8kNXnT/R95akvvTRozgbuCDJQ8CNDIacrgaOS7IvuNYDDx/syVW1uao2VNWGubm5SdQrSWvaxIOiqt5UVeurah64FPhgVb0c+BBwcbfZRuDWSdcmSTrQxIeeGt4I3JjkvwAfB67puZ6ROSQlaTXrNSiq6g7gjm75QeDMPuuRJB3IK7MlSU0GhSSpyaCQJDVN08HsVceD3JJWA3sUkqQmg0KS1OTQE/sPES10qCGj1nNWog6HqiRNC3sUkqQmg0KS1GRQSJKaDApJUpMHs7VmeLKAtDT2KCRJTfYotCTj+HY+zd/4p7k2adzsUUiSmuxRaNlWU++iz56DvRZNK4NixqzUh4kfSv0Zx1X90jg59CRJarJHscIW+23Rb5ezb5p7Z9Ncm2aHQXEY4578bxz8cBgvw11rjUNPkqQmexTSCrI3p9XIHoUkqckehVbUKN+oJznGf6j3Wuvf/Be7n4a3Wet/d2vRxIMiycnA9cAJQAGbq+rqJMcDNwHzwEPAJVX16KTr0+xbqSDq86D1cj6MRwlHaTH6GHraC/ynqjodOAt4TZLTgSuAbVV1GrCtuy9J6tnEexRVtQvY1S1/PckDwEnAhcA53WZbgDuAN066vlnlt8jpNounWUv79HqMIsk88CLgbuCELkQAHmEwNHWw52wCNgGccsop4y9yCvQ1pu/483SYlX2y8N/pNNeqxentrKckTwPeA7yhqh4ffqyqisHxiwNU1eaq2lBVG+bm5iZQqSStbb30KJIczSAkbqiq93arv5zkxKraleREYHcftc2StTj0MMqZONNiGms6nFmsWeM38R5FkgDXAA9U1e8NPbQV2NgtbwRunXRtkqQD9dGjOBv4OeBTSe7t1v0qcBVwc5LLgS8Al/RQm4ZMy5jzJOfG0spZ7LGVWTkWsxb1cdbTXwI5xMPnTrKWaTaNH17TWJOk8XMKD0lSk0EhSWoyKCRJTU4KqInw+Mb0GmXfTHr/eWB7uhgUEzLNH5Tjqs2zlaTVwaEnSVKTPQpphkxbL2q59Sz2+Q5J9cOgkDRR0xZ2OjyHniRJTfYoJM2kvoah1uLwlz0KSVKTPQpJYzfJU6VHmX5+rfQEVopBIWmqLefMKK0Mh54kSU32KCStaWvx4PRi2aOQJDUZFJKkJoNCktRkUEiSmjyYLWnN8RTaxTEoJKnjGVAH59CTJKnJHoUkHcahhqrWSg9k6noUSc5L8tkkO5Jc0Xc9krTWTVWPIsmRwH8H/jmwE/hokq1VdX+/lUlaa1ZqjqnV0NOYqqAAzgR2VNWDAEluBC4EDApJM2lhgIwyu+0oJhlA0zb0dBLwpaH7O7t1kqSeTFuP4rCSbAI2dXe/keSzS3ypdcDfrkxVU8M2zQbbNP3G1p68pbfXGW7TsxfzxGkLioeBk4fur+/WPamqNgObl/tGSbZX1Yblvs40sU2zwTZNv9XWHlhem6Zt6OmjwGlJTk1yDHApsLXnmiRpTZuqHkVV7U3y74H/CxwJvLOqPt1zWZK0pk1VUABU1fuB90/grZY9fDWFbNNssE3Tb7W1B5bRplTVShYiSVplpu0YhSRpyqz6oDjclCBJnpLkpu7xu5PMT77KxRmhTZcl2ZPk3u72b/qoc1RJ3plkd5L7DvF4kry1a+8nk5wx6RoXa4Q2nZPksaF99BuTrnExkpyc5ENJ7k/y6SSvP8g2M7WfRmzTrO2nY5N8JMknujb95kG2WfxnXlWt2huDA+J/Dfwj4BjgE8DpC7b5d8Dbu+VLgZv6rnsF2nQZ8Id917qINv0kcAZw3yEefwnwASDAWcDdfde8Am06B7it7zoX0Z4TgTO65R8APneQf3cztZ9GbNOs7acAT+uWjwbuBs5asM2iP/NWe4/iySlBquo7wL4pQYZdCGzplm8Bzk2SCda4WKO0aaZU1Z3AVxubXAhcXwN3AcclOXEy1S3NCG2aKVW1q6o+1i1/HXiAA2dNmKn9NGKbZkr3d/+N7u7R3W3hgehFf+at9qAYZUqQJ7epqr3AY8AzJ1Ld0ow6zcm/6rr/tyQ5+SCPz5LVOrXLi7shgg8k+fG+ixlVN1TxIgbfVofN7H5qtAlmbD8lOTLJvcBu4PaqOuR+GvUzb7UHxVr1Z8B8VT0fuJ3vfXvQ9PgY8OyqegHwNuBPe65nJEmeBrwHeENVPd53PSvhMG2auf1UVU9U1QsZzGxxZpLnLfc1V3tQHHZKkOFtkhwF/CDwlYlUtzSjTHPylar6dnf3HcA/mVBt4zLKfpwpVfX4viGCGlw7dHSSdT2X1ZTkaAYfqDdU1XsPssnM7afDtWkW99M+VfU14EPAeQseWvRn3moPilGmBNkKbOyWLwY+WN1Rnil12DYtGBe+gMHY6yzbCryyO6vmLOCxqtrVd1HLkeSH940LJzmTwf/Fqf2C0tV6DfBAVf3eITabqf00SptmcD/NJTmuW/4+Br/t85kFmy36M2/qrsxeSXWIKUGS/Bawvaq2MviH8q4kOxgcfLy0v4oPb8Q2vS7JBcBeBm26rLeCR5Dk3QzOLlmXZCdwJYODcFTV2xlcqf8SYAfwLeBV/VQ6uhHadDHwi0n2An8HXDrlX1DOBn4O+FQ3/g3wq8ApMLP7aZQ2zdp+OhHYksGPwB0B3FxVty33M88rsyVJTat96EmStEwGhSSpyaCQJDUZFJKkJoNCktRkUEjLkOSiJKf3XYc0TgaFtDwXAQcNiu6qV2nmGRTSAkle0c3pf2+S/9lNsvaNJG/uJoe7K8kJSX6CwZXvv9Nt+5wkdyT5gyTbgdcnmU/ywW6Cxm1JTune47okb0+yPcnnkry0W39nkhcO1fKXSV7Qy1+E1DEopCFJ/jHwM8DZ3cRqTwAvB54K3NVNDncn8G+r6sMMpkP45ap6YVX9dfcyx1TVhqr6XQYTyW3pJmi8AXjr0NvNM5g2/nzg7UmOZXDV7GVdLT8KHFtVnxhnm6XDMSik/Z3LYBLFj3bTOpzL4EeivgPc1m1zD4MP+UO5aWj5xcD/7pbfBfzTocdurqrvVtXngQeB5wJ/DLy0m6zu1cB1y2mMtBIcQ5X2FwY9gDfttzL5paE5fp6g/X/nmyO+18L5c6qqvpXkdgY/LnMJsz/zr1YBexTS/rYBFyf5IYAkxyd5dmP7rzP4Gc1D+TDfm3Tt5cBfDD32r5MckeQ5DHotn+3Wv4PBENVHq+rRJbRBWlH2KKQhVXV/kl8H/jzJEcA/AK9pPOVG4H8leR2DmUYXei1wbZJfBvaw/4yqXwQ+Ajwd+IWq+vuuhnuSPA5cu+wGSSvA2WOlHiS5Dritqm45yGPPAu4AnltV351wadIBHHqSpkiSVzL43eZfMyQ0LexRSJKa7FFIkpoMCklSk0EhSWoyKCRJTQaFJKnJoJAkNf1/XDHOOmBx2ooAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "x = entropy_ls\n",
    "plt.hist(x, bins=100)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('entropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "entropys = np.array(entropy_ls)\n",
    "mean = np.mean(entropys)\n",
    "var = np.var(entropys)\n",
    "median = np.median(entropys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "entropy distribution stats\n",
      "mean:  1.0701602250174433 variance:  0.5310554398056302 median:  1.0589543187748816\n"
     ]
    }
   ],
   "source": [
    "print(\"entropy distribution stats\")\n",
    "print(\"mean: \", mean, \"variance: \", var, \"median: \",median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. random shuffle vocab entropy distribution plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_ls = list(range(N_vocab))\n",
    "N_buckets = 8\n",
    "N_vocab = 50257\n",
    "bucket_size = N_vocab // N_buckets\n",
    "\n",
    "np.random.shuffle(index_ls)\n",
    "\n",
    "entropy_ls = []\n",
    "for i in range(len(prob_set)):\n",
    "    for j in range(len(prob_set[i])):\n",
    "        whole_p = prob_set[i,j][index_ls]\n",
    "        \n",
    "        bins_p_ls = [np.sum(whole_p[k*bucket_size: (k+1)*bucket_size]) for k in range(N_buckets-1)]\n",
    "        bins_p_ls.append(np.sum(whole_p[(N_buckets-1)*bucket_size:]))\n",
    "        bins_p = np.array(bins_p_ls)\n",
    "        entropy = -bins_p @ np.log2(bins_p)\n",
    "        entropy_ls.append(entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'entropy')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFHlJREFUeJzt3X2wZHV95/H3hwc18WHRzISMwHiVwiRo6ai3WBOjxYa4wYcFNYZA+QCaOLrRJFbtZhfNVnStsspsJC7RjewgCFgGMRIMQUykSJSkEtQZggZRFFhcoSbMZGAZjEoc+O4ffQZ7rr8703fu7T7d975fVV1z+ndO9/n+pmfOp8/Tr1NVSJK00CF9FyBJmk4GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNh/VdwHKsW7eu5ubm+i5DkmbKtm3b/rmq1h9ouZkOiLm5ObZu3dp3GZI0U5J8c5TlPMQkSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqmuk7qSVpLZk7+1MPT9/xnpeMfX3uQUiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpKaxBUSSC5PsSHLTUNtlSW7sHnckubFrn0vy3aF5542rLknSaMZ5o9xFwAeAS/Y2VNWv7J1Ocg5w39Dyt1XVpjHWI0lagrEFRFVdl2SuNS9JgNOAnx/X+iVJy9PXOYjnA3dX1TeG2p6c5B+SfC7J83uqS5LU6WsspjOAS4eebwc2VtWuJM8BPpnkaVW1e+ELk2wGNgNs3LhxIsVK0lo08T2IJIcBrwAu29tWVQ9U1a5uehtwG/DU1uuraktVzVfV/Pr16ydRsiStSX0cYvoF4GtVdefehiTrkxzaTT8FOA64vYfaJEmdsR1iSnIpcCKwLsmdwDuq6gLgdPY9vATwAuBdSb4PPAS8qaruGVdtkjQrhof4nrRxXsV0xiLtZzXaLgcuH1ctkqSl805qSVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU1jC4gkFybZkeSmobZ3JrkryY3d48VD896W5NYktyT5xXHVJUkazTj3IC4CTm60v6+qNnWPqwGSHA+cDjyte80fJTl0jLVJkg5gbAFRVdcB94y4+KnAx6rqgar6P8CtwAnjqk2SdGB9nIN4S5Ivd4egHt+1HQV8a2iZO7s2SVJPJh0QHwSOBTYB24FzlvoGSTYn2Zpk686dO1e6PklSZ6IBUVV3V9WDVfUQcD4/OIx0F3DM0KJHd22t99hSVfNVNb9+/frxFixJa9hEAyLJhqGnLwf2XuF0JXB6kkcmeTJwHPCFSdYmSdrXYeN64ySXAicC65LcCbwDODHJJqCAO4A3AlTVV5J8HLgZ2AO8uaoeHFdtkqQDG1tAVNUZjeYL9rP8u4F3j6seSdLSeCe1JKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpKaxBUSSC5PsSHLTUNvvJ/laki8nuSLJEV37XJLvJrmxe5w3rrokSaMZ5x7ERcDJC9quAZ5eVc8Avg68bWjebVW1qXu8aYx1SZJGMLaAqKrrgHsWtH2mqvZ0T68Hjh7X+iVJy9PnOYjXA58eev7kJP+Q5HNJnt9XUZKkgcP6WGmS3wH2AB/tmrYDG6tqV5LnAJ9M8rSq2t147WZgM8DGjRsnVbIkTczc2Z/quwSghz2IJGcBLwVeVVUFUFUPVNWubnobcBvw1Nbrq2pLVc1X1fz69esnVLUkrT0TDYgkJwP/BTilqr4z1L4+yaHd9FOA44DbJ1mbJGlfYzvElORS4ERgXZI7gXcwuGrpkcA1SQCu765YegHwriTfBx4C3lRV9zTfWJI0EWMLiKo6o9F8wSLLXg5cPq5aJElL553UkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJahopIJJcO0qbJGn12O9w30keBfwog990eDyQbtbjgKPGXJskqUcH+j2INwJvBZ4IbOMHAbEb+MAY65Ik9Wy/AVFV5wLnJvmNqnr/hGqSJE2BkX5Rrqren+Rngbnh11TVJWOqS5LUs5ECIslHgGOBG4EHu+YCDAhJWqVG/U3qeeD4qqqlvHmSC4GXAjuq6uld2xOAyxjsjdwBnFZV9yYJcC7wYuA7wFlVdcNS1idJs2ru7E/1XcIPGfU+iJuAnziI978IOHlB29nAtVV1HHBt9xzgRcBx3WMz8MGDWJ8kaYWMugexDrg5yReAB/Y2VtUp+3tRVV2XZG5B86nAid30xcBngf/atV/S7aVcn+SIJBuqavuINUqSVtCoAfHOFVznkUMb/X8CjuymjwK+NbTcnV3bPgGRZDODPQw2bty4gmVJkoaNehXT58ax8qqqJEs6r1FVW4AtAPPz80t6rSRpdKMOtXF/kt3d43tJHkyy+yDXeXeSDd37bgB2dO13AccMLXd01yZJ6sFIAVFVj62qx1XV44AfAX4J+KODXOeVwJnd9JnAnw21vzYDzwXu8/yDJPVnyaO51sAngV880LJJLgX+HvjJJHcm+VXgPcALk3wD+IXuOcDVwO3ArcD5wK8vtTZJ0soZ9Ua5Vww9PYTBfRHfO9DrquqMRWad1Fi2gDePUo8kafxGvYrpPwxN72Fwg9upK16NJGlqjHoV0+vGXYgkabqMehXT0UmuSLKje1ye5OhxFydJ6s+oJ6k/zOAqoyd2jz/v2iRJq9SoAbG+qj5cVXu6x0XA+jHWJUnq2agBsSvJq5Mc2j1eDewaZ2GSpH6NGhCvB05jMHbSduCVwFljqkmSNAVGvcz1XcCZVXUvPPybDu9lEBySpFVo1D2IZ+wNB4Cqugd41nhKkiRNg1ED4pAkj9/7pNuDGHXvQ5I0g0bdyJ8D/H2SP+me/zLw7vGUJEmaBqPeSX1Jkq3Az3dNr6iqm8dX1mQM/wbsHe95SY+VSNL0GfkwURcIMx8KkqTRLHm4b0nS2mBASJKaDAhJUpMBIUlqMiAkSU0GhCSpaeJ3Qyf5SeCyoaanAL8LHAG8AdjZtb+9qq6ecHmSpM7EA6KqbgE2ASQ5FLgLuAJ4HfC+qnrvpGuSpD4M36w7jfo+xHQScFtVfbPnOiRJC/QdEKcDlw49f0uSLye5cHhwwGFJNifZmmTrzp07W4tIklZAbwGR5BHAKcDeAQA/CBzL4PDTdgYDBP6QqtpSVfNVNb9+vb96Kknj0ucexIuAG6rqboCquruqHqyqh4DzgRN6rE2S1rw+A+IMhg4vJdkwNO/lwE0Tr0iS9LBefvQnyaOBFwJvHGr+H0k2AQXcsWCeJGnCegmIqvoX4McWtL2mj1okSW19X8UkSZpSBoQkqcmAkCQ19XIOQpJWu8V+837ah9cY5h6EJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlq8j4ISRqzWbr3YZgBIUkHaeGGf/iGuNXAQ0ySpCYDQpLUZEBIkpoMCElSkwEhSWrq7SqmJHcA9wMPAnuqaj7JE4DLgDkGv0t9WlXd21eNkrSW9b0H8e+qalNVzXfPzwaurarjgGu755KkHvQdEAudClzcTV8MvKzHWiRpTeszIAr4TJJtSTZ3bUdW1fZu+p+AI/spTZLU553UP1dVdyX5ceCaJF8bnllVlaQWvqgLk80AGzdunEylkrQG9RYQVXVX9+eOJFcAJwB3J9lQVduTbAB2NF63BdgCMD8//0MBIkl9mdUxlxbTyyGmJI9O8ti908C/B24CrgTO7BY7E/izPuqTJPW3B3EkcEWSvTX8cVX9RZIvAh9P8qvAN4HTeqpPkh42vGew2gbk259eAqKqbgee2WjfBZw0+YokSQtN22WukqQp4e9BSNISrLYT0fvjHoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTd4oJ0mdtTrm0mIMCElqWEt3TC/GQ0ySpCYDQpLUZEBIkpoMCElSkyepJa05Xq00GvcgJElNBoQkqWnih5iSHANcAhwJFLClqs5N8k7gDcDObtG3V9XVk65P0vQax6Eh73dYXB/nIPYA/6mqbkjyWGBbkmu6ee+rqvf2UJMkaYGJB0RVbQe2d9P3J/kqcNSk65DUv3GfLPZk9PL0ehVTkjngWcDngecBb0nyWmArg72Me/urTlJflrphNwjGo7eT1EkeA1wOvLWqdgMfBI4FNjHYwzhnkddtTrI1ydadO3e2FpEkrYBe9iCSHM4gHD5aVX8KUFV3D80/H7iq9dqq2gJsAZifn6/xVytpGnlyefz6uIopwAXAV6vqD4baN3TnJwBeDtw06dokzT6DY+X0sQfxPOA1wD8mubFreztwRpJNDC59vQN4Yw+1SZI6fVzF9LdAGrO850HSD3GPoD/eSS1JanKwPklrgnsiS2dASBo771OYTQaEpKblbtT9xj77DAhJE2VwzA4DQtIBLdyoe5hobTAgJC2L5xdWLwOiwX/wkmRASBri+QENMyAkLZlBsjZ4J7Ukqck9iI7fiDStJvmra32+l/8Hp48BIS3TODbgS91YemGFxsGAOAD/4+lg9RUci63Xb+haKgNCU2mtBLMbbU0zA2KM1spGbhaN+i17HN/8J/lvwQDSchgQK8z/kCtvsb/TpW5oD+azWerhGj9/rSYGxBJM2x7BKPVMW82T5gZbOngGxApYzonDpb7PStWzkusYJXhG+Sa+nACbRBAaNlprpi4gkpwMnAscCnyoqt7Tc0lN07YxX6mQGte6l7P8uN9n0ma1bq09U3UndZJDgf8FvAg4HjgjyfH9ViVJa9O07UGcANxaVbcDJPkYcCpwc69VrUKjfoudxfMWfe3RSKvNtAXEUcC3hp7fCfzbnmoZm1ndII277ln9e5FWq2kLiANKshnY3D39dpJblvF264B/Xn5VvRtbP/J743jX/VotnwnYl2m0WvpBfm9ZfXnSKAtNW0DcBRwz9Pzoru1hVbUF2LISK0uytarmV+K9+rRa+gH2ZVqtlr6sln7AZPoyVSepgS8CxyV5cpJHAKcDV/ZckyStSVO1B1FVe5K8BfhLBpe5XlhVX+m5LElak6YqIACq6mrg6gmtbkUOVU2B1dIPsC/TarX0ZbX0AybQl1TVuNchSZpB03YOQpI0JVZ9QCQ5OcktSW5NcnZj/iOTXNbN/3ySuclXOZoR+nJWkp1Jbuwev9ZHnQeS5MIkO5LctMj8JPnDrp9fTvLsSdc4qhH6cmKS+4Y+k9+ddI2jSHJMkr9OcnOSryT5rcYyM/G5jNiXWflcHpXkC0m+1PXlvzeWGd82rKpW7YPBie7bgKcAjwC+BBy/YJlfB87rpk8HLuu77mX05SzgA33XOkJfXgA8G7hpkfkvBj4NBHgu8Pm+a15GX04Eruq7zhH6sQF4djf9WODrjX9fM/G5jNiXWflcAjymmz4c+Dzw3AXLjG0bttr3IB4euqOq/hXYO3THsFOBi7vpTwAnJckEaxzVKH2ZCVV1HXDPfhY5FbikBq4HjkiyYTLVLc0IfZkJVbW9qm7opu8HvspgZINhM/G5jNiXmdD9XX+7e3p491h44nhs27DVHhCtoTsW/kN5eJmq2gPcB/zYRKpbmlH6AvBL3e7/J5Ic05g/C0bt66z4me4QwaeTPK3vYg6kO0TxLAbfVofN3Oeyn77AjHwuSQ5NciOwA7imqhb9XFZ6G7baA2Kt+XNgrqqeAVzDD75VqD83AE+qqmcC7wc+2XM9+5XkMcDlwFuranff9SzHAfoyM59LVT1YVZsYjCxxQpKnT2rdqz0gDjh0x/AySQ4D/g2wayLVLc0ow5DsqqoHuqcfAp4zodpW2iif20yoqt17DxHU4B6fw5Os67mspiSHM9igfrSq/rSxyMx8Lgfqyyx9LntV1f8D/ho4ecGssW3DVntAjDJ0x5XAmd30K4G/qu5sz5Q5YF8WHA8+hcGx11l0JfDa7qqZ5wL3VdX2vos6GEl+Yu/x4CQnMPg/N3VfQLoaLwC+WlV/sMhiM/G5jNKXGfpc1ic5opv+EeCFwNcWLDa2bdjU3Um9kmqRoTuSvAvYWlVXMviH9JEktzI42Xh6fxUvbsS+/GaSU4A9DPpyVm8F70eSSxlcRbIuyZ3AOxicfKOqzmNwJ/2LgVuB7wCv66fSAxuhL68E/mOSPcB3gdOn9AvI84DXAP/YHe8GeDuwEWbucxmlL7PyuWwALs7gx9QOAT5eVVdNahvmndSSpKbVfohJknSQDAhJUpMBIUlqMiAkSU0GhCSpyYCQliHJy5Ic33cd0jgYENLyvAxoBkR3V6s0swwIaYEkr+7G4L8xyf/uBkv7dpJ3d4O7XZ/kyCQ/y+CO9d/vlj02yWeT/M8kW4HfSjKX5K+6ARSvTbKxW8dFSc5LsjXJ15O8tGu/LsmmoVr+Nskze/mL0JpnQEhDkvw08CvA87oB0h4EXgU8Gri+G9ztOuANVfV3DIY5+O2q2lRVt3Vv84iqmq+qcxgMBHdxN4DiR4E/HFrdHINh3F8CnJfkUQzuij2rq+WpwKOq6kvj7LO0GANC2tdJDAY5/GI3TMNJDH6k6V+Bq7pltjHYuC/msqHpnwH+uJv+CPBzQ/M+XlUPVdU3gNuBnwL+BHhpN9jc64GLltMZaTk8RirtKwy+8b9tn8bkPw+N1fMg+/+/8y8jrmvhODdVVd9Jcg2DH4E5jdkdkVergHsQ0r6uBV6Z5McBkjwhyZP2s/z9DH7WcjF/xw8GT3sV8DdD8345ySFJjmWwl3JL1/4hBoeivlhV9x5EH6QV4R6ENKSqbk7y34DPJDkE+D7w5v285GPA+Ul+k8EIoQv9BvDhJL8N7GTfEVD/L/AF4HHAm6rqe10N25LsBj687A5Jy+BorlIPklwEXFVVn2jMeyLwWeCnquqhCZcmPcxDTNIUSfJaBr+f/DuGg/rmHoQkqck9CElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqSm/w8DTUlE6vXVXAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "x = entropy_ls\n",
    "plt.hist(x, bins=100)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('entropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "entropy distribution stats\n",
      "mean:  2.305038478763132 variance:  0.7263859842824284 median:  2.674860520966889\n"
     ]
    }
   ],
   "source": [
    "entropys = np.array(entropy_ls)\n",
    "mean = np.mean(entropys)\n",
    "var = np.var(entropys)\n",
    "median = np.median(entropys)\n",
    "print(\"entropy distribution stats\")\n",
    "print(\"mean: \", mean, \"variance: \", var, \"median: \",median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. KL divergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFuFJREFUeJzt3X2QZXV95/H3h0fdVQNIBycz4LjKxkUTR+wgxlSWQGkQdx2ToOJGRSU1Zhd3tTZJiVR2RSvuYlXU9SHBHRUFlyiITxPFVYIYy6yAgxlGHkRHxWKmRhlBEaKiM373j/truU5OT99m5vS9t+f9qrrV5/zOQ38Pd7if/p2H301VIUnS7g4YdwGSpMlkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKmTASFJ6nTQuAvYG0ceeWStXr163GVI0lS5/vrrv1tVMwutN9UBsXr1ajZu3DjuMiRpqiT51ijreYpJktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1Gmqn6TeG6vP+cS8y247/5lLWIkkTabeehBJHpTkuiQ3JLkpyWtb+3uTfDPJpvZa09qT5K1JtiTZnOT4vmqTJC2szx7EfcDJVXVvkoOBzyf5ZFv2Z1V1+W7rPwM4tr2eDFzQfkqSxqC3HkQN3NtmD26v2sMma4GL23bXAIclWdFXfZKkPev1InWSA5NsAu4Arqyqa9ui17fTSG9OcmhrWwncPrT51ta2+z7XJdmYZOOOHTv6LF+S9mu9BkRV7aqqNcAq4IQkjwdeDTwW+A3gCOBVi9zn+qqararZmZkFhzOXJD1AS3Kba1V9H7gaOLWqtrfTSPcB7wFOaKttA44e2mxVa5MkjUGfdzHNJDmsTT8YeBrwlbnrCkkCPBu4sW2yAXhRu5vpRODuqtreV32SpD3r8y6mFcBFSQ5kEESXVdXHk3wmyQwQYBPwx239K4DTgC3AD4GX9FibJGkBvQVEVW0GntjRfvI86xdwdl/1SJIWx6E2JEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR16i0gkjwoyXVJbkhyU5LXtvZHJbk2yZYklyY5pLUf2ua3tOWr+6pNkrSwPnsQ9wEnV9UTgDXAqUlOBN4AvLmqHgN8DzirrX8W8L3W/ua2niRpTHoLiBq4t80e3F4FnAxc3tovAp7dpte2edryU5Kkr/okSXvW6zWIJAcm2QTcAVwJfB34flXtbKtsBVa26ZXA7QBt+d3Awzv2uS7JxiQbd+zY0Wf5krRf6zUgqmpXVa0BVgEnAI/dB/tcX1WzVTU7MzOz1zVKkrotyV1MVfV94GrgKcBhSQ5qi1YB29r0NuBogLb8l4A7l6I+SdI/1+ddTDNJDmvTDwaeBtzCIChOb6udCXysTW9o87Tln6mq6qs+SdKeHbTwKg/YCuCiJAcyCKLLqurjSW4GPpDkL4B/BN7d1n838L4kW4C7gDN6rE2StIDeAqKqNgNP7Gj/BoPrEbu3/xh4Tl/1SJIWxyepJUmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR16i0gkhyd5OokNye5KckrWvt5SbYl2dRepw1t8+okW5LcmuR3+6pNkrSwg3rc907gT6rqS0keClyf5Mq27M1V9ZfDKyc5DjgDeBzwK8DfJfnXVbWrxxolSfPorQdRVdur6ktt+h7gFmDlHjZZC3ygqu6rqm8CW4AT+qpPkrRnS3INIslq4InAta3p5Uk2J7kwyeGtbSVw+9BmW+kIlCTrkmxMsnHHjh09Vi1J+7feAyLJQ4APAa+sqh8AFwCPBtYA24E3LmZ/VbW+qmaranZmZmaf1ytJGug1IJIczCAcLqmqDwNU1XeqaldV/Qx4J/efRtoGHD20+arWJkkagz7vYgrwbuCWqnrTUPuKodV+D7ixTW8AzkhyaJJHAccC1/VVnyRpz/q8i+mpwAuBLyfZ1NrOBZ6fZA1QwG3AywCq6qYklwE3M7gD6mzvYJKk8ektIKrq80A6Fl2xh21eD7y+r5okSaPzSWpJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR16vNBuam1+pxPdLbfdv4zl7gSSRofexCSpE4GhCSpkwEhSepkQEiSOhkQkqROBoQkqZMBIUnqZEBIkjoZEJKkTgaEJKnTSAGR5KpR2iRJy8ceAyLJg5IcARyZ5PAkR7TXamDlAtseneTqJDcnuSnJK1r7EUmuTPK19vPw1p4kb02yJcnmJMfvm0OUJD0QC/UgXgZcDzy2/Zx7fQx4+wLb7gT+pKqOA04Ezk5yHHAOcFVVHQtc1eYBngEc217rgAsWfTSSpH1mj6O5VtVbgLck+c9V9bbF7LiqtgPb2/Q9SW5h0OtYC5zUVrsI+CzwqtZ+cVUVcE2Sw5KsaPuRJC2xkYb7rqq3JflNYPXwNlV18Sjbt1NSTwSuBY4a+tD/NnBUm14J3D602dbWZkBI0hiMFBBJ3gc8GtgE7GrNBSwYEEkeAnwIeGVV/SDJz5dVVSWpxRScZB2DU1Acc8wxi9lUkrQIo35h0CxwXDv9M7IkBzMIh0uq6sOt+Ttzp46SrADuaO3bgKOHNl/V2n5BVa0H1gPMzs4uqh5J0uhGfQ7iRuARi9lxBl2FdwO3VNWbhhZtAM5s02cyuOA91/6idjfTicDdXn+QpPEZtQdxJHBzkuuA++Yaq+pZe9jmqcALgS8n2dTazgXOBy5LchbwLeC5bdkVwGnAFuCHwEtGPQhJ0r43akCct9gdV9Xngcyz+JSO9Qs4e7G/R5LUj1HvYvr7vguRJE2WUe9iuofBXUsAhwAHA/9UVQ/rqzBJ0niN2oN46Nx0u/i8lsHT0ZKkZWrRo7nWwEeB3+2hHknShBj1FNPvD80ewOC5iB/3UpEkaSKMehfTvx+a3gncxuA0kyRpmRr1GoTPJEjSfmbULwxaleQjSe5orw8lWdV3cZKk8Rn1IvV7GAyF8Svt9betTZK0TI0aEDNV9Z6q2tle7wVmeqxLkjRmowbEnUlekOTA9noBcGefhUmSxmvUgHgpg0H1vs3gC3xOB17cU02SpAkw6m2urwPOrKrvASQ5AvhLBsEhSVqGRu1B/PpcOABU1V0MvkJUkrRMjRoQByQ5fG6m9SBG7X1IkqbQqB/ybwS+kOSDbf45wOv7KUmSNAlGfZL64iQbgZNb0+9X1c39lSVJGreRTxO1QDAUJGk/sejhviVJ+wcDQpLUqbeASHJhG9jvxqG285JsS7KpvU4bWvbqJFuS3JrELyOSpDHrswfxXuDUjvY3V9Wa9roCIMlxwBnA49o2f53kwB5rkyQtoLeAqKrPAXeNuPpa4ANVdV9VfRPYApzQV22SpIWN4xrEy5Nsbqeg5h6+WwncPrTO1tYmSRqTpQ6IC4BHA2sYDPr3xsXuIMm6JBuTbNyxY8e+rk+S1CxpQFTVd6pqV1X9DHgn959G2gYcPbTqqtbWtY/1VTVbVbMzM34lhST1ZUkDIsmKodnfA+bucNoAnJHk0CSPAo4FrlvK2iRJv6i3AfeSvB84CTgyyVbgNcBJSdYABdwGvAygqm5KchmDJ7V3AmdX1a6+apMkLSxVNe4aHrDZ2dnauHHjA9p29Tmf2Gd13Hb+M/fZviSpb0mur6rZhdbzSWpJUicDQpLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ16e5J6fzLfQ3c+QCdpmtmDkCR1MiAkSZ0MCElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUyYCQJHUyICRJnRxqo0cOwSFpmvXWg0hyYZI7ktw41HZEkiuTfK39PLy1J8lbk2xJsjnJ8X3VJUkaTZ+nmN4LnLpb2znAVVV1LHBVmwd4BnBse60DLuixLknSCHoLiKr6HHDXbs1rgYva9EXAs4faL66Ba4DDkqzoqzZJ0sKW+iL1UVW1vU1/GziqTa8Ebh9ab2trkySNydjuYqqqAmqx2yVZl2Rjko07duzooTJJEix9QHxn7tRR+3lHa98GHD203qrW9s9U1fqqmq2q2ZmZmV6LlaT92VLf5roBOBM4v/382FD7y5N8AHgycPfQqShJ2m+N83b53gIiyfuBk4Ajk2wFXsMgGC5LchbwLeC5bfUrgNOALcAPgZf0VZckaTS9BURVPX+eRad0rFvA2X3VIklaPJ+kHgOfsJY0DRyLSZLUyYCQJHUyICRJnQwISVInA0KS1MmAkCR1MiAkSZ0MCElSJwNCktTJgJAkdXKojQniEBySJok9CElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLUydtcp4C3v0oaB3sQkqROY+lBJLkNuAfYBeysqtkkRwCXAquB24DnVtX3xlGfJGm8PYjfqao1VTXb5s8BrqqqY4Gr2rwkaUwm6RrEWuCkNn0R8FngVeMqZjnyWoakxRhXD6KATye5Psm61nZUVW1v098GjhpPaZIkGF8P4reqaluSXwauTPKV4YVVVUmqa8MWKOsAjjnmmP4rnULz9RQkaTHG0oOoqm3t5x3AR4ATgO8kWQHQft4xz7brq2q2qmZnZmaWqmRJ2u8seQ8iyb8EDqiqe9r004HXARuAM4Hz28+PLXVt02Zf9RS8NiGpyzhOMR0FfCTJ3O//m6r6v0m+CFyW5CzgW8Bzx1CbJKlZ8oCoqm8AT+hovxM4ZanrkSR180lqSVInA0KS1GmSHpTThNnTRXAvYEvLnwGhB8Q7n6Tlz4DQWBk00uTyGoQkqZMBIUnq5Ckm7VOeMpKWDwNCS8IBBKXp4ykmSVInexCaSJ6q0v5mEnvZ9iAkSZ3sQWi/ZA9FWpgBoamy2A/2xXbbDQ7pfp5ikiR1sgehZWGcF/gW+7vtjWhaGBDShBjX6S1Pq2k+BoQ0gkm8BdEP9vFb7u+BASEtsUkMG+1by+W7VAwIacLtq0BZzsHU97Htyw/1aXofJi4gkpwKvAU4EHhXVZ0/5pKkqdL3B9C+3P80/TW9P5qogEhyIPBXwNOArcAXk2yoqpvHW5m0/5mmO8P6Nmn1LJVJew7iBGBLVX2jqn4CfABYO+aaJGm/NFE9CGAlcPvQ/FbgyWOqRVLP9te/zKfFpAXEgpKsA9a12XuT3PoAd3Uk8N19U9XYeSyTabkcy3I5DlhGx5I37NWxPHKUlSYtILYBRw/Nr2ptP1dV64H1e/uLkmysqtm93c8k8Fgm03I5luVyHOCxLNakXYP4InBskkclOQQ4A9gw5pokab80UT2IqtqZ5OXApxjc5nphVd005rIkab80UQEBUFVXAFcswa/a69NUE8RjmUzL5ViWy3GAx7Ioqaq+f4ckaQpN2jUISdKEWPYBkeTUJLcm2ZLknI7lhya5tC2/Nsnqpa9yNCMcy4uT7Eiyqb3+aBx1LiTJhUnuSHLjPMuT5K3tODcnOX6paxzVCMdyUpK7h96T/77UNY4iydFJrk5yc5KbkryiY52peF9GPJZpeV8elOS6JDe0Y3ltxzr9fYZV1bJ9MbjQ/XXgXwGHADcAx+22zn8C3tGmzwAuHXfde3EsLwbePu5aRziW3waOB26cZ/lpwCeBACcC14675r04lpOAj4+7zhGOYwVwfJt+KPDVjn9fU/G+jHgs0/K+BHhImz4YuBY4cbd1evsMW+49iFGG7lgLXNSmLwdOSZIlrHFUy2YYkqr6HHDXHlZZC1xcA9cAhyVZsTTVLc4IxzIVqmp7VX2pTd8D3MJgZINhU/G+jHgsU6H9t763zR7cXrtfOO7tM2y5B0TX0B27/0P5+TpVtRO4G3j4klS3OKMcC8AftO7/5UmO7lg+DUY91mnxlHaK4JNJHjfuYhbSTlE8kcFfq8Om7n3Zw7HAlLwvSQ5Msgm4A7iyquZ9X/b1Z9hyD4j9zd8Cq6vq14Eruf+vCo3Pl4BHVtUTgLcBHx1zPXuU5CHAh4BXVtUPxl3P3ljgWKbmfamqXVW1hsHIEickefxS/e7lHhALDt0xvE6Sg4BfAu5ckuoWZ5RhSO6sqvva7LuAJy1RbfvaKO/bVKiqH8ydIqjBMz4HJzlyzGV1SnIwgw/US6rqwx2rTM37stCxTNP7Mqeqvg9cDZy626LePsOWe0CMMnTHBuDMNn068JlqV3smzILHstv54GcxOPc6jTYAL2p3zZwI3F1V28dd1AOR5BFz54OTnMDg/7mJ+wOk1fhu4JaqetM8q03F+zLKsUzR+zKT5LA2/WAG35Xzld1W6+0zbOKepN6Xap6hO5K8DthYVRsY/EN6X5ItDC42njG+iuc34rH8lyTPAnYyOJYXj63gPUjyfgZ3kRyZZCvwGgYX36iqdzB4kv40YAvwQ+Al46l0YSMcy+nAf0yyE/gRcMaE/gHyVOCFwJfb+W6Ac4FjYOrel1GOZVrelxXARRl8mdoBwGVV9fGl+gzzSWpJUqflfopJkvQAGRCSpE4GhCSpkwEhSepkQEiSOhkQmmhJ7h2aPi3JV5M8Msl5Sf50Eft5cZK3t+k/TvKiPuodl/ZswmeSPCzJ6sw/uuy7khzXps8daj8kyefag1YSYEBoSiQ5BXgr8Iyq+tbe7Kuq3lFVF++Dmibpw/Q04IaFhseoqj+qqpvb7LlD7T8BrgKe11+JmjYGhCZekt8G3gn8u6r6+iK2e0nrcVzH4OGpufbzkvxpkse2ZXPtq5N8uU0/KcnfJ7k+yafmnlJP8tkk/yvJRuAVSR6d5JokX07yF7v1eP4syRfb4ImvHfodtyR5Zwbj+3+6PSFLksck+bs2gNyXkjx6vv10+EPgY0PzByW5pP2uy5P8i6H6Z5OcDzw4g+9CuKRt89G2HwkwIDT5DmXwwfXsqtp9iIF5tQ/01zIIht8Cjtt9nba/Q5I8qjU9D7g0g3F83gacXlVPAi4EXj+06SFVNVtVbwTeArylqn6Nweimc7//6cCxDIZpXwM8qQUdrf2vqupxwPeBP2jtl7T2JwC/CWxfYD/DngpcPzT/q8BfV9W/AX7A4DsDho/9HOBHVbWmquZC4UbgNzr2rf2UAaFJ91Pg/wFnLXK7JwOfraod7fTJpfOsdxn3n1Z5XlvvV4HHA1e2oRr+nMHAdHOG9/UU4INt+m+G2p/eXv/IYOTQxzL4oAf4ZlXNDQFxPbA6yUOBlVX1EYCq+nFV/XCB/Qw7on33wZzbq+of2vT/YRCSe1RVu4CftFqk5T0Wk5aFnwHPBa5Kcm5V/Y99vP9LgQ8m+TCD72f5WpJfA26qqqfMs80/jbDfAP+zqv73LzQOvp/gvqGmXcCDF7ufDjuTHFBVP2vzu4+hM+qYOocCPx5xXS1z9iA08dpf0s8E/jDJqD2Ja4F/m+Th7ZTRc+bZ99cZfEj/N+7vGdwKzCR5CgyGjs78XyhzDfefIhoeJO1TwEsz+E4CkqxM8svzFdv++t+a5Nlt/UPbdYNR93Mrg6+jnXPMXP3AfwA+37HNT9t/G9q+Hw58t6p+Ol+d2r8YEJoKVXUXg3Hw/7yNWEub3jr32m397cB5wBeAf2DPQ59fCryAwemmuTt6TgfekOQGYBODawJdXgn81ySbgccw+DYvqurTDE45faFd+L6cwfcj78kLGYzIu5nBabVHLGI/n2AwquycW4Gzk9wCHA5c0LHNemDz0EXq32n7kQBHc5X2Svsr/0dVVUnOAJ5fVUv+XeHtovzFVfW0vdjHh4Fzquqr+64yTTOvQUh750nA25OEwR1JLx1HEVW1vd06+7AH8lWhGXwJ1UcNBw2zByFJ6uQ1CElSJwNCktTJgJAkdTIgJEmdDAhJUicDQpLU6f8Dg9ct5dry2VsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# %matplotlib inline\n",
    "f = plt.figure()\n",
    "x = 3 - np.array(entropy_ls)\n",
    "plt.hist(x, bins=50)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('KL divergence (bit)')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "f.savefig(\"KL.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. huffman KL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.18086862564086914\n",
      "0.1814568042755127\n",
      "0.1811816692352295\n",
      "0.18133974075317383\n",
      "0.18090343475341797\n",
      "0.18214631080627441\n",
      "0.18069911003112793\n",
      "0.18113327026367188\n",
      "0.1812286376953125\n",
      "0.1816577911376953\n",
      "0.18108654022216797\n",
      "0.18172192573547363\n",
      "0.18178367614746094\n",
      "0.18185162544250488\n",
      "0.181898832321167\n",
      "0.1808938980102539\n",
      "0.18141889572143555\n",
      "0.18092942237854004\n",
      "0.1816849708557129\n",
      "0.1816234588623047\n",
      "0.18149399757385254\n",
      "0.1814732551574707\n",
      "0.18223285675048828\n",
      "0.18181228637695312\n",
      "0.18217682838439941\n",
      "0.18214178085327148\n",
      "0.18228960037231445\n",
      "0.18323874473571777\n",
      "0.18213105201721191\n",
      "0.18186211585998535\n",
      "0.18234539031982422\n",
      "0.18163323402404785\n",
      "0.18158507347106934\n",
      "0.18282079696655273\n",
      "0.1819901466369629\n",
      "0.18200302124023438\n",
      "0.18413567543029785\n",
      "0.18240809440612793\n",
      "0.18257498741149902\n",
      "0.1838221549987793\n",
      "0.18305325508117676\n",
      "0.1818842887878418\n",
      "0.1819782257080078\n",
      "0.18168950080871582\n",
      "0.1821756362915039\n",
      "0.18121027946472168\n",
      "0.18140721321105957\n",
      "0.18227052688598633\n",
      "0.1817631721496582\n",
      "0.18213152885437012\n"
     ]
    }
   ],
   "source": [
    "from huffman import build_min_heap, huffman_tree, tv_huffman\n",
    "import time \n",
    "entropy_ls = []\n",
    "for i in range(len(prob_set)):\n",
    "    for j in range(len(prob_set[i])):\n",
    "        p = prob_set[i,j]\n",
    "        heap = build_min_heap(p)\n",
    "        start_time = time.time()\n",
    "        tree = huffman_tree(heap)\n",
    "        end_time = time.time()\n",
    "        time_diff = end_time - start_time\n",
    "        entropy = tv_huffman(tree, p)\n",
    "        entropy_ls.append(entropy)\n",
    "    print(time_diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "kl_ls = [kl for tv, kl in entropy_ls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFNlJREFUeJzt3X+0ZWV93/H3B/mhRJEfMyE4kAxVoiHaRJwihi7zYxKCmDKsRBErYURa2pQaDUmaqbULY5oWVmMMqMWgoJBSAxIj00pL6ABhxQhxEER+BBkRZCjIKD80EgLot3+c52YOZH6cB+45556579dad929n/2cfb7P3LnzmWfvffZOVSFJ0qh2mnYBkqTZYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSeqy87QLGIclS5bU8uXLp12GJM2U66+//htVtXR7/XbI4Fi+fDnr16+fdhmSNFOS3D1KPw9VSZK6GBySpC5jC44k5yV5IMnNQ217J7kiyR3t+16tPUnOSrIhyU1JDhl6zerW/44kq8dVryRpNOOccXwcOPJpbWuAdVV1ELCurQO8DjiofZ0MnA2DoAFOA14NHAqcNhc2kqTpGFtwVNU1wINPa14FnN+WzweOGWq/oAauBfZMsh/w88AVVfVgVT0EXME/DCNJ0gRN+hzHvlV1X1u+H9i3LS8D7hnqt7G1ba1dkjQlUzs5XoNHD87b4weTnJxkfZL1mzZtmq/dSpKeZtLB8fV2CIr2/YHWfi9wwFC//Vvb1tr/gao6p6pWVNWKpUu3+/kVSdIzNOngWAvMXRm1Grh0qP2EdnXVYcAj7ZDW5cARSfZqJ8WPaG2SpCkZ2yfHk3wC+ClgSZKNDK6OOh24OMlJwN3Asa37ZcBRwAbgUeBEgKp6MMnvAJ9v/d5bVU8/4T4xy9d8Zovtd53++glXIknTM7bgqKo3b2XTyi30LeCUreznPOC8eSxNkvQs+MlxSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUpepBEeSX0tyS5Kbk3wiyXOTHJjkuiQbklyUZNfWd7e2vqFtXz6NmiVJAxMPjiTLgF8FVlTVy4HnAMcBZwDvr6qXAA8BJ7WXnAQ81Nrf3/pJkqZkWoeqdgael2RnYHfgPuBngEva9vOBY9ryqrZO274ySSZYqyRpyMSDo6ruBX4P+BqDwHgEuB54uKqebN02Asva8jLgnvbaJ1v/fSZZsyRps2kcqtqLwSziQOBFwPcBR87Dfk9Osj7J+k2bNj3b3UmStmIah6p+FvhqVW2qqieATwGHA3u2Q1cA+wP3tuV7gQMA2vYXAt98+k6r6pyqWlFVK5YuXTruMUjSojWN4PgacFiS3du5ipXArcBVwBtan9XApW15bVunbb+yqmqC9UqShkzjHMd1DE5yfwH4UqvhHOC3gFOTbGBwDuPc9pJzgX1a+6nAmknXLEnabOftd5l/VXUacNrTmu8EDt1C38eAN06iLknS9vnJcUlSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVKXqQRHkj2TXJLkr5PcluQ1SfZOckWSO9r3vVrfJDkryYYkNyU5ZBo1S5IGpjXjOBP4P1X1MuDHgNuANcC6qjoIWNfWAV4HHNS+TgbOnny5kqQ5Ew+OJC8EXgucC1BVj1fVw8Aq4PzW7XzgmLa8CrigBq4F9kyy34TLliQ105hxHAhsAj6W5IYkH03yfcC+VXVf63M/sG9bXgbcM/T6ja1NkjQF0wiOnYFDgLOr6pXAd9h8WAqAqiqgenaa5OQk65Os37Rp07wVK0l6qmkEx0ZgY1Vd19YvYRAkX587BNW+P9C23wscMPT6/VvbU1TVOVW1oqpWLF26dGzFS9JiN/HgqKr7gXuSvLQ1rQRuBdYCq1vbauDStrwWOKFdXXUY8MjQIS1J0oTtPKX3fTtwYZJdgTuBExmE2MVJTgLuBo5tfS8DjgI2AI+2vpKkKZlKcFTVjcCKLWxauYW+BZwy9qIkSSMZ6VBVknWjtEmSdnzbnHEkeS6wO7CkfZI7bdMeeEmsJC1K2ztU9a+AdwIvAq5nc3B8C/jgGOuSJC1Q2wyOqjoTODPJ26vqAxOqSZK0gI10cryqPpDkJ4Dlw6+pqgvGVJckaYEaKTiS/BHwYuBG4LutuQCDQ5IWmVEvx10BHNwujZUkLWKjfnL8ZuAHxlmIJGk2jDrjWALcmuSvgL+ba6yqo8dSlSRpwRo1ON4zziIkSbNj1Kuq/nzchUiSZsOoV1V9m83Px9gV2AX4TlXtMa7Cpmn5ms9MuwRJWrBGnXG8YG45SRg8zvWwcRUlSVq4up/H0Z79/Wng58dQjyRpgRv1UNUvDq3uxOBzHY+NpSJJ0oI26lVV/2xo+UngLgaHqyRJi8yo5zh86p4kCRj9QU77J/nTJA+0rz9Jsv+4i5MkLTyjnhz/GLCWwXM5XgT8z9YmSVpkRg2OpVX1sap6sn19HFg6xrokSQvUqMHxzSTHJ3lO+zoe+OY4C5MkLUyjBsfbgGOB+4H7gDcAbx1TTZKkBWzUy3HfC6yuqocAkuwN/B6DQJEkLSKjzjj+8VxoAFTVg8Arx1OSJGkhGzU4dkqy19xKm3GMOluRJO1ARv3H/33A55J8sq2/Efjd8ZQkSVrIRv3k+AVJ1gM/05p+sapuHV9ZkqSFauTDTS0oDAtJWuS6b6suSVrcDA5JUheDQ5LUxeCQJHUxOCRJXaYWHO1miTck+V9t/cAk1yXZkOSiJLu29t3a+oa2ffm0apYkTXfG8Q7gtqH1M4D3V9VLgIeAk1r7ScBDrf39rZ8kaUqmEhzt6YGvBz7a1sPgw4WXtC7nA8e05VVtnbZ9ZesvSZqCac04/gD4d8D32vo+wMNV9WRb3wgsa8vLgHsA2vZHWn9J0hRMPDiS/ALwQFVdP8/7PTnJ+iTrN23aNJ+7liQNmcaM43Dg6CR3AX/M4BDVmcCeSeZugbI/cG9bvhc4AKBtfyFbePpgVZ1TVSuqasXSpT7VVpLGZeLBUVX/vqr2r6rlwHHAlVX1FuAqBk8WBFgNXNqW17Z12vYrq6omWLIkachC+hzHbwGnJtnA4BzGua39XGCf1n4qsGZK9UmSmPLDmKrqauDqtnwncOgW+jzG4PkfkqQFYCHNOCRJM8DgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUZeLBkeSAJFcluTXJLUne0dr3TnJFkjva971ae5KclWRDkpuSHDLpmiVJm01jxvEk8OtVdTBwGHBKkoOBNcC6qjoIWNfWAV4HHNS+TgbOnnzJkqQ5Ew+Oqrqvqr7Qlr8N3AYsA1YB57du5wPHtOVVwAU1cC2wZ5L9Jly2JKmZ6jmOJMuBVwLXAftW1X1t0/3Avm15GXDP0Ms2tjZJ0hRMLTiSPB/4E+CdVfWt4W1VVUB17u/kJOuTrN+0adM8VipJGjaV4EiyC4PQuLCqPtWavz53CKp9f6C13wscMPTy/VvbU1TVOVW1oqpWLF26dHzFS9IiN42rqgKcC9xWVb8/tGktsLotrwYuHWo/oV1ddRjwyNAhLUnShO08hfc8HPhl4EtJbmxt7wJOBy5OchJwN3Bs23YZcBSwAXgUOHGy5UqShk08OKrqL4BsZfPKLfQv4JSxFiVJGpmfHJckdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1mcYtR3Y4y9d8Zovtd53++glXIknj54xDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MVPjkvSDJrmHSuccUiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6uIHAMfIR8pK2hE545AkdTE4JEldDA5JUhfPcUyB5z4kzbKZmXEkOTLJ7Uk2JFkz7XokabGaiRlHkucAHwJ+DtgIfD7J2qq6dbqVTYYzFEkLyUwEB3AosKGq7gRI8sfAKmCHCo6tBcR89d9a0GxrP4aT1K/3d3NbFuLv4KwExzLgnqH1jcCrp1TLzHomf5nn8xdgsXkmQa0B/+w2W4hjnpXg2K4kJwMnt9W/SXJ75y6WAN+Y36pmwmIc90TGnDPG/Q5dZurnPI9/djM17vmQM57VmH9olE6zEhz3AgcMre/f2v5eVZ0DnPNM3yDJ+qpa8UxfP6sW47gd8+KxGMc9iTHPylVVnwcOSnJgkl2B44C1U65JkhalmZhxVNWTSf4tcDnwHOC8qrplymVJ0qI0E8EBUFWXAZeN8S2e8WGuGbcYx+2YF4/FOO6xjzlVNe73kCTtQGblHIckaYFYdMGxvVuXJNktyUVt+3VJlk++yvk1wphPTXJrkpuSrEsy0iV5C92ot6lJ8ktJKsnMX30zypiTHNt+3rck+R+TrnG+jfD3+weTXJXkhvZ3/Khp1DmfkpyX5IEkN29le5Kc1f5MbkpyyLwWUFWL5ovBifWvAP8I2BX4InDw0/r8G+DDbfk44KJp1z2BMf80sHtb/pVZH/Oo4279XgBcA1wLrJh23RP4WR8E3ADs1da/f9p1T2DM5wC/0pYPBu6adt3zMO7XAocAN29l+1HA/wYCHAZcN5/vv9hmHH9/65KqehyYu3XJsFXA+W35EmBlkkywxvm23TFX1VVV9WhbvZbB52Rm3Sg/a4DfAc4AHptkcWMyypj/JfChqnoIoKoemHCN822UMRewR1t+IfD/JljfWFTVNcCD2+iyCrigBq4F9kyy33y9/2ILji3dumTZ1vpU1ZPAI8A+E6luPEYZ87CTGPxPZdZtd9xt+n5AVS28ezo8M6P8rH8Y+OEkn01ybZIjJ1bdeIwy5vcAxyfZyODKzLdPprSp6v297zIzl+Nq/JIcD6wAfnLatYxbkp2A3wfeOuVSJm1nBoerforBzPKaJK+oqoenWtV4vRn4eFW9L8lrgD9K8vKq+t60C5tVi23Gsd1blwz3SbIzg6ntNydS3XiMMmaS/CzwH4Cjq+rvJlTbOG1v3C8AXg5cneQuBseB1874CfJRftYbgbVV9URVfRX4MoMgmVWjjPkk4GKAqvoc8FwG97DakY30e/9MLbbgGOXWJWuB1W35DcCV1c42zajtjjnJK4E/ZBAas37Me842x11Vj1TVkqpaXlXLGZzbObqq1k+n3Hkxyt/vTzOYbZBkCYNDV3dOssh5NsqYvwasBEjyIwyCY9NEq5y8tcAJ7eqqw4BHquq++dr5ojpUVVu5dUmS9wLrq2otcC6DqewGBiefjptexc/eiGP+r8DzgU+26wC+VlVHT63oeTDiuHcoI475cuCIJLcC3wV+s6pmdkY94ph/HfhIkl9jcKL8rTP+n0GSfILBfwCWtHM3pwG7AFTVhxmcyzkK2AA8Cpw4r+8/439+kqQJW2yHqiRJz5LBIUnqYnBIkroYHJKkLgaHJKmLwaGZlORvhpaPSvLlJD+U5D1JfqNjP29N8sG2/K+TnDCOeqelXcd/ZZI9kizfxt1UP5rk4Lb8rqH2XZNc0z4MKwEGh2ZckpXAWcDrquruZ7OvqvpwVV0wDzUtpH9kjwK+WFXf2lanqvoXVXVrW33XUPvjwDrgTeMrUbPG4NDMSvJa4CPAL1TVVzped2KbofwVcPhQ+3uS/EaSl7Vtc+3Lk3ypLb8qyZ8nuT7J5XN3HE1ydZI/SLIeeEeSF7ebCH4pyX962gzpN5N8vj0n4beH3uO2JB/J4DkZf5bkeW3bS5L83yRfTPKFJC/e2n624C3ApUPrOye5sL3XJUl2H6p/RZLTgecluTHJhe01n277kQCDQ7NrNwb/oB1TVX896ovaP/S/zSAw/imD5zM8RdvfrkkObE1vAi5KsgvwAeANVfUq4Dzgd4deumtVraiq9wFnAmdW1SsY3B9q7v2PYHBvqEOBHwde1QKQ1v6hqvpR4GHgl1r7ha39x4CfAO7bzn6GHQ5cP7T+UuC/VdWPAN9i8PyZ4bGvAf62qn68qubC4mbgn2xh31qkDA7NqieAv2RwA7serwaurqpN7TDMRVvpdzGbD8+8qfV7KYMbI16R5Ebg3Tz12SXD+3oN8Mm2PPyUvSPa1w3AF4CXsfkmg1+tqhvb8vXA8iQvAJZV1Z8CVNVj7dkp29rPsL2r6ttD6/dU1Wfb8n9nEJ7bVFXfBR5vtUiL615V2qF8DzgWWJfkXVX1n+d5/xcxuHfXp4CqqjuSvAK4papes5XXfGeE/Qb4L1X1h09pHDyiePiuxN8Fnte7ny14MslOQ7cQf/o9hka959Bu7BgPu9I8cMahmdX+5/164C1JRp15XAf8ZJJ92qGnN25l319h8I/3f2TzTOJ2YGkGz3QgyS5JfnQr73Mtmw81Dd8o83LgbUme3/axLMn3b63YNlvYmOSY1n+3dl5i1P3czuCxqnN+cK5+4J8Df7GF1zzR/mxo+94H+EZVPbG1OrW4GByaaVX1IHAk8O4kc3f0fXeSjXNfT+t/H4Mnwn0O+Cxw2zZ2fxFwPJuf5fA4g1vtn5Hki8CNDM45bMk7gVOT3AS8hMGTJKmqP2Nw6Opz7YT7JQyeDbItvwz8atvXXwI/0LGfz9Buo97cDpyS5DZgL+DsLbzmHOCmoZPjP932IwHeHVcaizYr+NuqqiTHAW+uqi0983zcdezH4NnTP/cs9vEpYE1VfXn+KtMs8xyHNB6vAj6YJAyukHrbNIqoqvvaJb57bO+zHFuSwcORPm1oaJgzDklSF89xSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQu/x8KWAn0KbtEdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "# %matplotlib inline\n",
    "f = plt.figure()\n",
    "x = np.array(kl_ls)\n",
    "plt.hist(x, bins=50)\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('KL divergence (bit)')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "f.savefig(\"huffman_KL.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1236073943203487 0.06345455840699488 0.030881696101359966\n"
     ]
    }
   ],
   "source": [
    "entropys = np.array(kl_ls)\n",
    "mean = np.mean(entropys)\n",
    "var = np.var(entropys)\n",
    "median = np.median(entropys)\n",
    "print(\"entropy distribution stats\")\n",
    "print(\"mean: \", mean, \"variance: \", var, \"median: \",median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. some stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for kl in kl_ls:\n",
    "    if kl > 0.9:\n",
    "        count += 1\n",
    "print(\"kl greater than 0.9 numbers: \", count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_kl = 1000.0\n",
    "for kl in kl_ls:\n",
    "    if kl < min_kl:\n",
    "        min_kl = kl\n",
    "print(\"min kl: \", min_kl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1696\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFLFJREFUeJzt3Xu0pXV93/H3RxG8V5CRwAA5xGDSoVHUE7yulIQVQUjFWIOQKATtmnQFW1nV1JHYFcwqzZilsagpCUYUWhJAFKWBFYVpjDWCOEMAuQQdYSgzHWFEK6gRHPLtH/sZ2Ux+M7PP7POcvc+Z92utvc6zf89lf39zYH/O77mmqpAkaXtPmHQBkqTpZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1LTXpAsYx/77718zMzOTLkOSFpV169Z9q6qW7Wq5RR0QMzMzrF27dtJlSNKikuSeUZZzF5MkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKlpUV9JrcmZWXVVs33D6hMWuBJJfXEEIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTb0FRJJDkvx1ktuT3JbkbV372Uk2Jbmpex0/tM67kqxPcmeSY/uqTZK0a30+k3or8PaqujHJM4B1Sa7p5n2gqt43vHCSFcDJwBHAQcC1SZ5XVY/2WKMkaQd6G0FU1eaqurGbfgi4A1i+k1VOBC6pqoer6m5gPXBUX/VJknZuQY5BJJkBXgh8uWt6a5JbklyQZN+ubTlw79BqG9l5oEiSetR7QCR5OvBJ4MyqehA4D3gucCSwGXj/HLe3MsnaJGu3bNky7/VKkgb6PAZBkicxCIeLq+pTAFV139D8jwB/2b3dBBwytPrBXdvjVNX5wPkAs7Oz1U/l2l0zq65qtm9YfcICVyJpXH2exRTgo8AdVfVHQ+0HDi32q8Ct3fSVwMlJ9klyGHA4cENf9UmSdq7PEcQrgDcBX01yU9d2FnBKkiOBAjYAvwVQVbcluQy4ncEZUGd4BpMkTU5vAVFVXwTSmHX1TtY5Bzinr5okSaPzSmpJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpqbeASHJIkr9OcnuS25K8rWvfL8k1Sb7e/dy3a0+SDyZZn+SWJC/qqzZJ0q71OYLYCry9qlYALwXOSLICWAWsqarDgTXde4BXA4d3r5XAeT3WJknahd4Coqo2V9WN3fRDwB3AcuBE4MJusQuB13bTJwIX1cD1wLOSHNhXfZKknVuQYxBJZoAXAl8GDqiqzd2sbwIHdNPLgXuHVtvYtW2/rZVJ1iZZu2XLlt5qlqQ9Xe8BkeTpwCeBM6vqweF5VVVAzWV7VXV+Vc1W1eyyZcvmsVJJ0rBeAyLJkxiEw8VV9amu+b5tu466n/d37ZuAQ4ZWP7hrkyRNQJ9nMQX4KHBHVf3R0KwrgdO66dOAzwy1n9qdzfRS4LtDu6IkSQtsrx63/QrgTcBXk9zUtZ0FrAYuS/IW4B7gpG7e1cDxwHrgB8DpPdYmSdqF3gKiqr4IZAezj2ksX8AZfdUjSZobr6SWJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNfV4oJ/3YzKqrmu0bVp+wwJVIGpUjCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDWNFBBJ1ozSJklaOnZ6u+8kTwaeCuyfZF8g3axnAst7rk2SNEG7eh7EbwFnAgcB63gsIB4EPtxjXZKkCdtpQFTVucC5Sf5dVX1ogWqSJE2BkZ4oV1UfSvJyYGZ4naq6qKe6JEkTNlJAJPnvwHOBm4BHu+YCDAhJWqJGfSb1LLCiqqrPYiRJ02PU6yBuBX5iLhtOckGS+5PcOtR2dpJNSW7qXscPzXtXkvVJ7kxy7Fw+S5I0/0YdQewP3J7kBuDhbY1V9ZqdrPNxBmc6bb8b6gNV9b7hhiQrgJOBIxicMXVtkudV1aNIkiZi1IA4e64brqovJJkZcfETgUuq6mHg7iTrgaOA6+b6uZKk+THqWUx/M4+f+dYkpwJrgbdX1XcYXHR3/dAyG9nBhXhJVgIrAQ499NB5LEuSNGzUW208lOTB7vXDJI8meXA3Pu88BmdDHQlsBt4/1w1U1flVNVtVs8uWLduNEiRJoxh1BPGMbdNJwmCX0Evn+mFVdd/Qdj4C/GX3dhNwyNCiB3dtkqQJmfPdXGvg08CczzRKcuDQ219lcHYUwJXAyUn2SXIYcDhww1y3L0maP6NeKPe6obdPYHBdxA93sc5fAEczuNHfRuD3gKOTHMngIrsNDO71RFXdluQy4HZgK3CGZzBJ0mSNehbTvxqa3srgy/3Ena1QVac0mj+6k+XPAc4ZsR5JUs9GPQZxet+FSJKmy6hnMR2c5Iruyuj7k3wyycF9FydJmpxRD1J/jMGB5IO61//s2iRJS9SoAbGsqj5WVVu718cBL0KQpCVs1IB4IMkbkzyxe70ReKDPwiRJkzXqWUxvBj4EfIDBKapfAn6zp5o0JWZWXTXpEiRN0KgB8fvAad19k0iyH/A+BsEhSVqCRt3F9Pxt4QBQVd8GXthPSZKkaTBqQDwhyb7b3nQjiFFHH5KkRWjUL/n3A9cl+UT3/tfwqmdJWtJGvZL6oiRrgV/qml5XVbf3V5YkadJG3k3UBYKhIEl7iDnf7luStGcwICRJTQaEJKnJgJAkNRkQkqQmL3bTRO3ofk8bVp+wwJVI2p4jCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUlNvAZHkgiT3J7l1qG2/JNck+Xr3c9+uPUk+mGR9kluSvKivuiRJo+lzBPFx4Ljt2lYBa6rqcGBN9x7g1cDh3WslcF6PdUmSRtBbQFTVF4Bvb9d8InBhN30h8Nqh9otq4HrgWUkO7Ks2SdKuLfQxiAOqanM3/U3ggG56OXDv0HIbuzZJ0oRM7CB1VRVQc10vycoka5Os3bJlSw+VSZJg4QPivm27jrqf93ftm4BDhpY7uGv7J6rq/KqararZZcuW9VqsJO3JFjogrgRO66ZPAz4z1H5qdzbTS4HvDu2KkiRNQG8PDEryF8DRwP5JNgK/B6wGLkvyFuAe4KRu8auB44H1wA+A0/uqS5I0mt4CoqpO2cGsYxrLFnBGX7VIkubOK6klSU0GhCSpyYCQJDX1dgxCi8fMqqsmXYKkKeQIQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJ01y1qOzolNwNq09Y4Eqkpc8RhCSpyYCQJDUZEJKkJo9BaCp5+w9p8hxBSJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqWkid3NNsgF4CHgU2FpVs0n2Ay4FZoANwElV9Z1J1CdJmuwI4her6siqmu3erwLWVNXhwJruvSRpQqbpeRAnAkd30xcCnwfeOalitLj4rGpp/k1qBFHA55KsS7KyazugqjZ3098EDphMaZIkmNwI4pVVtSnJc4Brkvz98MyqqiTVWrELlJUAhx56aP+VLiE+pU3SXExkBFFVm7qf9wNXAEcB9yU5EKD7ef8O1j2/qmaranbZsmULVbIk7XEWPCCSPC3JM7ZNA68CbgWuBE7rFjsN+MxC1yZJeswkdjEdAFyRZNvn/3lV/VWSrwCXJXkLcA9w0gRqWxLclSRpPix4QFTVXcALGu0PAMcsdD2SpDavpJYkNRkQkqQmA0KS1GRASJKapulWG9K88xYc0u5zBCFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLU5Gmu0hBPi5Ue4whCktRkQEiSmgwISVKTASFJavIgtbRIeUBdfXMEIUlqcgSxiPns6d3nv520awaE1BN3AWmxMyCmiF8okqaJASGNYGe7pKYtwPvefTZt/VV/DIhFwP3lewZHkJo2nsUkSWpyBCGNyRGelipHEJKkJgNCktSUqpp0Dbttdna21q5dO+ky5o27KrRU7ehAuwfmJyPJuqqa3dVyU3cMIslxwLnAE4E/q6rVEy5p3hkE0vwyaPoxVbuYkjwR+GPg1cAK4JQkKyZblSTtmaZqF1OSlwFnV9Wx3ft3AVTVH7SWn5ZdTI4IpMVlvnZ5LcT/+32MghbrLqblwL1D7zcCL5lQLZKWqLl+se+pfwROW0DsUpKVwMru7feS3Lmbm9of+Nb8VLVo2Oc9g31eQvLeHc4ap88/OcpC0xYQm4BDht4f3LX9WFWdD5w/7gclWTvKEGspsc97Bvu8Z1iIPk/VQWrgK8DhSQ5LsjdwMnDlhGuSpD3SVI0gqmprkrcCn2VwmusFVXXbhMuSpD3SVAUEQFVdDVy9AB819m6qRcg+7xns856h9z5P1WmukqTpMW3HICRJU2JJBkSS45LcmWR9klWN+fskubSb/+UkM0Pz3tW135nk2IWsexy72+ckv5xkXZKvdj9/aaFr313j/J67+Ycm+V6SdyxUzeMa87/t5ye5Lslt3e/7yQtZ++4a47/tJyW5sOvrHdsuvJ12I/T3F5LcmGRrktdvN++0JF/vXqeNXUxVLakXg4Pb3wB+CtgbuBlYsd0yvw38STd9MnBpN72iW34f4LBuO0+cdJ967vMLgYO66X8BbJp0f/ru89D8y4FPAO+YdH8W4Pe8F3AL8ILu/bP3gP+2fx24pJt+KrABmJl0n+ahvzPA84GLgNcPte8H3NX93Leb3necepbiCOIoYH1V3VVVjwCXACdut8yJwIXd9OXAMUnStV9SVQ9X1d3A+m570263+1xVf1dV/7drvw14SpJ9FqTq8YzzeybJa4G7GfR5sRinz68CbqmqmwGq6oGqenSB6h7HOH0u4GlJ9gKeAjwCPLgwZe+2Xfa3qjZU1S3AP2637rHANVX17ar6DnANcNw4xSzFgGjdrmP5jpapqq3Adxn8RTXKutNonD4P+9fAjVX1cE91zqfd7nOSpwPvBN6zAHXOp3F+z88DKslnu90T/3EB6p0P4/T5cuD7wGbg/wDvq6pv913wmMb5Dpr376+pO81Vk5HkCOC9DP7SXOrOBj5QVd/rBhR7gr2AVwI/D/wAWNPdsG3NZMvq1VHAo8BBDHa5/O8k11bVXZMta/FYiiOIXd6uY3iZbvj5z4AHRlx3Go3TZ5IcDFwBnFpV3+i92vkxTp9fAvxhkg3AmcBZ3QWa026cPm8EvlBV36qqHzC41uhFvVc8vnH6/OvAX1XVj6rqfuBvgWm/Hcc430Hz//016YMyPRzk2YvBwZnDeOwgzxHbLXMGjz+odVk3fQSPP0h9F4vjQN44fX5Wt/zrJt2PherzdsuczeI5SD3O73lf4EYGB2v3Aq4FTph0n3ru8zuBj3XTTwNuB54/6T6N29+hZT/OPz1IfXf3u963m95vrHom/Q/S0z/y8cDXGJwN8Ltd2+8Dr+mmn8zg7JX1wA3ATw2t+7vdencCr550X/ruM/BuBvtpbxp6PWfS/en79zy0jUUTEOP2GXgjg4PytwJ/OOm+9N1n4Old+21dOPzOpPsyT/39eQYjwu8zGCndNrTum7t/h/XA6ePW4pXUkqSmpXgMQpI0DwwISVKTASFJajIgJElNBoQkqcmA0FRL8r2h6eOTfC3JTyY5ey53YU3ym0k+3E3/2ySn9lHvpGTgfyV5ZpKZJLfuYLk/S7Kimz5rqH3vJF/oLjSTAANCi0SSY4APMrg25Z5xtlVVf1JVF81DTdP0ZXo8cHNV7fRmdFX1b6rq9u7tWUPtjwBrgDf0V6IWGwNCUy/JLwAfAX6l5nArkCSndyOOG4BXDLWfneQdSX62m7etfSbJV7vpFyf5m+4ZGZ9NcmDX/vkk/zXJWuBtSZ6b5PrumQP/ebsRz+8k+UqSW5K8Z+gz7kjyke65DJ9L8pRu3k8nuTbJzd0N9Z67o+00/AbwmaH3eyW5uPusy5M8daj+2SSrGdy596YkF3frfLrbjgQYEJp++zD44nptVf39qCt1X+jvYRAMr2TwrI/H6ba3d5LDuqY3AJcmeRLwIQa3MXgxcAFwztCqe1fVbFW9HzgXOLeqfo7B1a3bPv9VwOEMbhh3JPDiLujo2v+4qo4A/h+Du+gCXNy1vwB4ObB5F9sZ9gpg3dD7nwH+W1X9cwa3uP7t7fq+CviHqjqyqraFwq0MrtKVAANC0+9HwJeAt8xxvZcAn6+qLd3uk0t3sNxlPLZb5Q3dcj/D4OFJ1yS5icHtSA4eWmd4Wy9jcDsHgD8fan9V9/o7BvdA+lkGX/QAd1fVTd30OmAmyTOA5VV1BUBV/bAGN9Xb2XaG7VdVDw29v7eq/rab/h8MQnKnavB8iEe6WiRv962p94/ASQxuT31WVf2Xed7+pcAnknwKqKr6epKfY3B/m5ftYJ3vj7DdAH9QVX/6uMbB4zCHn7fxKIOH2cxpOw1bkzyhqrY9RGb7e+iMek+dfYAfjrisljhHEJp63V/SJwC/kWTUkcSXgX+Z5NndLqNf28G2v8HgS/o/8djI4E5gWZKXwY+fbXzEDj7neh7bRXTyUPtngTd3DyciyfIkz9lRsd1f/xu7J91te87yU+ewnTsZPKZym0O31c/gttdfbKzzo+7fhm7bzwa+VVU/2lGd2rMYEFoUavAksOOAdyd5Tdf87iQbt722W34zgzu1XsfgOQB37GTzlzK40+ll3bqPAK8H3pvkZgZ3uH35DtY9E/gPSW4BfprB08yoqs8x2OV0XXfg+3JgV7tu3gT8+25bXwJ+Yg7buQo4euj9ncAZSe5gcOvn8xrrnA/cMnSQ+he77UgA3s1VGkf3V/4/VFUlORk4paq2f2byQtRxIHBRVf3yGNv4FLCqqr42f5VpMfMYhDSeFwMfThIGZyS9eRJFVNXm7tTZZ+7qWoiWJHsDnzYcNMwRhCSpyWMQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU3/H95S0t3V7HAjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "min_ls = []\n",
    "for kl in kl_ls:\n",
    "    if kl <= 0.1:\n",
    "        min_ls.append(kl)\n",
    "print(len(min_ls), \"tokens have less than 0.1 KL\")\n",
    "f = plt.figure()\n",
    "x = np.array(min_ls)\n",
    "plt.hist(x, bins=50, range=[0.0, 0.1])\n",
    "plt.ylabel('count');\n",
    "plt.xlabel('KL divergence (bit)')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "f.savefig(\"huffman_KL_0.1.pdf\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}