Generating fake data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "FULLY = 0.25\n",
    "PARTIALLY = 0.73"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "N = 100  # items\n",
    "D = 2071  # users\n",
    "q = 10  # latent dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Real data\n",
    "from mangaki.algo.dataset import Dataset\n",
    "rating_values = {'dislike': 1, 'wontsee': 2, 'neutral': 3, 'willsee': 4, 'like': 5, 'favorite': 6}\n",
    "#from mangaki.utils.values import rating_values\n",
    "\n",
    "dataset = Dataset()\n",
    "dataset.load_csv('ratings-ordered.csv', convert=lambda choice: rating_values[choice])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import coo_matrix\n",
    "\n",
    "rows = dataset.anonymized.X[:, 0]\n",
    "cols = dataset.anonymized.X[:, 1]\n",
    "data = dataset.anonymized.y\n",
    "mangaki = coo_matrix((data, (rows, cols)), shape=(dataset.anonymized.nb_users, dataset.anonymized.nb_works)).tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.,  5.,  5., ...,  2.,  4.,  2.])"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mangaki.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import ShuffleSplit\n",
    "import random\n",
    "\n",
    "splitter = ShuffleSplit(n_splits=1, train_size=FULLY)\n",
    "X_fulltrain = []\n",
    "y_fulltrain = []\n",
    "X_parttrain = []\n",
    "y_parttrain = []\n",
    "X_test = []\n",
    "y_test = []\n",
    "for i_full_train, i_train_test in splitter.split(range(D)):\n",
    "    i_full_train = set(i_full_train)\n",
    "    i_train_test = set(i_train_test)\n",
    "    for (user_id, work_id), rating in zip(dataset.anonymized.X, dataset.anonymized.y):\n",
    "        if user_id in i_full_train:\n",
    "            X_fulltrain.append((user_id, work_id))\n",
    "            y_fulltrain.append(rating)\n",
    "        elif random.random() < PARTIALLY:\n",
    "            X_parttrain.append((user_id, work_id))\n",
    "            y_parttrain.append(rating)\n",
    "        else:\n",
    "            X_test.append((user_id, work_id))\n",
    "            y_test.append(rating)\n",
    "X_fulltrain = np.array(X_fulltrain)\n",
    "y_fulltrain = np.array(y_fulltrain)\n",
    "X_parttrain = np.array(X_parttrain)\n",
    "y_parttrain = np.array(y_parttrain)\n",
    "X_train = np.vstack((X_fulltrain, X_parttrain))\n",
    "y_train = np.concatenate((y_fulltrain, y_parttrain))\n",
    "X_test = dataset.anonymized.X[i_test]\n",
    "y_test = dataset.anonymized.y[i_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_all_errors(model, X_train, y_train, X_test, y_test):\n",
    "    y_train_pred = model.predict(X_train)\n",
    "    print('Train RMSE=', model.compute_rmse(y_train, y_train_pred))\n",
    "    y_test_pred = model.predict(X_test)\n",
    "    print('Test RMSE=', model.compute_rmse(y_test, y_test_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When FULLY = 50%, PARTIALLY = 60% (total: 80%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n",
      "Chrono: fill and center matrix [0q, 874ms]\n",
      "Shapes (2071, 20) (20, 10086)\n",
      "Chrono: factor matrix [0q, 6387ms]\n",
      "Train RMSE= 0.895676955371\n",
      "Test RMSE= 0.961083202139\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.als import MangakiALS\n",
    "\n",
    "als = MangakiALS()\n",
    "als.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "als.fit(X_train, y_train)\n",
    "compute_all_errors(als, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jilljenn/code/mangaki/mangaki/mangaki/algo/svd.py:36: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  means[i] = np.sum(matrix[i]) / np.sum(matrix[i] != 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chrono: fill and center matrix [0q, 633ms]\n",
      "Shapes (2071, 20) (20,) (20, 10086)\n",
      "Chrono: factor matrix [0q, 693ms]\n",
      "Train RMSE= 1.08928598343\n",
      "Test RMSE= 1.11938618551\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.svd import MangakiSVD\n",
    "\n",
    "svd = MangakiSVD()\n",
    "svd.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "svd.fit(X_train, y_train)\n",
    "compute_all_errors(svd, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE= 2.03534214322\n",
      "Test RMSE= 2.06726144326\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.knn import MangakiKNN\n",
    "\n",
    "knn = MangakiKNN()\n",
    "knn.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "knn.fit(X_train, y_train)\n",
    "compute_all_errors(knn, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When FULLY = 25%, PARTIALLY = 73% (total: 80%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n",
      "Chrono: fill and center matrix [0q, 853ms]\n",
      "Shapes (2071, 20) (20, 10086)\n",
      "Chrono: factor matrix [0q, 6462ms]\n",
      "Train RMSE= 0.896739922264\n",
      "Test RMSE= 0.958452733722\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.als import MangakiALS\n",
    "\n",
    "als = MangakiALS()\n",
    "als.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "als.fit(X_train, y_train)\n",
    "compute_all_errors(als, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jilljenn/code/mangaki/mangaki/mangaki/algo/svd.py:36: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  means[i] = np.sum(matrix[i]) / np.sum(matrix[i] != 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chrono: fill and center matrix [0q, 586ms]\n",
      "Shapes (2071, 20) (20,) (20, 10086)\n",
      "Chrono: factor matrix [0q, 650ms]\n",
      "Train RMSE= 1.09278685532\n",
      "Test RMSE= 1.11839989583\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.svd import MangakiSVD\n",
    "\n",
    "svd = MangakiSVD()\n",
    "svd.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "svd.fit(X_train, y_train)\n",
    "compute_all_errors(svd, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE= 2.04675025879\n",
      "Test RMSE= 2.07535256316\n"
     ]
    }
   ],
   "source": [
    "from mangaki.algo.knn import MangakiKNN\n",
    "\n",
    "knn = MangakiKNN()\n",
    "knn.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "knn.fit(X_train, y_train)\n",
    "compute_all_errors(knn, X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.138281790056\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11d159320>]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXZyYBcSMCEVkFFHcviAFErTuKouJGf9ZW\nsaK4cV1bpdpe216vYq1al4qiqKitS60LrogKKiJLWBWVRUQhIiBo2AxkZr6/P86hTTGQSTLJd2bO\n+/l45DFnzpyZeR/HvDk5c873mHMOERGJhpjvACIi0nhU+iIiEaLSFxGJEJW+iEiEqPRFRCJEpS8i\nEiEqfRGRCFHpi4hEiEpfRCRCCnwHAGjVqpXr1KmT7xgiIjll+vTp3zrnimvznKwo/U6dOlFaWuo7\nhohITjGzL2v7HO3eERGJEJW+iEiEqPRFRCJEpS8iEiEqfRGRCFHpi4hEiEpfRCRCVPoiIhGi0hcR\niZCsOCNXJFt1GvZqvZ6/eHj/DCURyQxt6YuIRIhKX0QkQlT6IiIRotIXEYkQlb6ISISo9EVEIkSl\nLyISISp9EZEIUemLiESISl9EJEJU+iIiEaLSFxGJkLRK38wWm9lHZjbLzErDeS3MbJyZLQhvdwnn\nm5ndY2YLzWyOmfVoyBUQEZH01WZL/2jnXHfnXEl4fxjwtnOuK/B2eB/gRKBr+DMEGJGpsCIiUj/1\n2b0zABgdTo8GTqsy/3EXmAwUmVmberyPiIhkSLql74A3zWy6mQ0J57V2zi0Lp78BWofT7YAlVZ67\nNJwnIiKepXsRlcOdc2Vmtiswzsw+q/qgc86ZmavNG4f/eAwB6NixY22eKiIidZTWlr5zriy8XQG8\nAPQClm/ebRPerggXLwM6VHl6+3Delq850jlX4pwrKS4urvsaiIhI2mosfTPbwcx22jwNHA98DIwB\nBoWLDQJeCqfHAOeFR/EcApRX2Q0kIiIepbN7pzXwgpltXv7vzrk3zGwa8KyZDQa+BH4aLv8acBKw\nENgA/DLjqUVEpE5qLH3n3CKgWzXzVwHHVjPfAZdnJJ2IiGSUzsgVEYkQlb6ISISo9EVEIkSlLyIS\nISp9EZEIUemLiESISl9EJEJU+iIiEaLSFxGJEJW+iEiEqPRFRCJEpS8iEiEqfRGRCFHpi4hEiEpf\nRCRCVPoiIhGi0hcRiRCVvohIhKj0RUQiRKUvIhIhKn0RkQhR6YuIRIhKX0QkQlT6IiIRotIXEYkQ\nlb6ISISo9EVEIkSlLyISISp9EZEISbv0zSxuZjPN7JXwfmczm2JmC83sGTNrEs5vGt5fGD7eqWGi\ni4hIbdVmS/9K4NMq928D7nLO7Ql8BwwO5w8Gvgvn3xUuJyIiWSCt0jez9kB/4OHwvgHHAM+Fi4wG\nTgunB4T3CR8/NlxeREQ8S3dL/y/AdUAqvN8S+N45lwjvLwXahdPtgCUA4ePl4fIiIuJZjaVvZicD\nK5xz0zP5xmY2xMxKzax05cqVmXxpERHZinS29A8DTjWzxcDTBLt17gaKzKwgXKY9UBZOlwEdAMLH\nmwOrtnxR59xI51yJc66kuLi4XishIiLpqbH0nXO/cc61d851As4G3nHO/RwYD5wVLjYIeCmcHhPe\nJ3z8Heecy2hqERGpk/ocp389cI2ZLSTYZz8qnD8KaBnOvwYYVr+IIiKSKQU1L/JvzrkJwIRwehHQ\nq5plKoCBGcgmIiIZpjNyRUQiRKUvIhIhKn0RkQhR6YuIRIhKX0QkQlT6IiIRotIXEYmQWh2nLyLV\n241V7Bf7kg62kkISLHXFLHRtwTnQILOSRVT6InXm6Bebxjnxtzk89jExq2a0kQcehx7nBT+FzRo/\nosgWVPoiddDJlnFrwSj6xD9hqWvF3YkzeD91IF+51myigA62koNiC7g5NhNevw6mPACn3A2dj/Ad\nXSJOpS9SSwNiE7mt8CE2UchvKgfzdPJo3BZfj811OzA32YmbL/4LLJoAL18Fo0+BQ/8bjvsjxPR1\nmvih0hdJm+PS+MtcX/g0k1P78t+bhrKSXWp+Wpej4NJJ8OaNMOleKC+D0x+AgqYNHVjkR1T6Imm6\nruAZLisYw5hkH35VeQmbKEz/yU22h/53QtHu8NZN8MN3cM6zUNCk4QKLVEN/Y4qk4fz4G1xWMIa/\nJ47hysrLa1f4m5nB4VfBqffBovHw4qWQStX8PJEM0pa+SA1Oik3mfwqeYGyyhN8mLvjR/vta63Eu\nrF8Jb/8Bdm4Dx9+cmaAiaVDpi2zDnraUPxc+yAzXlSsqh5LK1B/Hh18Na74O9vG37QEHnJGZ1xWp\ngXbviGzNpg38tfAeNtCUyzZdyUYyuP/dDPrdCh16w5grYNXnmXttkW1Q6Ytszeu/pquVcXXlZaxI\n5yid2ooXwpmjIBaHf5wPiY2Zfw+RLaj0Raoz7w2Y+SQjkqfwfuq/Gu59ijoEh29+Mwfeu73h3kck\npNIX2VLFGnjlaijel7sTZzb8++19InT7GUy8C775qOHfTyJNpS+ypbdugrXLYMB9dTs0sy5OuAWa\n7QIvDYVkonHeUyJJpS9S1VdToPQROOQyaF/SeO+7fQs46XZYNgumPth47yuRo9IX2SyVCgZH26kN\nHH1D47//fqfBnsfBhNtg3crGf3+JBJW+yGaz/x5saff9IzTdsfHf3wxOuBUq18N4nbAlDUOlLwLB\nl7dv/QHa94IDB/rLUbwX9LoYpo+GZXP85ZC8pdIXAfjgbli/Ak4c7v9KV0deF+zjf/NGvzkkL6n0\nRdYuh8n3wwFnQruDfaeBZkVwxHXwxXvw+XjfaSTPqPRF3v9zcDbs0Vm0ZV3yS2jeMRiUzVVzGUaR\nOlLpS7R9txhKHw1Gvmy5h+80/1bQFI4aBl/PhE/H+E4jeaTG0jez7cxsqpnNNrO5ZvaHcH5nM5ti\nZgvN7BkzaxLObxreXxg+3qlhV0GkHibcFox9c+T1vpP8WLezodXe8M7/QSrpO43kiXS29DcCxzjn\nugHdgX5mdghwG3CXc25P4DtgcLj8YOC7cP5d4XIi2Wf1IpjzDJRcADu39Z3mx2LxYGv/23nwyUu+\n00ieqLH0XWBdeLcw/HHAMcBz4fzRwGnh9IDwPuHjx5r5PhxCpBrv3wmxAjjsSt9Jtm6/AcHW/nu3\n6ypbkhFp7dM3s7iZzQJWAOOAz4HvnXObBwlZCrQLp9sBSwDCx8uBlpkMLVJv338Fs5+CgwfBTrv5\nTrN1sTgc8StY8QnMe9V3GskDaV05yzmXBLqbWRHwArBPfd/YzIYAQwA6duxY35cTqZ2JdwHW4Fv5\nnYbVvagXD+8fTOx/BkwYDu/eBvuc7P88AslptTp6xzn3PTAe6AMUmdnmfzTaA2XhdBnQASB8vDmw\nqprXGumcK3HOlRQXF9cxvkgdrF0OM5+Eg34Ozdv7TlOzeAH85Npg2OWFb/lOIzkunaN3isMtfMys\nGdAX+JSg/M8KFxsEbP6maUx4n/Dxd5zTgcaSRaaMgGQlHHqF7yTpO3Ag7NwuOHNYpB7S2dJvA4w3\nsznANGCcc+4V4HrgGjNbSLDPflS4/CigZTj/GmBY5mOL1FHFGpj2COx3anYdl1+TgibBcM+L34el\n032nkRxW4z5959wc4KBq5i8CelUzvwLwOGKVyDZMfww2lmf3ETtbc/AgeO9P8MFf4P894TuN5Cid\nkSvRkdgEk0dAp59kxxg7tdV0J+h5IXz6Mny70HcayVFpHb0jkss2H0Fzeux97mryNeev+gUT6nFU\njVe9L4FJ9wYDxJ18p+80koO0pS8R4Rhc8DoLUu2YkOrmO0zd7bgrHPjT4ByDDat9p5EcpNKXSOht\nn3FAbDGPJPsBOX6ce5/LoHJD8P2ESC2p9CUSBhe8xmq3I88nf+I7Sv213h86HwlTHwoOPRWpBZW+\n5L2OtpzjYjN4MnkcG2niO05m9BkKa7+GuS/6TiI5RqUveW9Q/E2SxHgi0dd3lMzZ8zhouSdMecB3\nEskxKn3JbxvXMTA+gVdTvVnJLr7TZE4sFlxAvaxUJ2tJraj0Jb/Nfoqd7QdGJ07wnSTzuv8MmuwE\nUx/0nURyiEpf8pdzMHUks1JdmOn29J0m85ruBN3PgY+fDwaRE0mDSl/y16Lx8O38cCs/xw/T3Jpe\nQyBVqcM3JW0qfclfU0bC9q14NXWI7yQNp9WewZe6pY/o8E1Ji0pf8tN3X8L8N+DgQWyi0HeahtXz\nIlj3DXz2iu8kkgNU+pKfSh8JrjB18C99J2l4XftCUUeY+rDvJJIDVPqSfyorYMbjsPdJUNTBd5qG\nF4tDyWD4ciIs/8R3GslyKn3JP3NfgB9WB8MQR8VB50K8KUzT1r5sm4ZWlvwz7WFo2RW6HOU7Sb3U\n9qLqfy7sTb9pf+OQiX34eLiuYyTV05a+5JevZwVnqfa8MNinHyGPJ/qyo1VwWvwD31Eki6n0Jb+U\njoLC7aHb2b6TNLo5bg/mpDpzbnxccGKaSDVU+pI/fvge5vwDDjwLmhX5TuPFE8m+7B1bCl996DuK\nZCmVvuSP2U9D4ofgSJaIejnZh3K3vb7Qla1S6Ut+cC7YtdOuBNp2953Gmwqa8lzySPhkDKxb4TuO\nZCGVvuSHxe/Dt/OhZ3S38jf7W/LYYDyeGY/7jiJZSKUv+WHaKNiuCPY/3XcS7xa5ttD5iGAQtlTS\ndxzJMip9yX1rw3FnDvoFFDbznSY7lAyG8iWwYJzvJJJlVPqS+2Y8AakElFzgO0n22Kc/7Lhb8D2H\nSBUqfcltyUSwG6PL0dByD99pske8EHqcF2zpf7fYdxrJIip9yW0LxsKapfoCtzoHnx+clVz6qO8k\nkkVU+pLbpo2CndrCXif6TpJ9mrcLRhqd+QQkNvpOI1mixtI3sw5mNt7MPjGzuWZ2ZTi/hZmNM7MF\n4e0u4Xwzs3vMbKGZzTGzHg29EhJRqz6Hz98OtmjjGjuwWiUXwIZVwXH7IqS3pZ8ArnXO7QccAlxu\nZvsBw4C3nXNdgbfD+wAnAl3DnyHAiIynFgGY/ihYPNh3LdXrcjS06KIzdOVfaix959wy59yMcHot\n8CnQDhgAjA4XGw2cFk4PAB53gclAkZm1yXhyibbKH2Dmk8FRKjvrf6+tisWCrf0lk+Gbj32nkSxQ\nq336ZtYJOAiYArR2zi0LH/oGaB1OtwOWVHna0nCeSObMfRF++C5aF0qpq+4/h4LttLUvQC1K38x2\nBP4JXOWcW1P1MeecA2o1lquZDTGzUjMrXblyZW2eKgLTHoJWewVnnsq2bd8CDjgT5jwLFeW+04hn\naZW+mRUSFP7fnHPPh7OXb95tE95uHt2pDKh6YdL24bz/4Jwb6Zwrcc6VFBcX1zW/RFHZDCibHskL\npdRZz8FQuR5mP+M7iXhW4yEPZmbAKOBT59ydVR4aAwwChoe3L1WZP9TMngZ6A+VVdgOJ1EnVSwf+\nqeBB+sebcsgLLVj7Qu0uKRhZ7Q6Gtj2CXTy9LtI/lhGWzpb+YcC5wDFmNiv8OYmg7Pua2QLguPA+\nwGvAImAh8BBwWeZjS1QVsZZT45N4MXk4a9ned5zc0usi+HYefPGe7yTiUY1b+s65icDWNguOrWZ5\nB1xez1wi1RoYf5ftrJLHk319R8k9+58BY28Mvg/pcqTvNOKJzsiVnBEjxbnxcUxJ7cM819F3nNxT\nuB30OBc+ew3Kl/pOI56o9CVnHB2bScfYSkYnjvcdJXf1vBBwwfAVEkkqfckZg+Jvssy14M1Uie8o\nuauoYzAez4zRUFnhO414oNKXnLCHlXFE/COeTBxHouavomRbel0UjMcz9/mal5W8o9KXnHBufBwb\nXQFPJ4/2HSX3dT4SiveBKQ8GF5SXSFHpS/arKOes+Hu8kjqEVTT3nSb3mQVb+8tmwZKpvtNII1Pp\nS/ab+SQ7WgWPJvr5TpI/uv0MtmsOUzQIbtSo9CW7pZIw5QGmpPbhY9fFd5r80WQH6DEoGGf/+yU1\nLy95Q6Uv2W3ea/D9VzyirfzM6zUkuJ32kN8c0qhU+pLdJj8AzTsyTodpZl5RB9j3lODC8pvW+04j\njUTHvkn2+noWfDkR+v4vqZe1fVIbVQeo25Ye1p3nm77I7/5wA08kg5PeFg/v35DRxDP9Jkn2+vA+\naLITHDzId5K8NcN1ZWZqTwbHXydGynccaQQqfclO3y+Bj58PCn87HabZcIyHEifRKbacvrHpvsNI\nI1DpS3aa8kBw2/sSvzkiYGyqJ1+lirmoQNcmiAKVvmSfinKYPhr2Pz34slEaVJI4jyRPpCQ2nx42\n33ccaWAqfck+0x+DTWvh0KG+k0TGs8mjKHfbM0Rb+3lPpS/ZpbICPvwrdDkK2h7kO01kbGA7RieP\n5/hYKazU1n4+U+lLdpnzNKxbDodf7TtJ5DyW6MdGCmHS3b6jSANS6Uv2SCXhg7uDLfzOupxfY1vN\nzjyTPApmPwPlZb7jSANR6Uv2+OQlWL0o2Mq3rV2WWRrSw8n+4FLBLjbJSyp9yQ6pFLx/B7TcE/Y5\n2XeayFrqiuHAgTD9UVj/re840gBU+pId5r8Oyz+GI34NsbjvNNH2k2uh8ofgjGjJOyp98c85ePc2\n2KUzHHCW7zRSvBcccAZMfQg2rPadRjJMpS/+LXgTls0OtjDjGgMwKxzxa9i0Dibf7zuJZJhKX/xy\nDiYMh+YdodvZvtPIZrvuC/sNCIa21tZ+XlHpi1/zXoevZ8CRv4Z4oe80UtWRw4Kt/Un3+E4iGaTS\nF39SKRj/f9CiC3Q7x3ca2VLr/eCAM2HKg7B2ue80kiEqffHnkxeCI3aOukH78rPV0TdAYiNMvMt3\nEskQlb74kUzA+Ftg13BrUrJTyz2g+zlQOkoXUM8TNZa+mT1iZivM7OMq81qY2TgzWxDe7hLONzO7\nx8wWmtkcM+vRkOElh80YDasWwjG/hZi2PbLakdcDFvwjLTkvnd+2x4B+W8wbBrztnOsKvB3eBzgR\n6Br+DAFGZCam5JWNa2HCrdCxD+x9ku80UpOiDtD7Ypj9FHzzke80Uk81lr5z7j1gy2O2BgCjw+nR\nwGlV5j/uApOBIjNrk6mwkicm3QfrV8LxN2uMnVzxk2uhWRGM+x/fSaSe6vp3dWvn3LJw+hugdTjd\nDqi6429pOE8ksPYbmHQv7HcatC/xnUbS1awoOGHr83dg4Vu+00g91HtnqnPOAa62zzOzIWZWamal\nK1eurG8MyRVv/R5SlXDcTb6TSG31vDAYKuONGyBZ6TuN1FFdS3/55t024e2KcH4ZUPWipu3DeT/i\nnBvpnCtxzpUUFxfXMYbklCVTg/3CfYYGx+ZLbiloCv1uhW/nBePySE6qa+mPAQaF04OAl6rMPy88\niucQoLzKbiCJslQKXr8OdmoT7B+W3LRXP9jzuGDojHX6Cz0XpXPI5lPAh8DeZrbUzAYDw4G+ZrYA\nOC68D/AasAhYCDwEXNYgqSX3zHwCvp4Jff8Xmu7oO43UlRn0Gw6V6+Ht3/tOI3VQ42mQzrmfbeWh\nY6tZ1gGX1zeU5Jl1K4OjPnY/DA7U0MnZrtOwV2tc5vqCE7l05pP8dHJnprp9/+OxxcP7N1Q0yQCd\nFSMNb+wNsGk9nPwXHaKZJ+5OnMGSVDG3FI6iCfpSN5dowBNpWJ+/Ax89C0deT6c7FgALfCeSDKig\nKb9L/JLHmvyJS+Ivc0/yDN+RJE3a0peGU7EGxlwZXPf28Gt8p5EMm5DqzphkH4YWvMDe9pXvOJIm\nlb40nDd/C2uWwoD7oXA732mkAfy+chBr2IE7Ch+ggITvOJIGlb40jAVvBYOq9RkKHXv7TiMNZDU7\nc2PlYA6ILWZowYu+40gaVPqSeetXwZihULwPHH2j7zTSwMamevJC8jAuj79Ed1voO47UQKUvmeUc\nvHQZbFgFZ4zUbp2IuKnyfJazC/cU3gsV5b7jyDao9CWzpjwA898ITsJq0813Gmkka9iBKzYNpa2t\ngpevCv7xl6yk0pfMWVoanIS114nB+OsSKTPcXtyZGAhznw+utCVZSaUvmbF2OTzzi2BsndPu10lY\nETUieQp0PR5eHwZfTfYdR6qh0pf6S2yCfwyCH76Hs/8G27fwnUg8ccTgjIeCq209ex6s+dp3JNmC\nSl/qxzl45Wr46kMYcB/sdqDvROJbsyI4+++wcR08dXZwK1lDpS/18+6fYNaTcOQwDaYm/7brvjDw\n0eCaus9dAEmduJUtNPaO1ErVERgHxidwe+FI/pE4gl+PPRDG1jw6o0TIXidA/zuCvwRfvQZOuVvf\n9WQBlb7UyamxSdxW8BDvJv+L3yQuBPTLLNUouQDKl8L7d0CTHeCEW1T8nqn0pdZOiE3lzsL7mZLa\nl4srryah/41kW475XTC09uT7g0suHnuTit8j/bZKrQyITeSOwgeY7fbgwsprqaCp70iSZaq/CMtP\nuKVgPudMvItREz7j5sTPgyN9tqALsDQ8lb6kb+pD3FU4gsmpfbmo8lrW08x3IskZxo2JC9hIIYML\nXmcnNvCbxIUkifsOFjkqfalZKglv3QST7uXtVA+GVl7BRpr4TiU5xhHjD4nzKGcHrip4ntb2HUMr\nr2At2/uOFik6ZFO2rWJNcKbtpHuh1xAuqbxahS/1YPwlcRbDKi/k0Nhc/tnkJjract+hIkWlL1u3\nbA6MPBLmj4UTb4eTbtef45IRTyeP4bzKYexq3/NKkxs4MTbFd6TIUOnLj6VSMHkEPHwcVFbA+a9C\n7yG+U0me+TC1PydvuoVFri0jmtzNzQWjdPZuI1Dpy39a9Tk81h/eGAZdjoJL3ofd+/hOJXlqqStm\n4KabGJnozznxd2DEofDF+75j5TV9kSuBTRtg4p3wwT3BsdQD7ofu5+h4amlwlRRwS+LnjEsezO2r\nH6TT6JN5IXkYt1aewwp2Sft1dLhnelT6UZeshFl/C8bQWVMGB/4U+v4Rdm7jO5lEzDS3D/02DefS\ngjFcEn+F45uW8nDyJB5O9NcRPhmk0o+qyh9g9lPBlv13X0D7nnDmw7D7ob6TSYRV0JS7EgP5Z/II\nrit4misLXmBQ/E1GJ09gdOJ4VrOz74g5T6UfNd8thhmPBz/rV0Lbg6DfM8HgWNqVI1niK9eaoZVX\nMiKxmKsK/smVBc9zcfxlXkwexlPJY5jt9kDjPdWNSj8KNqyGT8fAR8+R+mIiDngndRCjkhczedG+\nsCgJvOY7pciPzHWduKjyWvZIlDE4/hqnxSdxdsEEPkt1YEzyUF5OHcIS19p3zJxiLgsuYFxSUuJK\nS0t9x8gfqSQsmw1fvAvz34Qlk8GloMUe3LW8O88mj2IZLX2nFKm1HdnAgPgkTo9PpCQ2H4DPUh0Y\nn+rOpRdcCB16Q2F0hgcxs+nOuZJaPachSt/M+gF3A3HgYefc8G0tr9KvB+dg7bLgRKpls2DJ1OAC\n5RvLg8d3OxD26gf7nAxtutHpN9qil/zQ3lZyQmwqx8Zm0jM2j0JLstEV8pHrzIxUVz5IHcC7qW5p\nvVauHvlTl9LP+O4dM4sDfwX6AkuBaWY2xjn3SabfKzIqK2Dd8uDomvIy/vzMWHa35XSJLWNPK6O5\nbQAg5Yx5rj0zUz2YnNqfD1P7sXJxESwG3iwDynyuhUhGLXXFjEr2Z1SyPzvwAz1jn3Fo7BMOjs1n\nUHwsHWxF2qUfJQ2xT78XsNA5twjAzJ4GBgDRLv1lc6ByAyQ2QqKCSx/7gGZsZHvbyPZUsKP9wI5U\nsLNtoDnrKbK1tGAtLW0NRbb+P17qV4Ww3BXxhWvDy8k+zHftmZvqxGeuo0a+lEhaTzMmpA5iQuog\nAJpQyc5saJT3rn4o6fQ19l8ZDVH67YAlVe4vBXo3wPsA9f8P3limNb2EYlvzr/sjthizLOWMdWzH\nGnZgjduBcrcDn9KR1amdWeGKWEERy1xLvnYtKXOtNI69yDZsopBvaZ728rnSI5ng7egdMxsCbB7Q\nZZ2ZzQunWwHf+knVcHYNbmpYt/JGydJA8vJzC2ndclNOrJvdVqenbV633Wv7xIYo/TKgQ5X77alm\nZ7JzbiQwcsv5ZlZa2y8mcoXWLTdp3XKT1q16DTHg2jSgq5l1NrMmwNnAmAZ4HxERqaWMb+k75xJm\nNhQYS3DI5iPOubmZfh8REam9Btmn75x7jbqf4vmjXT55ROuWm7RuuUnrVo2sOCNXREQahy6iIiIS\nId5L38wGmtlcM0uZ2Va/jTazxWb2kZnNMrOcGLOhFuvWz8zmmdlCMxvWmBnrysxamNk4M1sQ3lZ7\ntQszS4af2Swzy+ov9Gv6HMysqZk9Ez4+xcw6NX7Kuklj3c43s5VVPqsLfeSsLTN7xMxWmNnHW3nc\nzOyecL3nmFmPxs5YV2ms21FmVl7lM/uftF7YOef1B9gX2BuYAJRsY7nFQCvfeTO9bgRfdn8OdAGa\nALOB/XxnT2Pd/gQMC6eHAbdtZbl1vrOmuT41fg7AZcAD4fTZwDO+c2dw3c4H7vOdtQ7rdgTQA/h4\nK4+fBLxOMA7zIcAU35kzuG5HAa/U9nW9b+k75z51zs2recnck+a6/WvYCufcJmDzsBXZbgAwOpwe\nDZzmMUsmpPM5VF3n54BjzXLiIgS5+v9YjZxz7wGrt7HIAOBxF5gMFJlZTlwWLo11qxPvpV8LDnjT\nzKaHZ/Pmi+qGrWjnKUtttHbOLQunvwG2Nqj5dmZWamaTzSyb/2FI53P41zLOuQTBKdS5MEZ1uv+P\nnRnuAnnOzDpU83guytXfr3T1MbPZZva6me2fzhMaZRgGM3sL2K2ah250zr2U5ssc7pwrM7NdgXFm\n9ln4L6FXGVq3rLStdat6xznnzGxrh4HtHn5uXYB3zOwj59znmc4q9fYy8JRzbqOZXUzwF80xnjPJ\nts0g+P1aZ2YnAS8CXWt6UqOUvnPuuAy8Rll4u8LMXiD4k9V76Wdg3dIatsKHba2bmS03szbOuWXh\nn8srtvIamz+3RWY2ATiIYP9ytknnc9i8zFIzKwCaA6saJ1691Lhuzrmq6/EwwXc2+SBrf7/qyzm3\npsr0a2Z2v5m1cs5tc7yhnNi9Y2Y7mNlOm6eB44Fqv9HOQbk6bMUYYFA4PQj40V81ZraLmTUNp1sB\nh5G9Q2x4mxNVAAABDUlEQVSn8zlUXeezgHdc+I1alqtx3bbYz30q8Gkj5mtIY4DzwqN4DgHKq+yW\nzGlmttvm75TMrBdBn9e8EZIF31CfTrCfbSOwHBgbzm8LvBZOdyE44mA2MJdg14n37JlYt/D+ScB8\ngi3gXFm3lsDbwALgLaBFOL+E4GppAIcCH4Wf20fAYN+5a1inH30OwB+BU8Pp7YB/AAuBqUAX35kz\nuG63hr9bs4HxwD6+M6e5Xk8By4DK8HdtMHAJcEn4uBFc1Onz8P/BrR4hmG0/aazb0Cqf2WTg0HRe\nV2fkiohESE7s3hERkcxQ6YuIRIhKX0QkQlT6IiIRotIXEYkQlb6ISISo9EVEIkSlLyISIf8fGc4g\nKy0T5xMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11d155550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.mlab as mlab\n",
    "import numpy as np\n",
    "\n",
    "values = als.U[:, 19]\n",
    "nb_samples = len(values)\n",
    "mu = values.mean()\n",
    "sigma = values.std()\n",
    "\n",
    "nb_boxes = 20\n",
    "size = (values.max() - values.min()) / nb_boxes\n",
    "print(size)\n",
    "x = np.linspace(-1, 1, 100)\n",
    "plt.hist(als.U[:, 19], nb_boxes)\n",
    "plt.plot(x, nb_samples * size * mlab.normpdf(x, mu, sigma))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "mu_pop = als.U.mean(axis=0)\n",
    "sigma_pop = np.cov(als.U.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mu_bias = als.means.mean()\n",
    "sigma_bias = als.means.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "nb_ratings = Counter(X_fulltrain[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(255, 1606), (1571, 1552), (121, 1522), (1410, 1248), (1945, 1115)]"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_ratings.most_common(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "popularity = Counter(X_train[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1283), (1, 1188), (2, 1140), (4, 1124), (3, 1100)]"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "popularity.most_common(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fake_bias = np.random.normal(mu_bias, sigma_bias, size=len(i_full_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_faketrain = []\n",
    "y_faketrain = []\n",
    "fake_U = np.random.multivariate_normal(mu_pop, sigma_pop, size=len(i_full_train))\n",
    "for i, user_id in enumerate(i_full_train):\n",
    "    weights = np.array(list(popularity.values())) / sum(popularity.values())\n",
    "    work_ids = np.random.choice(list(popularity.keys()), size=nb_ratings[user_id], p=weights, replace=False)\n",
    "    X_faketrain.extend([(user_id, work_id) for work_id in work_ids])\n",
    "    y_faketrain.extend(als.VT.T[work_ids].dot(fake_U[i]) + fake_bias[i])\n",
    "X_faketrain = np.array(X_faketrain)\n",
    "y_faketrain = np.array(y_faketrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_fake = np.vstack((X_faketrain, X_parttrain))\n",
    "y_fake = np.concatenate((y_faketrain, y_parttrain))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When FULLY = 50%, PARTIALLY = 60% (total: 80%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n",
      "Chrono: fill and center matrix [0q, 887ms]\n",
      "Shapes (2071, 20) (20, 10086)\n",
      "Chrono: factor matrix [0q, 6118ms]\n",
      "Train RMSE= 0.587909831104\n",
      "Test RMSE= 1.4260386546\n"
     ]
    }
   ],
   "source": [
    "als = MangakiALS()\n",
    "als.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "als.fit(X_fake, y_fake)\n",
    "compute_all_errors(als, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jilljenn/code/mangaki/mangaki/mangaki/algo/svd.py:36: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  means[i] = np.sum(matrix[i]) / np.sum(matrix[i] != 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chrono: fill and center matrix [0q, 618ms]\n",
      "Shapes (2071, 20) (20,) (20, 10086)\n",
      "Chrono: factor matrix [0q, 699ms]\n",
      "Train RMSE= 0.722305878199\n",
      "Test RMSE= 1.46782574115\n"
     ]
    }
   ],
   "source": [
    "svd = MangakiSVD()\n",
    "svd.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "svd.fit(X_fake, y_fake)\n",
    "compute_all_errors(svd, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE= 2.10213686971\n",
      "Test RMSE= 2.15983322509\n"
     ]
    }
   ],
   "source": [
    "knn = MangakiKNN()\n",
    "knn.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "knn.fit(X_fake, y_fake)\n",
    "compute_all_errors(knn, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# When FULLY = 25%, PARTIALLY = 73% (total: 80%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n",
      "Chrono: fill and center matrix [0q, 928ms]\n",
      "Shapes (2071, 20) (20, 10086)\n",
      "Chrono: factor matrix [0q, 6916ms]\n",
      "Train RMSE= 0.751709581523\n",
      "Test RMSE= 1.18777848707\n"
     ]
    }
   ],
   "source": [
    "als = MangakiALS()\n",
    "als.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "als.fit(X_fake, y_fake)\n",
    "compute_all_errors(als, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing M: (2071 × 10086)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jilljenn/code/mangaki/mangaki/mangaki/algo/svd.py:36: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  means[i] = np.sum(matrix[i]) / np.sum(matrix[i] != 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chrono: fill and center matrix [0q, 631ms]\n",
      "Shapes (2071, 20) (20,) (20, 10086)\n",
      "Chrono: factor matrix [0q, 690ms]\n",
      "Train RMSE= 0.926449672331\n",
      "Test RMSE= 1.28065678659\n"
     ]
    }
   ],
   "source": [
    "svd = MangakiSVD()\n",
    "svd.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "svd.fit(X_fake, y_fake)\n",
    "compute_all_errors(svd, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train RMSE= 2.18239929253\n",
      "Test RMSE= 2.1875031604\n"
     ]
    }
   ],
   "source": [
    "knn = MangakiKNN()\n",
    "knn.set_parameters(dataset.anonymized.nb_users, dataset.anonymized.nb_works)\n",
    "knn.fit(X_fake, y_fake)\n",
    "compute_all_errors(knn, X_fake, y_fake, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Django Shell-Plus",
   "language": "python",
   "name": "django_extensions"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}