04. Recommender System.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nagaa\\Downloads\\Book-Recommendation-System-master (1)\\Book-Recommendation-System-master\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.chdir(\"C:\\\\Users\\\\nagaa\\\\Desktop\\\\ABHI\\\\COURSES\\\\256chang\\\\Book-Recommendation-System\")\n",
    "\n",
    "print(os.getcwd())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.metrics.pairwise import linear_kernel, cosine_similarity\n",
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Data loading and reading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The data used for this project consist of 5 different datasets, which are loaded and presented above: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "books = pd.read_csv('books.csv')\n",
    "ratings = pd.read_csv('ratings.csv')\n",
    "tags = pd.read_csv('tags.csv')\n",
    "bookTags = pd.read_csv('book_tags.csv')\n",
    "toRead = pd.read_csv('to_read.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**1) Books** contains all the information about the rated books, including author, title, book ID, average rating, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(books.id.unique())  # There is information for 10.000 different books in total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, let us look at the books with the highest average rating, let's sort the dataset by average rating.\n",
    "\n",
    "Apparently, the selected sample of users seem to like quite a lot Bill Watterson's 'Calvin and Hobbes' comic series."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>ratings_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3627</th>\n",
       "      <td>3628</td>\n",
       "      <td>24812</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Complete Calvin and Hobbes</td>\n",
       "      <td>4.82</td>\n",
       "      <td>28900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3274</th>\n",
       "      <td>3275</td>\n",
       "      <td>8</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter Boxed Set, Books 1-5 (Harry Potte...</td>\n",
       "      <td>4.77</td>\n",
       "      <td>33220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>862</td>\n",
       "      <td>17332218</td>\n",
       "      <td>Brandon Sanderson</td>\n",
       "      <td>Words of Radiance (The Stormlight Archive, #2)</td>\n",
       "      <td>4.77</td>\n",
       "      <td>73572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8853</th>\n",
       "      <td>8854</td>\n",
       "      <td>95602</td>\n",
       "      <td>Francine Rivers</td>\n",
       "      <td>Mark of the Lion Trilogy</td>\n",
       "      <td>4.76</td>\n",
       "      <td>9081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7946</th>\n",
       "      <td>7947</td>\n",
       "      <td>5031805</td>\n",
       "      <td>Anonymous, Lane T. Dennis, Wayne A. Grudem</td>\n",
       "      <td>ESV Study Bible</td>\n",
       "      <td>4.76</td>\n",
       "      <td>8953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4482</th>\n",
       "      <td>4483</td>\n",
       "      <td>24814</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>It's a Magical World: A Calvin and Hobbes Coll...</td>\n",
       "      <td>4.75</td>\n",
       "      <td>22351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6360</th>\n",
       "      <td>6361</td>\n",
       "      <td>70489</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>There's Treasure Everywhere: A Calvin and Hobb...</td>\n",
       "      <td>4.74</td>\n",
       "      <td>16766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>422</td>\n",
       "      <td>862041</td>\n",
       "      <td>J.K. Rowling</td>\n",
       "      <td>Harry Potter Boxset (Harry Potter, #1-7)</td>\n",
       "      <td>4.74</td>\n",
       "      <td>190050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3752</th>\n",
       "      <td>3753</td>\n",
       "      <td>10</td>\n",
       "      <td>J.K. Rowling</td>\n",
       "      <td>Harry Potter Collection (Harry Potter, #1-6)</td>\n",
       "      <td>4.73</td>\n",
       "      <td>24618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6919</th>\n",
       "      <td>6920</td>\n",
       "      <td>24815</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Indispensable Calvin and Hobbes</td>\n",
       "      <td>4.73</td>\n",
       "      <td>14597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6589</th>\n",
       "      <td>6590</td>\n",
       "      <td>59715</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Authoritative Calvin and Hobbes: A Calvin ...</td>\n",
       "      <td>4.73</td>\n",
       "      <td>16087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9565</th>\n",
       "      <td>9566</td>\n",
       "      <td>70487</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>Attack of the Deranged Mutant Killer Monster S...</td>\n",
       "      <td>4.72</td>\n",
       "      <td>9713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1307</th>\n",
       "      <td>1308</td>\n",
       "      <td>17927395</td>\n",
       "      <td>Sarah J. Maas</td>\n",
       "      <td>A Court of Mist and Fury (A Court of Thorns an...</td>\n",
       "      <td>4.72</td>\n",
       "      <td>108384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8977</th>\n",
       "      <td>8978</td>\n",
       "      <td>121792</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Revenge of the Baby-Sat</td>\n",
       "      <td>4.71</td>\n",
       "      <td>11503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9075</th>\n",
       "      <td>9076</td>\n",
       "      <td>2350129</td>\n",
       "      <td>The Church of Jesus Christ of Latter-day Saints</td>\n",
       "      <td>Preach My Gospel: A Guide To Missionary Service</td>\n",
       "      <td>4.71</td>\n",
       "      <td>9891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7253</th>\n",
       "      <td>7254</td>\n",
       "      <td>24816</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>Homicidal Psycho Jungle Cat: A Calvin and Hobb...</td>\n",
       "      <td>4.71</td>\n",
       "      <td>14113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5206</th>\n",
       "      <td>5207</td>\n",
       "      <td>24818</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Days Are Just Packed: A Calvin and Hobbes ...</td>\n",
       "      <td>4.68</td>\n",
       "      <td>19143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5918</th>\n",
       "      <td>5919</td>\n",
       "      <td>265205</td>\n",
       "      <td>Anonymous, Ronald A. Beers, Ronald A. Beers</td>\n",
       "      <td>Life Application Study Bible: NIV</td>\n",
       "      <td>4.67</td>\n",
       "      <td>13614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9140</th>\n",
       "      <td>9141</td>\n",
       "      <td>9329354</td>\n",
       "      <td>Brandon Sanderson</td>\n",
       "      <td>The Way of Kings, Part 1 (The Stormlight Archi...</td>\n",
       "      <td>4.67</td>\n",
       "      <td>9792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2100</th>\n",
       "      <td>2101</td>\n",
       "      <td>99298</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>The Harry Potter Collection 1-4 (Harry Potter,...</td>\n",
       "      <td>4.66</td>\n",
       "      <td>43929</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id   book_id                                          authors  \\\n",
       "3627  3628     24812                                   Bill Watterson   \n",
       "3274  3275         8                      J.K. Rowling, Mary GrandPré   \n",
       "861    862  17332218                                Brandon Sanderson   \n",
       "8853  8854     95602                                  Francine Rivers   \n",
       "7946  7947   5031805       Anonymous, Lane T. Dennis, Wayne A. Grudem   \n",
       "4482  4483     24814                                   Bill Watterson   \n",
       "6360  6361     70489                                   Bill Watterson   \n",
       "421    422    862041                                     J.K. Rowling   \n",
       "3752  3753        10                                     J.K. Rowling   \n",
       "6919  6920     24815                                   Bill Watterson   \n",
       "6589  6590     59715                                   Bill Watterson   \n",
       "9565  9566     70487                                   Bill Watterson   \n",
       "1307  1308  17927395                                    Sarah J. Maas   \n",
       "8977  8978    121792                                   Bill Watterson   \n",
       "9075  9076   2350129  The Church of Jesus Christ of Latter-day Saints   \n",
       "7253  7254     24816                                   Bill Watterson   \n",
       "5206  5207     24818                                   Bill Watterson   \n",
       "5918  5919    265205      Anonymous, Ronald A. Beers, Ronald A. Beers   \n",
       "9140  9141   9329354                                Brandon Sanderson   \n",
       "2100  2101     99298                      J.K. Rowling, Mary GrandPré   \n",
       "\n",
       "                                                  title  average_rating  \\\n",
       "3627                     The Complete Calvin and Hobbes            4.82   \n",
       "3274  Harry Potter Boxed Set, Books 1-5 (Harry Potte...            4.77   \n",
       "861      Words of Radiance (The Stormlight Archive, #2)            4.77   \n",
       "8853                           Mark of the Lion Trilogy            4.76   \n",
       "7946                                    ESV Study Bible            4.76   \n",
       "4482  It's a Magical World: A Calvin and Hobbes Coll...            4.75   \n",
       "6360  There's Treasure Everywhere: A Calvin and Hobb...            4.74   \n",
       "421            Harry Potter Boxset (Harry Potter, #1-7)            4.74   \n",
       "3752       Harry Potter Collection (Harry Potter, #1-6)            4.73   \n",
       "6919                The Indispensable Calvin and Hobbes            4.73   \n",
       "6589  The Authoritative Calvin and Hobbes: A Calvin ...            4.73   \n",
       "9565  Attack of the Deranged Mutant Killer Monster S...            4.72   \n",
       "1307  A Court of Mist and Fury (A Court of Thorns an...            4.72   \n",
       "8977                        The Revenge of the Baby-Sat            4.71   \n",
       "9075    Preach My Gospel: A Guide To Missionary Service            4.71   \n",
       "7253  Homicidal Psycho Jungle Cat: A Calvin and Hobb...            4.71   \n",
       "5206  The Days Are Just Packed: A Calvin and Hobbes ...            4.68   \n",
       "5918                  Life Application Study Bible: NIV            4.67   \n",
       "9140  The Way of Kings, Part 1 (The Stormlight Archi...            4.67   \n",
       "2100  The Harry Potter Collection 1-4 (Harry Potter,...            4.66   \n",
       "\n",
       "      ratings_count  \n",
       "3627          28900  \n",
       "3274          33220  \n",
       "861           73572  \n",
       "8853           9081  \n",
       "7946           8953  \n",
       "4482          22351  \n",
       "6360          16766  \n",
       "421          190050  \n",
       "3752          24618  \n",
       "6919          14597  \n",
       "6589          16087  \n",
       "9565           9713  \n",
       "1307         108384  \n",
       "8977          11503  \n",
       "9075           9891  \n",
       "7253          14113  \n",
       "5206          19143  \n",
       "5918          13614  \n",
       "9140           9792  \n",
       "2100          43929  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Book with the highest average rating:\n",
    "\n",
    "books[['id', 'book_id', 'authors', 'title', 'average_rating', 'ratings_count']] \\\n",
    "     .sort_values('average_rating', ascending = False).head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**2) Ratings** includes all the rates given by our selected group of users to the books they have chosen to rate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>314</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>439</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>588</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1169</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1185</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   book_id  user_id  rating\n",
       "0        1      314       5\n",
       "1        1      439       3\n",
       "2        1      588       5\n",
       "3        1     1169       4\n",
       "4        1     1185       4"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(981756, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.shape  # 981.756 rows x 3 columns imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.book_id.unique())  # There are ratings for 10.000 different books in total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53424"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.user_id.unique())  # Ratings to these 10k books have been given by 53.424 unique users"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**3) Tags** collects all the tags included in 'bookTags' and explains their meaning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>--1-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>--10-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>--12-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>--122-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id tag_name\n",
       "0       0        -\n",
       "1       1     --1-\n",
       "2       2    --10-\n",
       "3       3    --12-\n",
       "4       4   --122-"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_id</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>--1-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>--10-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>--12-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>--122-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag_id tag_name\n",
       "0       0        -\n",
       "1       1     --1-\n",
       "2       2    --10-\n",
       "3       3    --12-\n",
       "4       4   --122-"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags.head()   # 34.252 rows x 2 columns importedtags.shape    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34252"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tags.tag_id.unique())  # There are 34.252 different tags, which means no duplicates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4) Book Tags** contains all the tags associated to each and every book included in the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30574</td>\n",
       "      <td>167697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>11305</td>\n",
       "      <td>37174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>11557</td>\n",
       "      <td>34173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>8717</td>\n",
       "      <td>12986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>33114</td>\n",
       "      <td>12716</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id  tag_id   count\n",
       "0                  1   30574  167697\n",
       "1                  1   11305   37174\n",
       "2                  1   11557   34173\n",
       "3                  1    8717   12986\n",
       "4                  1   33114   12716"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bookTags.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(999912, 3)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bookTags.shape    # 999,912 rows imported x 3 columns imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bookTags.goodreads_book_id.unique())  # As expected, there are 10.000 books in total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34252"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bookTags.tag_id.unique())  # As expected, there are 34.252 different tags in total"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**5) to Read** indicates all the books that have been flagged as 'to read' by the readers included in the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1874</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  book_id\n",
       "0        1      112\n",
       "1        1      235\n",
       "2        1      533\n",
       "3        1     1198\n",
       "4        1     1874"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toRead.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(912705, 2)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toRead.shape    # 912.705 rows x 2 columns imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9986"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(toRead.book_id.unique())  # 9.986 different books have been given the 'toRead' tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48871"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(toRead.user_id.unique())   # 48.871 unique users have used the 'toRead' tag to flag at least one the 10k books"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.1 Adding own personal ratings to the dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to make the analysis more interesting and easier to follow, it is good to select a specific user and keep track of him along the whole process, and see what recommendations the system will be able to give him. \n",
    "For this purpose, I have leveraged my own personal Goodreads account, scraping all the ratings I have been given, since I had a Goodreads account, to the books I have read."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The ratings are stored in the 'maxRatings.csv' file, that I have created through the carping process included in the previous notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxRatings = pd.read_csv('maxRatings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>title</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>La ragazza di Bube</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Crash</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Flow My Tears, the Policeman Said</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Lost in Music</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Things The Grandchildren Should Know</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                 title  rating\n",
       "0           0                    La ragazza di Bube       3\n",
       "1           1                                 Crash       3\n",
       "2           2     Flow My Tears, the Policeman Said       5\n",
       "3           3                         Lost in Music       3\n",
       "4           4  Things The Grandchildren Should Know       4"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxRatings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(97, 3)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxRatings.shape    # 97 rows x 3 columns imported"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the purpose of the analysis, we have to assign a user ID to my account. In the following lines, I have checked that the maximum userID in the dataset is 53424, hence I have assigned the user number 55000 to my account, and appended a column that indicates it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53424"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(ratings['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxRatings['user_id'] = 55000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to see how many of the 97 books I have actually read and rated are in the books dataset selected for this analysis, let's merge maxRatings with the rating dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>title</th>\n",
       "      <th>rating</th>\n",
       "      <th>user_id</th>\n",
       "      <th>id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>best_book_id</th>\n",
       "      <th>work_id</th>\n",
       "      <th>books_count</th>\n",
       "      <th>isbn</th>\n",
       "      <th>...</th>\n",
       "      <th>ratings_count</th>\n",
       "      <th>work_ratings_count</th>\n",
       "      <th>work_text_reviews_count</th>\n",
       "      <th>ratings_1</th>\n",
       "      <th>ratings_2</th>\n",
       "      <th>ratings_3</th>\n",
       "      <th>ratings_4</th>\n",
       "      <th>ratings_5</th>\n",
       "      <th>image_url</th>\n",
       "      <th>small_image_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Crash</td>\n",
       "      <td>3</td>\n",
       "      <td>55000</td>\n",
       "      <td>7232</td>\n",
       "      <td>70241</td>\n",
       "      <td>70241</td>\n",
       "      <td>68058</td>\n",
       "      <td>81</td>\n",
       "      <td>312420331</td>\n",
       "      <td>...</td>\n",
       "      <td>13217</td>\n",
       "      <td>15187</td>\n",
       "      <td>925</td>\n",
       "      <td>825</td>\n",
       "      <td>1804</td>\n",
       "      <td>3900</td>\n",
       "      <td>5091</td>\n",
       "      <td>3567</td>\n",
       "      <td>https://images.gr-assets.com/books/1281416649m...</td>\n",
       "      <td>https://images.gr-assets.com/books/1281416649s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Flow My Tears, the Policeman Said</td>\n",
       "      <td>5</td>\n",
       "      <td>55000</td>\n",
       "      <td>4326</td>\n",
       "      <td>22584</td>\n",
       "      <td>22584</td>\n",
       "      <td>949696</td>\n",
       "      <td>92</td>\n",
       "      <td>1857983416</td>\n",
       "      <td>...</td>\n",
       "      <td>22063</td>\n",
       "      <td>25396</td>\n",
       "      <td>1187</td>\n",
       "      <td>206</td>\n",
       "      <td>1104</td>\n",
       "      <td>6014</td>\n",
       "      <td>10985</td>\n",
       "      <td>7087</td>\n",
       "      <td>https://images.gr-assets.com/books/1398026028m...</td>\n",
       "      <td>https://images.gr-assets.com/books/1398026028s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>Chronicle of a Death Foretold</td>\n",
       "      <td>5</td>\n",
       "      <td>55000</td>\n",
       "      <td>1239</td>\n",
       "      <td>23878</td>\n",
       "      <td>23878</td>\n",
       "      <td>59137</td>\n",
       "      <td>269</td>\n",
       "      <td>140003471X</td>\n",
       "      <td>...</td>\n",
       "      <td>56200</td>\n",
       "      <td>81653</td>\n",
       "      <td>3996</td>\n",
       "      <td>955</td>\n",
       "      <td>4077</td>\n",
       "      <td>18369</td>\n",
       "      <td>33190</td>\n",
       "      <td>25062</td>\n",
       "      <td>https://images.gr-assets.com/books/1430736599m...</td>\n",
       "      <td>https://images.gr-assets.com/books/1430736599s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>The Taming of the Shrew</td>\n",
       "      <td>3</td>\n",
       "      <td>55000</td>\n",
       "      <td>773</td>\n",
       "      <td>47021</td>\n",
       "      <td>47021</td>\n",
       "      <td>3152341</td>\n",
       "      <td>689</td>\n",
       "      <td>074347757X</td>\n",
       "      <td>...</td>\n",
       "      <td>126318</td>\n",
       "      <td>134240</td>\n",
       "      <td>2370</td>\n",
       "      <td>2869</td>\n",
       "      <td>9611</td>\n",
       "      <td>35666</td>\n",
       "      <td>47453</td>\n",
       "      <td>38641</td>\n",
       "      <td>https://images.gr-assets.com/books/1327935253m...</td>\n",
       "      <td>https://images.gr-assets.com/books/1327935253s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>The Merchant of Venice</td>\n",
       "      <td>3</td>\n",
       "      <td>55000</td>\n",
       "      <td>839</td>\n",
       "      <td>24128</td>\n",
       "      <td>24128</td>\n",
       "      <td>2682703</td>\n",
       "      <td>1088</td>\n",
       "      <td>743477561</td>\n",
       "      <td>...</td>\n",
       "      <td>115621</td>\n",
       "      <td>128289</td>\n",
       "      <td>2616</td>\n",
       "      <td>1925</td>\n",
       "      <td>9212</td>\n",
       "      <td>36917</td>\n",
       "      <td>46745</td>\n",
       "      <td>33490</td>\n",
       "      <td>https://images.gr-assets.com/books/1327871054m...</td>\n",
       "      <td>https://images.gr-assets.com/books/1327871054s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                              title  rating  user_id    id  \\\n",
       "0           1                              Crash       3    55000  7232   \n",
       "1           2  Flow My Tears, the Policeman Said       5    55000  4326   \n",
       "2           6      Chronicle of a Death Foretold       5    55000  1239   \n",
       "3           7            The Taming of the Shrew       3    55000   773   \n",
       "4           8             The Merchant of Venice       3    55000   839   \n",
       "\n",
       "   book_id  best_book_id  work_id  books_count        isbn  ...  \\\n",
       "0    70241         70241    68058           81   312420331  ...   \n",
       "1    22584         22584   949696           92  1857983416  ...   \n",
       "2    23878         23878    59137          269  140003471X  ...   \n",
       "3    47021         47021  3152341          689  074347757X  ...   \n",
       "4    24128         24128  2682703         1088   743477561  ...   \n",
       "\n",
       "   ratings_count work_ratings_count  work_text_reviews_count ratings_1  \\\n",
       "0          13217              15187                      925       825   \n",
       "1          22063              25396                     1187       206   \n",
       "2          56200              81653                     3996       955   \n",
       "3         126318             134240                     2370      2869   \n",
       "4         115621             128289                     2616      1925   \n",
       "\n",
       "  ratings_2  ratings_3  ratings_4  ratings_5  \\\n",
       "0      1804       3900       5091       3567   \n",
       "1      1104       6014      10985       7087   \n",
       "2      4077      18369      33190      25062   \n",
       "3      9611      35666      47453      38641   \n",
       "4      9212      36917      46745      33490   \n",
       "\n",
       "                                           image_url  \\\n",
       "0  https://images.gr-assets.com/books/1281416649m...   \n",
       "1  https://images.gr-assets.com/books/1398026028m...   \n",
       "2  https://images.gr-assets.com/books/1430736599m...   \n",
       "3  https://images.gr-assets.com/books/1327935253m...   \n",
       "4  https://images.gr-assets.com/books/1327871054m...   \n",
       "\n",
       "                                     small_image_url  \n",
       "0  https://images.gr-assets.com/books/1281416649s...  \n",
       "1  https://images.gr-assets.com/books/1398026028s...  \n",
       "2  https://images.gr-assets.com/books/1430736599s...  \n",
       "3  https://images.gr-assets.com/books/1327935253s...  \n",
       "4  https://images.gr-assets.com/books/1327871054s...  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxRatingsWithId = pd.merge(maxRatings, books, on = 'title')\n",
    "maxRatingsWithId.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next step will be to append my ratings to the ratings dataset. To do so, firstly I have to recondut maxRatings dataset to the exact same layout and format as the ratings one. Hence, I select only the columns needed, renaming one of them, given that 'id' in the 'books' dataset corresponds to 'book_id' in the rating dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>314</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>439</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>588</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1169</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1185</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   book_id  user_id  rating\n",
       "0        1      314       5\n",
       "1        1      439       3\n",
       "2        1      588       5\n",
       "3        1     1169       4\n",
       "4        1     1185       4"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxRatingsWithId = maxRatingsWithId[['id', 'user_id', 'rating']] \\\n",
    "                    .rename(columns = {'id' : 'book_id'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7232</td>\n",
       "      <td>55000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4326</td>\n",
       "      <td>55000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1239</td>\n",
       "      <td>55000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>773</td>\n",
       "      <td>55000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>839</td>\n",
       "      <td>55000</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   book_id  user_id  rating\n",
       "0     7232    55000       3\n",
       "1     4326    55000       5\n",
       "2     1239    55000       5\n",
       "3      773    55000       3\n",
       "4      839    55000       3"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxRatingsWithId.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = ratings.append(maxRatingsWithId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53425"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.user_id.unique())    # As expected, now ratings has one more unique user id"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Taking a closer look at the ratings dataset I have realized that there are several users who have rated the same book more than once, for whatever reason.\n",
    "For this reason, we will proceed to erase those duplicated ratings from the rating table: for those user-book pairs that have more than 1 rating we will then keep only one record, indicating the average of the given ratings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59787</th>\n",
       "      <td>3204</td>\n",
       "      <td>8946</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>42</td>\n",
       "      <td>8946</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42643</th>\n",
       "      <td>2308</td>\n",
       "      <td>8946</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95001</th>\n",
       "      <td>5091</td>\n",
       "      <td>8946</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>665335</th>\n",
       "      <td>34548</td>\n",
       "      <td>7420</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11033</th>\n",
       "      <td>691</td>\n",
       "      <td>6472</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81315</th>\n",
       "      <td>4359</td>\n",
       "      <td>2515</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729574</th>\n",
       "      <td>38259</td>\n",
       "      <td>3996</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11203</th>\n",
       "      <td>702</td>\n",
       "      <td>8336</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404837</th>\n",
       "      <td>20532</td>\n",
       "      <td>3114</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20700</th>\n",
       "      <td>1240</td>\n",
       "      <td>9858</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404840</th>\n",
       "      <td>20532</td>\n",
       "      <td>3233</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138165</th>\n",
       "      <td>7318</td>\n",
       "      <td>8928</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468001</th>\n",
       "      <td>23851</td>\n",
       "      <td>3848</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1321</th>\n",
       "      <td>113</td>\n",
       "      <td>6389</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  book_id  rating\n",
       "59787      3204     8946       5\n",
       "572          42     8946       4\n",
       "42643      2308     8946       4\n",
       "95001      5091     8946       4\n",
       "665335    34548     7420       4\n",
       "11033       691     6472       4\n",
       "81315      4359     2515       4\n",
       "729574    38259     3996       4\n",
       "11203       702     8336       3\n",
       "404837    20532     3114       3\n",
       "20700      1240     9858       3\n",
       "404840    20532     3233       3\n",
       "138165     7318     8928       3\n",
       "468001    23851     3848       3\n",
       "1321        113     6389       3"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The following table will show which pair of user-book have more than 1 rating in the dataset: \n",
    "\n",
    "userRatesPerBookCount = ratings.groupby(['user_id', 'book_id'], as_index = False).count().sort_values('rating', ascending = False)\n",
    "userRatesPerBookCount.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883706</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883707</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883708</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883709</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883710</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        book_id  user_id  rating\n",
       "883706     8946     3204       5\n",
       "883707     8946     3204       5\n",
       "883708     8946     3204       5\n",
       "883709     8946     3204       5\n",
       "883710     8946     3204       4"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's double check if this is true:\n",
    "\n",
    "ratings[(ratings.user_id == 3204) & (ratings.book_id == 8946)]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>845103</th>\n",
       "      <td>45150</td>\n",
       "      <td>3004</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363206</th>\n",
       "      <td>18285</td>\n",
       "      <td>8931</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575921</th>\n",
       "      <td>29542</td>\n",
       "      <td>4058</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754214</th>\n",
       "      <td>39703</td>\n",
       "      <td>131</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505067</th>\n",
       "      <td>25877</td>\n",
       "      <td>3657</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754213</th>\n",
       "      <td>39703</td>\n",
       "      <td>130</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>575925</th>\n",
       "      <td>29542</td>\n",
       "      <td>7052</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754212</th>\n",
       "      <td>39703</td>\n",
       "      <td>126</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>830329</th>\n",
       "      <td>44239</td>\n",
       "      <td>8849</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195951</th>\n",
       "      <td>9992</td>\n",
       "      <td>8640</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754211</th>\n",
       "      <td>39703</td>\n",
       "      <td>125</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363208</th>\n",
       "      <td>18285</td>\n",
       "      <td>9100</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>754210</th>\n",
       "      <td>39703</td>\n",
       "      <td>121</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894945</th>\n",
       "      <td>48272</td>\n",
       "      <td>8437</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>852588</th>\n",
       "      <td>45612</td>\n",
       "      <td>2369</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  book_id  rating\n",
       "845103    45150     3004     5.0\n",
       "363206    18285     8931     5.0\n",
       "575921    29542     4058     5.0\n",
       "754214    39703      131     5.0\n",
       "505067    25877     3657     5.0\n",
       "754213    39703      130     5.0\n",
       "575925    29542     7052     5.0\n",
       "754212    39703      126     5.0\n",
       "830329    44239     8849     5.0\n",
       "195951     9992     8640     5.0\n",
       "754211    39703      125     5.0\n",
       "363208    18285     9100     5.0\n",
       "754210    39703      121     5.0\n",
       "894945    48272     8437     5.0\n",
       "852588    45612     2369     5.0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here I calculate the rating mean per user and book, so that I can later replace the mean to the rating value\n",
    "# and get rid of the duplicates rows\n",
    "\n",
    "userRatesPerBookMean = ratings.groupby(['user_id', 'book_id'], as_index = False).mean().round(0).sort_values('rating', ascending = False)\n",
    "userRatesPerBookMean.head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59787</th>\n",
       "      <td>3204</td>\n",
       "      <td>8946</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  book_id  rating\n",
       "59787     3204     8946     5.0"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's double check if that worked:\n",
    "\n",
    "userRatesPerBookMean[(userRatesPerBookMean.user_id == 3204) & (userRatesPerBookMean.book_id == 8946)]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>59787</th>\n",
       "      <td>3204</td>\n",
       "      <td>8946</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>42</td>\n",
       "      <td>8946</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42643</th>\n",
       "      <td>2308</td>\n",
       "      <td>8946</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95001</th>\n",
       "      <td>5091</td>\n",
       "      <td>8946</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>665335</th>\n",
       "      <td>34548</td>\n",
       "      <td>7420</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  book_id  mean  count\n",
       "59787      3204     8946   5.0      5\n",
       "572          42     8946   5.0      4\n",
       "42643      2308     8946   5.0      4\n",
       "95001      5091     8946   5.0      4\n",
       "665335    34548     7420   5.0      4"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We create now a table grouped by user and book pair, calculating mean and number of rating that a user has made for the\n",
    "# same book\n",
    "\n",
    "ratings['count'] = ratings['rating']\n",
    "\n",
    "userRatesPerBook = ratings.groupby(['user_id', 'book_id'], as_index = False) \\\n",
    "                   .agg({'rating' : 'mean', 'count' : 'count'}) \\\n",
    "                   .rename(columns = {'rating':'mean'}) \\\n",
    "                   .sort_values('count', ascending = False)\n",
    "\n",
    "ratings = ratings.drop(['count'], axis = 1)\n",
    "            \n",
    "userRatesPerBook['mean'] = userRatesPerBook['mean'].round(0)\n",
    "userRatesPerBook.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>314</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>439</td>\n",
       "      <td>3</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>588</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1169</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1185</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   book_id  user_id  rating  mean  count\n",
       "0        1      314       5   5.0      1\n",
       "1        1      439       3   3.0      1\n",
       "2        1      588       5   5.0      1\n",
       "3        1     1169       4   4.0      1\n",
       "4        1     1185       4   4.0      1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now I append the mean and count per each user and book to the ratings table:\n",
    "\n",
    "ratings = pd.merge(ratings, userRatesPerBook, on = ['user_id', 'book_id'])\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883706</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883707</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883709</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883708</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883710</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736954</th>\n",
       "      <td>7420</td>\n",
       "      <td>34548</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736955</th>\n",
       "      <td>7420</td>\n",
       "      <td>34548</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736956</th>\n",
       "      <td>7420</td>\n",
       "      <td>34548</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736957</th>\n",
       "      <td>7420</td>\n",
       "      <td>34548</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644167</th>\n",
       "      <td>6472</td>\n",
       "      <td>691</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644168</th>\n",
       "      <td>6472</td>\n",
       "      <td>691</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251241</th>\n",
       "      <td>2515</td>\n",
       "      <td>4359</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251242</th>\n",
       "      <td>2515</td>\n",
       "      <td>4359</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251243</th>\n",
       "      <td>2515</td>\n",
       "      <td>4359</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251244</th>\n",
       "      <td>2515</td>\n",
       "      <td>4359</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        book_id  user_id  rating  mean  count\n",
       "883706     8946     3204       5   5.0      5\n",
       "883707     8946     3204       5   5.0      5\n",
       "883709     8946     3204       5   5.0      5\n",
       "883708     8946     3204       5   5.0      5\n",
       "883710     8946     3204       4   5.0      5\n",
       "736954     7420    34548       5   5.0      4\n",
       "736955     7420    34548       5   5.0      4\n",
       "736956     7420    34548       5   5.0      4\n",
       "736957     7420    34548       5   5.0      4\n",
       "644167     6472      691       5   5.0      4\n",
       "644168     6472      691       5   5.0      4\n",
       "251241     2515     4359       5   5.0      4\n",
       "251242     2515     4359       5   5.0      4\n",
       "251243     2515     4359       5   5.0      4\n",
       "251244     2515     4359       5   5.0      4"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.sort_values('count', ascending = False).head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(981818, 5)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.shape    # 981.820 rows x 5 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can now drop the duplicates records for the combination user/book with more than 1 rating\n",
    "\n",
    "ratings = ratings.drop_duplicates(subset = ['book_id', 'user_id'], keep = 'first', inplace = False)\\\n",
    "                                  .sort_values('count', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = ratings.drop(['rating', 'count'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>883706</th>\n",
       "      <td>8946</td>\n",
       "      <td>3204</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883651</th>\n",
       "      <td>8946</td>\n",
       "      <td>42</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251241</th>\n",
       "      <td>2515</td>\n",
       "      <td>4359</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644166</th>\n",
       "      <td>6472</td>\n",
       "      <td>691</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>883717</th>\n",
       "      <td>8946</td>\n",
       "      <td>5091</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        book_id  user_id  mean\n",
       "883706     8946     3204   5.0\n",
       "883651     8946       42   5.0\n",
       "251241     2515     4359   5.0\n",
       "644166     6472      691   5.0\n",
       "883717     8946     5091   5.0"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(979540, 3)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.shape  # 979.542 rows x 3 columns: 2.278 records removed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I restore the 'ratings' table column names as they were prior to the modification I have made\n",
    "ratings.columns = ['book_id', 'user_id', 'rating']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['book_id', 'user_id', 'rating'], dtype='object')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "With the purpose of making more relevant recommendations, let's have a look at what is the ratings distribution per users: in other words, let's see if there are users who have rated very few books."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>totalRatings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12873</th>\n",
       "      <td>12874</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30943</th>\n",
       "      <td>30944</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12380</th>\n",
       "      <td>12381</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52035</th>\n",
       "      <td>52036</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28157</th>\n",
       "      <td>28158</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6629</th>\n",
       "      <td>6630</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45553</th>\n",
       "      <td>45554</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9805</th>\n",
       "      <td>9806</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24142</th>\n",
       "      <td>24143</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9667</th>\n",
       "      <td>9668</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  totalRatings\n",
       "12873    12874           200\n",
       "30943    30944           200\n",
       "12380    12381           199\n",
       "52035    52036           199\n",
       "28157    28158           199\n",
       "6629      6630           197\n",
       "45553    45554           197\n",
       "9805      9806           196\n",
       "24142    24143           196\n",
       "9667      9668           196"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratesPerUser = ratings[['user_id', 'rating']].groupby(['user_id'], as_index = False) \\\n",
    "                      .count() \\\n",
    "                      .rename(columns = {'rating' : 'totalRatings'}) \\\n",
    "                      .sort_values('totalRatings', ascending = False)\n",
    "\n",
    "ratesPerUser.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABJEAAAFlCAYAAABbdXkaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3df7RkZXkn+u8jqFETI0prCGAaHTSjTkK0Q8xEHUaioiaiuXECKzcy6iyiS2ZicjM3eJ0VnXhZi8QY7/VOBhdRFGdUdKJGJuAoIT+cH/6gUcBGIbTYCS0EWkmixlwy6Dt/7H1C2V1Vu2pXHfvQ/fmsddapemvv57z7vOfZu85T7967WmsBAAAAgHnuc7A7AAAAAMDWp4gEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADDryYHdgrKOPPrpt3779YHcDAAAA4JBx9dVXf6m1tm3aa/faItL27duzc+fOg90NAAAAgENGVf3ZrNeczgYAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAw68mB3gGT7uZetNd6e85+71ngAAAAAgzORquqiqrqjqnZNtL2nqq7pv/ZU1TV9+/aq+tuJ1948sc6TquozVbW7qt5UVdW3P7Sqrqiqm/rvR23GhgIAAAAw3iKns709yWmTDa21n2mtndRaOynJ+5K8f+Llz2+81lp72UT7BUnOTnJi/7UR89wkV7bWTkxyZf8cAAAAgC1ksIjUWvtokjunvdbPJvpnSd49L0ZVHZPkwa21j7XWWpJ3JHl+//LpSS7uH1880Q4AAADAFrHqhbWfmuT21tpNE20nVNWnq+pPquqpfduxSfZOLLO3b0uSR7TWbkuS/vvDV+wTAAAAAGu26oW1z8y3zkK6LckjW2tfrqonJfm9qnp8kpqyblv2h1XV2elOicsjH/nIEd0FAAAAYIzRM5Gq6sgkP5XkPRttrbW7Wmtf7h9fneTzSR6TbubRcROrH5fk1v7x7f3pbhunvd0x62e21i5sre1ore3Ytm3b2K4DAAAAsKRVTmf78SQ3tNb+/jS1qtpWVUf0jx+V7gLaN/enqX21qp7cX0fpRUk+2K92aZKz+sdnTbQDAAAAsEUMFpGq6t1JPpbksVW1t6pe2r90Rg68oPbTklxXVdcm+d0kL2utbVyU++VJ3pJkd7oZSh/q289P8oyquinJM/rnAAAAAGwhg9dEaq2dOaP9n09pe1+S981YfmeSJ0xp/3KSU4f6AQAAAMDBs+rd2QAAAAA4DCgiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBg0Wkqrqoqu6oql0Tba+tqi9W1TX913MmXntVVe2uqhur6lkT7af1bbur6tyJ9hOq6hNVdVNVvaeq7rfODQQAAABgdYvMRHp7ktOmtL+xtXZS/3V5klTV45KckeTx/Tr/vqqOqKojkvx2kmcneVySM/tlk+TX+1gnJvnLJC9dZYMAAAAAWL/BIlJr7aNJ7lww3ulJLmmt3dVa+0KS3UlO7r92t9Zubq39XZJLkpxeVZXk6Ul+t1//4iTPX3IbAAAAANhkq1wT6Zyquq4/3e2ovu3YJLdMLLO3b5vV/rAkf9Vau3u/dgAAAAC2kLFFpAuSPDrJSUluS/KGvr2mLNtGtE9VVWdX1c6q2rlv377legwAAADAaKOKSK2121tr32itfTPJ76Q7XS3pZhIdP7HocUlundP+pSQPqaoj92uf9XMvbK3taK3t2LZt25iuAwAAADDCqCJSVR0z8fQFSTbu3HZpkjOq6v5VdUKSE5N8MslVSU7s78R2v3QX3760tdaS/FGSn+7XPyvJB8f0CQAAAIDNc+TQAlX17iSnJDm6qvYmeU2SU6rqpHSnnu1J8vNJ0lq7vqrem+SzSe5O8orW2jf6OOck+XCSI5Jc1Fq7vv8Rv5Lkkqr6v5N8Oslb17Z1AAAAAKzFYBGptXbmlOaZhZ7W2nlJzpvSfnmSy6e035x7TocDAAAAYAta5e5sAAAAABwmFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwKDBIlJVXVRVd1TVrom211fVDVV1XVV9oKoe0rdvr6q/rapr+q83T6zzpKr6TFXtrqo3VVX17Q+tqiuq6qb++1GbsaEAAAAAjLfITKS3Jzltv7YrkjyhtfYDSf40yasmXvt8a+2k/utlE+0XJDk7yYn910bMc5Nc2Vo7McmV/XMAAAAAtpDBIlJr7aNJ7tyv7SOttbv7px9Pcty8GFV1TJIHt9Y+1lprSd6R5Pn9y6cnubh/fPFEOwAAAABbxDquifSSJB+aeH5CVX26qv6kqp7atx2bZO/EMnv7tiR5RGvttiTpvz981g+qqrOramdV7dy3b98aug4AAADAIlYqIlXVq5PcneSdfdNtSR7ZWvuhJL+U5F1V9eAkNWX1tuzPa61d2Frb0VrbsW3btrHdBgAAAGBJR45dsarOSvITSU7tT1FLa+2uJHf1j6+uqs8neUy6mUeTp7wdl+TW/vHtVXVMa+22/rS3O8b2CQAAAIDNMWomUlWdluRXkjyvtfb1ifZtVXVE//hR6S6gfXN/mtpXq+rJ/V3ZXpTkg/1qlyY5q3981kQ7AAAAAFvE4Eykqnp3klOSHF1Ve5O8Jt3d2O6f5IquJpSP93die1qSX6uqu5N8I8nLWmsbF+V+ebo7vT0g3TWUNq6jdH6S91bVS5P8eZIXrmXLAAAAAFibwSJSa+3MKc1vnbHs+5K8b8ZrO5M8YUr7l5OcOtQPAAAAAA6eddydDQAAAIBDnCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABi0UBGpqi6qqjuqatdE20Or6oqquqn/flTfXlX1pqraXVXXVdUTJ9Y5q1/+pqo6a6L9SVX1mX6dN1VVrXMjAQAAAFjNojOR3p7ktP3azk1yZWvtxCRX9s+T5NlJTuy/zk5yQdIVnZK8JsmPJDk5yWs2Ck/9MmdPrLf/zwIAAADgIFqoiNRa+2iSO/drPj3Jxf3ji5M8f6L9Ha3z8SQPqapjkjwryRWttTtba3+Z5Iokp/WvPbi19rHWWkvyjolYAAAAAGwBq1wT6RGttduSpP/+8L792CS3TCy3t2+b1753SvsBqursqtpZVTv37du3QtcBAAAAWMZmXFh72vWM2oj2Axtbu7C1tqO1tmPbtm0rdBEAAACAZaxSRLq9PxUt/fc7+va9SY6fWO64JLcOtB83pR0AAACALWKVItKlSTbusHZWkg9OtL+ov0vbk5P8dX+624eTPLOqjuovqP3MJB/uX/tqVT25vyvbiyZiAQAAALAFHLnIQlX17iSnJDm6qvamu8va+UneW1UvTfLnSV7YL355kuck2Z3k60lenCSttTur6nVJruqX+7XW2sbFul+e7g5wD0jyof4LAAAAgC1ioSJSa+3MGS+dOmXZluQVM+JclOSiKe07kzxhkb4AAAAA8O23UBGJe5ft51621nh7zn/uWuMBAAAA9z6bcXc2AAAAAA4xikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGji0hV9diqumbi6ytV9cqqem1VfXGi/TkT67yqqnZX1Y1V9ayJ9tP6tt1Vde6qGwUAAADAeh05dsXW2o1JTkqSqjoiyReTfCDJi5O8sbX2m5PLV9XjkpyR5PFJvjfJH1TVY/qXfzvJM5LsTXJVVV3aWvvs2L4BAAAAsF6ji0j7OTXJ51trf1ZVs5Y5PcklrbW7knyhqnYnObl/bXdr7eYkqapL+mUVkQAAAAC2iHVdE+mMJO+eeH5OVV1XVRdV1VF927FJbplYZm/fNqsdAAAAgC1i5SJSVd0vyfOS/Ke+6YIkj053qtttSd6wseiU1duc9mk/6+yq2llVO/ft27dSvwEAAABY3DpmIj07yadaa7cnSWvt9tbaN1pr30zyO7nnlLW9SY6fWO+4JLfOaT9Aa+3C1tqO1tqObdu2raHrAAAAACxiHUWkMzNxKltVHTPx2guS7OofX5rkjKq6f1WdkOTEJJ9MclWSE6vqhH5W0xn9sgAAAABsEStdWLuqHpjurmo/P9H8G1V1UrpT0vZsvNZau76q3pvugtl3J3lFa+0bfZxzknw4yRFJLmqtXb9KvwAAAABYr5WKSK21ryd52H5tPzdn+fOSnDel/fIkl6/SFwAAAAA2z7ruzgYAAADAIUwRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGKSIBAAAAMAgRSQAAAAABikiAQAAADBIEQkAAACAQYpIAAAAAAxSRAIAAABgkCISAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAoCMPdgfY+rafe9la4+05/7lrjQcAAABsPjORAAAAABikiAQAAADAIEUkAAAAAAatXESqqj1V9ZmquqaqdvZtD62qK6rqpv77UX17VdWbqmp3VV1XVU+ciHNWv/xNVXXWqv0CAAAAYH3WNRPpn7bWTmqt7eifn5vkytbaiUmu7J8nybOTnNh/nZ3kgqQrOiV5TZIfSXJyktdsFJ4AAAAAOPg263S205Nc3D++OMnzJ9rf0TofT/KQqjomybOSXNFau7O19pdJrkhy2ib1DQAAAIAlraOI1JJ8pKqurqqz+7ZHtNZuS5L++8P79mOT3DKx7t6+bVb7t6iqs6tqZ1Xt3Ldv3xq6DgAAAMAijlxDjB9rrd1aVQ9PckVV3TBn2ZrS1ua0f2tDaxcmuTBJduzYccDrAAAAAGyOlWcitdZu7b/fkeQD6a5pdHt/mlr673f0i+9NcvzE6scluXVOOwAAAABbwEpFpKp6UFV918bjJM9MsivJpUk27rB2VpIP9o8vTfKi/i5tT07y1/3pbh9O8syqOqq/oPYz+zYAAAAAtoBVT2d7RJIPVNVGrHe11v5LVV2V5L1V9dIkf57khf3ylyd5TpLdSb6e5MVJ0lq7s6pel+Sqfrlfa63duWLfAAAAAFiTlYpIrbWbk/zglPYvJzl1SntL8ooZsS5KctEq/QEAAABgc6zj7mwAAAAAHOIUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYNCRB7sDHH62n3vZ2mPuOf+5a48JAAAA3MNMJAAAAAAGKSIBAAAAMEgRCQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAwSBEJAAAAgEGKSAAAAAAMUkQCAAAAYJAiEgAAAACDFJEAAAAAGHTkwe4ArMP2cy9ba7w95z93rfEAAADg3s5MJAAAAAAGKSIBAAAAMGh0Eamqjq+qP6qqz1XV9VX1C337a6vqi1V1Tf/1nIl1XlVVu6vqxqp61kT7aX3b7qo6d7VNAgAAAGDdVrkm0t1J/o/W2qeq6ruSXF1VV/SvvbG19puTC1fV45KckeTxSb43yR9U1WP6l387yTOS7E1yVVVd2lr77Ap9AwAAAGCNRheRWmu3Jbmtf/zVqvpckmPnrHJ6kktaa3cl+UJV7U5ycv/a7tbazUlSVZf0yyoiAQAAAGwRa7kmUlVtT/JDST7RN51TVddV1UVVdVTfdmySWyZW29u3zWqf9nPOrqqdVbVz37596+g6AAAAAAtYuYhUVd+Z5H1JXtla+0qSC5I8OslJ6WYqvWFj0SmrtzntBza2dmFrbUdrbce2bdtW7ToAAAAAC1rlmkipqvumKyC9s7X2/iRprd0+8frvJPn9/uneJMdPrH5cklv7x7PaAQAAANgCVrk7WyV5a5LPtdZ+a6L9mInFXpBkV//40iRnVNX9q+qEJCcm+WSSq5KcWFUnVNX90l18+9Kx/QIAAABg/VaZifRjSX4uyWeq6pq+7f9KcmZVnZTulLQ9SX4+SVpr11fVe9NdMPvuJK9orX0jSarqnCQfTnJEkotaa9ev0C8AAAAA1myVu7P9t0y/ntHlc9Y5L8l5U9ovn7ceAAAAAAfXWu7OBgAAAMChbaULa8Ohavu5l6013p7zn7vWeAAAAPDtZiYSAAAAAIMUkQAAAAAYpIgEAAAAwCBFJAAAAAAGKSIBAAAAMEgRCQAAAIBBRx7sDsDhYPu5l6095p7zn7v2mAAAADCLmUgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABjkwtpwL7Xui3W7UDcAAADzmIkEAAAAwCBFJAAAAAAGKSIBAAAAMMg1kYAkrrEEAADAfGYiAQAAADBIEQkAAACAQYpIAAAAAAxyTSRgU7jGEgAAwKHFTCQAAAAABikiAQAAADDI6WzAvcK6T49LnCIHAACwDEUk4LDluk0AAACLczobAAAAAIPMRAJYEzObAACAQ5mZSAAAAAAMMhMJYItyMXEAAGArUUQCOIw45Q4AABhLEQmA0TZjttQ6KXIBAMD6uCYSAAAAAIPMRALgkLXVZ0ptBrOvAADYLFumiFRVpyX5f5MckeQtrbXzD3KXAOBe595QOFPoAgC4d9oSRaSqOiLJbyd5RpK9Sa6qqktba589uD0DANbt3lDoWjeFMwDgULAlikhJTk6yu7V2c5JU1SVJTk+iiAQA3OsdjoWzrU5hD+Ae7uDLorZKEenYJLdMPN+b5EcOUl8AADjEKezBoWPdBYt7Q0Flq+/Dtnr/EoWusbZKEammtLUDFqo6O8nZ/dOvVdWNm9qrb5+jk3zpMIq3GTEPxz7a5q0Z83Ds4+G4zZsRc6vH24yYh2MfD8dt3oyYWz3eZsQ8HPtom7dmzC3Xx/r19cabYt39WznmtyHeZsTcUvG+TeNyb/V9s17YKkWkvUmOn3h+XJJb91+otXZhkgu/XZ36dqmqna21HYdLvM2IeTj20TZvzZiHYx8Px23ejJhbPd5mxDwc+3g4bvNmxNzq8TYj5uHYR9u8NWMejn08HLd5M2Ju9XibFfNQc5+D3YHeVUlOrKoTqup+Sc5IculB7hMAAAAAvS0xE6m1dndVnZPkw0mOSHJRa+36g9wtAAAAAHpbooiUJK21y5NcfrD7cZCs+xS9rR5vM2Iejn20zVsz5uHYx8Nxmzcj5laPtxkxD8c+Ho7bvBkxt3q8zYh5OPbRNm/NmIdjHw/Hbd6MmFs93mbFPKRUawdcvxoAAAAAvsVWuSYSAAAAAFuYItJBUlUXVdUdVbVrzXH3VNVnquqaqtq5hni/UFW7qur6qnrlyBgHbGtVva6qruv7+ZGq+t4V472nj3VN/zu4Zol4x1fVH1XV5/rt/IW+/YX9829W1cJX6J8T77VV9cWJfj5nDTF/sKo+1o/5f66qBy8Y7zuq6pNVdW0f79/27W/t266rqt+tqu9coo+zYv7XiW2+tap+b9GY/fpHVNWnq+r3++fnVNXuqmpVdfQysWbEe3tVfWGijyetGO/pVfWpPm8urqqlThuelsMr5su0eKPzpV//If3fxw393+SPjs2XOfFG58ucmGPz5bET/bimqr5SVa8cOy5z4q06Lr/Yj8Guqnp3n5Oj82VGvFXzZVrM0TlTU45RK+bLtHhLjUtNP0Y9tKquqKqb+u9H9e3f3/9N3lVVv7yGeKdU1V9P9PdX1xDzqKr6QP87/WRVPWHBeFP3CVV18kT/rq2qFyzRx1kxf3a/nPrm/n+bM+K9vrp9xHX9Nj6kb39Ydcfcr1XVv1uif7Piba+qv53o35vXEPN+VfW26vZn11bVKQvGm5ofVXX6RPvOqnrKEn2cFfNfT2zzrqr6RlU9dCjexGu/XBP7rrH5Mife6HyZE3Nsvkw95i2SL0vGG8yVedtcVf+yqm6sLg9/o28blS9z4o3Olzkxx+bL1P3/CuMyK97ocamqk6rq43VP7p7ct489vsyKt8rxZVbMsfky9f3cCuMyK95C43JYaq35OghfSZ6W5IlJdq057p4kR68p1hOS7ErywHTXz/qDJCeuY1uTPHji8b9K8uZ1/e6SvCHJry4R75gkT+wff1eSP03yuCT/MMljk/xxkh1riPfaJL88cixmxbwqyT/p21+S5HULxqsk39k/vm+STyR58n7j8ltJzl2ij1Nj7rfM+5K8aMlt/6Uk70ry+/3zH0qyfezf+pR4b0/y02PGZf946QrztyR5TP/aryV56ZLxDtiuFfNl7u9p2Xzp17k4yb/oH98vyUPG5suceKPzZU7MUfmyX9wjkvxFku9bZVymxVtlXJIcm+QLSR7QP39vkn8+Nl/mxBudLzNivmRszmTGMWrsuMyKt+y4ZPox7zfS70+TnJvk1/vHD0/yw0nOm/X3vmS8U9Lv29bYx9cneU3/+PuTXLlgvKn7hI3fb//4mCR3bDwfG3O/9f5RkpsXjPfMib78+sQ2PyjJU5K8LMm/W+J3OCve9izwfm/JmK9I8raJv6Ork9xngXhT8yPJd+aey1z8QJIblujjYM4l+ckkf7hIvL79+HQ33Pmz9PuujMyXOfFOych8mRNzbL68dto2ZYF8WSbeIrkyJ+Y/TbdfvP/GeKyYL7Pibd//d72GmKPyZb/X/37/P3ZcZsVbcVw+kuTZ/ePnJPnjVfJlTrxTMv74Mivm2HyZ+n5u7LjMirfouByOX2YiHSSttY8mufNg92PAP0zy8dba11trdyf5kyRTPzGcZ9q2tta+MvH0QUnaKvE2VFUl+WdJ3r1EvNtaa5/qH381yeeSHNta+1xr7cZF4wzFWzbOgjEfm+Sj/WJXJPnfFozXWmtf65/et/9qG+PS/x4fkOXGZWrMjder6ruSPD3JwjORquq4JM9N8paJn/Pp1tqeRWMMxVvFlHgPS3JXa+1P++cLj8k8q+TLPGPypf905mlJ3tr37e9aa381Nl9mxVs2zoIxR+XLfk5N8vnW2p+taVz+Pt5E/5cel96RSR5Q3UyeBya5dZV8mRZvZJx5Mf8m43Nm6jFqhXGZe8xbdFxmHKNOT1fYTP/9+f2yd7TWrkryP9cRb1FLxnxckiv79W5Isr2qHjEUb9Y+YeL3myTfkRnjs0zM/ZyZKWM0I95HJvry8STH9e1/01r7b0n+/1k/ZJl4i1oy5uS43JHkr5Ls2G/dhd9/tda+1lpr+7cv2MdFcm7hcem9Mcn/ORlrhXyZGm9RS8YclS9zfvZgvqzwP8XUMZkT8+VJzm+t3dUvc0f/fVS+zIq3qCVjjsqXDfvv/1cdl4HjybLj0pJszKz+7vTH6hXyZWq8RS0Zc2y+TH0/t8K4LPL+cOa4HI4UkQ49LclHqurqqjp7xVi7kjytummqD0xXOT5+5R72quq8qrolyc8mmTolcoSnJrm9tXbTyD5tT/fJ/SfW0Zkp8c7pp2xeVP2pAivG3JXkef1LL8wS41PdaVjXpKvSX9Fa+0Tf/rZ0MyO+P8n/t2TfpsbsvSDdJwxfmb72VP9Pujdo31ymHyPindePyxur6v4rxPtSkvvWPadZ/HSWz5mpObxCvszbJ4zJl0cl2ZfkbdWdxveWqnrQEusvE29svsyKOTpfJpyRiTcRa9iPfUu83tLj0lr7YpLfTPLnSW5L8tettY+M6M8i8Ubly7SY6WYjjc2ZmceokeMydMxb5fjyiNbabUn3oUC6T4hXMS/ej/bT+D9UVbZJ19AAAAkVSURBVI9fQ8xrk/xU0p0qkG4W3lLFkf1V1Y9U1fVJPpPkZRNv+tfhZzLujf5Lknxojf3YP94J/f7oT6rqqWuIeW2S06vqyKo6IcmTsmDuzMqPqnpBVd2Q5LL+Zy1sXs71+XRautnIi8R6XpIvttauXaYPI+ONypc5MVfJl6nHvBXyZegYumyuPCbJU6vqE/3f8Q8vse6y8cbmy6yYo/Old8D+f8X92LzjybLj8sokr+/z7zeTvGqJdZeNN/b4Mivm2HyZ+X5u5Lgs8v5w7LHlkKSIdOj5sdbaE5M8O8krquppYwO11j6Xbur0FUn+S7pEX9sbvdbaq1trxyd5Z5Jz1hR2dJW4uuv/vC/JK5csdCwa74Ikj05yUrp/oN6whpgvSTfOV6c7ze3vFo3VWvtGa+2kdDvrk6s/D7m19uIk35tuttPPLNO/WTF7S41NVf1Ekjtaa1cv04cR8V6VrmD2w0kemuRXxsbrP8U9I8kbq+qTSb6a5XNmag6vkC/z9glj8uXIdNOAL2it/VC6mSTnLhljkXir5MusmKPzJemuqZDuTcZ/2mhbZT82LV5v6XHp/0E4PckJ6fL3QVX1vy8TY8F4o/JlVsx0/3COypl5x6gx47LAMe/e8Cnkp9KdGvmD6T4EWOoadDOcn+So/gOCf5nk01nxvUBr7ROttcen+zt6VVV9x+rd7P55SPL11tpS15usqlen26Z3rqkf+8e7Lckj+/3RLyV5Vy14TbY5MS9KsjfJznQfaPyPLJ47U/OjtfaB1tr3p5uF9rpl+jeQcz+Z5L+31gZny/QFp1dnTR8uDsQblS8DMcfmy8xj3sh8mXsMHZkrRyY5Kt3lD/51kvdWVS2x/qLxVsmXWTFH50vvgP3/ivuxqceTkePy8iS/2OffL6afhb2CWfFWOb7Mijk2X2a+nxs5LnPfH449thzKFJEOMa21v5/CmOQDSU5eMd5bW2tPbK09Ld3Uv1EzfAa8K2s45ae60yN+Ksl7Rqx733TFmXe21t6/hr4cEK+1dntfZPlmkt/JkmMzI+YNrbVnttaelO5g9Pll+9q603z+ON2nhBtt30j3exw1LvvHrKqHpdvey5YI82NJnldVe5JckuTpVfUfx/RnXrzWnSrY+qnPb8vi4zIr3sdaa09trZ2cbmrsUjmzQA4vlS+z4q2QL3uT7J2YZfa76Qo2Y02Nt2K+zIq5ar48O8mnWmu3T3ltzH7sgHgrjMuPJ/lCa21fa+1/Jnl/kn+8ZIzBeCvky7yYo3NmgWPUsvkyNd4qx5fe7VV1TB9r4zoNq5gar7X2ldafVtxauzzdLK9FL6g+L+aL+w8IXpRkW7prW62sL9z9TbrrUa3DtJl9c1XVWUl+IsnP9h8ErGRavNbaXa21L/ePr06373nMijHvbq39YmvtpNba6emu+7bse7Sp+dG60z0evcTfzlDMZcbl0ekKzdf2x9fjknyqqr5nRF/mxlshX4ZiLp0vixzzlsmXBeItnSvpjq3v748Bn0w3C3vM38jceCvmy6yYo/NlaP+/7H5sIN6YcTkr3fE06T6UWul/v1nxVjy+zIs5Jl8G388tmS9D8caMyyFNEekQUlUPqu66M6nu1I1nppuet0rMh/ffH5luh7eWBKqqEyeePi/JDWsI++PpLgS5d8m+VLqK+Odaa7+1aidmxdt4c957QZYYmzkxN8bnPkn+TZKpd7CYEm9b3XOHlwek+93dWFX/YOLn/WSWGJcZMTfWf2G6i/HNPF9+f621V7XWjmutbU+38/7D1tromRWz4k3801TpPn1daFzmxNsYk/unm6Wx0Jj060zN4bH5MrBPGJUvrbW/SHJLVT22bzo1yWeXibFIvFXyZU7MUfky4Vs+OVzDfmzaJ5GjxiXdKWJPrqoH9n/Lp6abTTjW1Hhj82Ug5io5c8AxapVxmXPMGzsuGy5N9yY6/fcPjowzN15Vfc/GzIDqTg24T5IvrxjzIdXNmkuSf5Hko22F2bpVdUL/T1Sq6vvSXYtiz9h4E3Hvk+5Yc8kS65yW7m/uea21r6+hD1Pj9cfHI/rHj0p3AfibV4z5wH6/nqp6RpK7W2uD++JZ+VFV/2Dib+eJ6W5IsNDfzrycq6rvTvJPsuDffGvtM621h7fWtvfH173pPgT4i0XWXybe2HwZiDkqX2Yd88bmy7xj6Jhc6f1eumtbpqoek+5v5EtLxhiMt0q+zIk5Kl96B+z/V9yPTT2erDAut6bLsaTb9lU/8J8ab8Xjy6yYY/Nl6vu5FfJl5vvDFcbl0Na2wNW9D8evdG9Mb0t3sbO9WfIOTjNiPird9Ptrk1yf5NVriPlf0/2DeG2SU9e1relm1OxKcl2S/5zuQtYr/e7S3TXoZSP695R01425Lsk1/ddz0h109ya5K8ntST68Yrz/kO783OvSvVk/Zg19/IV0d2r703RTQmvBeD+Qbsrodf04/Gq6g8F/7/u4K92U9Acv0ccDYk689sdJTlvh7/CU3HM3tX/Vj8vd6Q5Kb1kx3h9ObPN/TH+HuRXivT7dP+83pjvtcOUcHpsvs+Ktki/9uielmxJ+Xbo3bEeNzZc58Ubny5yYo/Klj/fAdG+WvnuibZX92AHx1jAu/zbdP2+7+t/f/VfJlxnxVsqXGTFXyZkDjlErjsvUY94y45Lpx7yHpbt46E3994f2y35Pv8xX0l3odW/22+8uGe+cdLl+bbqLMP/jNfTxR/u2G9J9knzUgvGm7hOS/Fzfx2vSnR7x/CX6OHM/k24//PElx2V3ursDbhxX3zyx/J50s9G+1i//uLHx0s3M2RiXTyX5yVX7mO4OVjemy50/yH53eJwTb2p+pCtUbYzLx5I8ZYk+zsy5dHd1vGSZcdnv9T25585no/JlTrzR+TIn5th8mXrMywL5sky8RXJlTsz7pdvv7+r78vQV82VqvKyWL7Nibs+IfJm1/x87LrPirTguT0l3t7lr010v9Umr5MuceKscX2bFHJsvU9/PjR2XWfEWHZfD8WvjFw4AAAAAMzmdDQAAAIBBikgAAAAADFJEAgAAAGCQIhIAAAAAgxSRAAAAABikiAQAAADAIEUkAAAAAAYpIgEAAAAw6H8BwBrtlYSbI0wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20, 6]\n",
    "plt.hist(ratesPerUser['totalRatings'], bins = 50)\n",
    "plt.xticks(np.arange(min(ratesPerUser['totalRatings']), max(ratesPerUser['totalRatings'])+1, 4.0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The histogram above shows how many users have rated only 1 to 15 books. Considered that we are using only a subset (10k) of all the books listed in Goodreads, it will be pretty difficult to give accurate recommendation to users with that few ratings.  \n",
    "Cosnidering also the fact that this project has rather an illustrative purpose, I have decided to include only user that have rated at least 20 of the 10K books included in the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = pd.merge(ratings, ratesPerUser, on = ['user_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>totalRatings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>977649</th>\n",
       "      <td>1653</td>\n",
       "      <td>30944</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489087</th>\n",
       "      <td>3252</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489034</th>\n",
       "      <td>172</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489035</th>\n",
       "      <td>171</td>\n",
       "      <td>12874</td>\n",
       "      <td>3.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489036</th>\n",
       "      <td>219</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489037</th>\n",
       "      <td>213</td>\n",
       "      <td>12874</td>\n",
       "      <td>3.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489038</th>\n",
       "      <td>216</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489039</th>\n",
       "      <td>228</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489040</th>\n",
       "      <td>230</td>\n",
       "      <td>12874</td>\n",
       "      <td>3.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489041</th>\n",
       "      <td>229</td>\n",
       "      <td>12874</td>\n",
       "      <td>4.0</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        book_id  user_id  rating  totalRatings\n",
       "977649     1653    30944     4.0           200\n",
       "489087     3252    12874     4.0           200\n",
       "489034      172    12874     4.0           200\n",
       "489035      171    12874     3.0           200\n",
       "489036      219    12874     4.0           200\n",
       "489037      213    12874     3.0           200\n",
       "489038      216    12874     4.0           200\n",
       "489039      228    12874     4.0           200\n",
       "489040      230    12874     3.0           200\n",
       "489041      229    12874     4.0           200"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.sort_values('totalRatings', ascending = False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "272086"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Keeping only users with more than a 'minimum' of ratings\n",
    "minimumRatings = ratings.totalRatings <= 20\n",
    "np.sum(minimumRatings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "707454"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = ratings[-minimumRatings]\n",
    "len(ratings)   # only 707.456 ratings have been kept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13972"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.user_id.unique())  # total of remaining users after removing the ones with less than the established minimum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9998"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.book_id.unique())  # total of remaining books after removing users with less than the established minimum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then I had a look at the same thing but about the books: how many books have very few ratings? Looking at the results, I have decided - for the same reason as above, that is to avoid 'obscure' recommendations of books that only few users have read - to remove books with less than 30 ratings received."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>receivedRatings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9342</th>\n",
       "      <td>9345</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9855</th>\n",
       "      <td>9858</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9087</th>\n",
       "      <td>9090</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8802</th>\n",
       "      <td>8804</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8371</th>\n",
       "      <td>8372</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      book_id  receivedRatings\n",
       "9342     9345                2\n",
       "9855     9858                1\n",
       "9087     9090                1\n",
       "8802     8804                1\n",
       "8371     8372                1"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratesPerBook = ratings[['book_id', 'rating']].groupby(['book_id'], as_index = False) \\\n",
    "                      .count() \\\n",
    "                      .rename(columns = {'rating' : 'receivedRatings'}) \\\n",
    "                      .sort_values('receivedRatings', ascending = False)\n",
    "\n",
    "ratesPerBook.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIoAAAFlCAYAAACEOwMFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3df7Cld10f8PfHLFBAGYJZaExCF22gRqYNuI20DJSKhRAtAVs0mVYi0ll1QpVa2wbbKVSbGWwFOrQ2TjSRMIVgNFBSiEpEKzrDrw3EEAjIAtEsSZO1UaClTQ18+sf5Xj3ZPffuj3v3Ps9mX6+ZM/ec73nOOe/747nfe973+VHdHQAAAAD4mqkDAAAAADAPiiIAAAAAkiiKAAAAABgURQAAAAAkURQBAAAAMCiKAAAAAEiS7Jg6wOGcdtppvWvXrqljAAAAADxk3HzzzX/U3TsPHp99UbRr167s3bt36hgAAAAADxlV9Qerxu16BgAAAEASRREAAAAAg6IIAAAAgCSKIgAAAAAGRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQJNkxdQAAAACA42HXZe/e0ue747XfuaXPN0e2KAIAAAAgiaIIAAAAgEFRBAAAAECSIyiKquqsqvqtqrq9qj5eVT86xh9XVTdV1afHx1PHeFXVG6tqX1XdWlVPX3quS8byn66qS47fpwUAAADA0TqSLYoeSPJPu/ubkzwjyaVVdU6Sy5K8t7vPTvLecTtJXpDk7HHZk+SKZFEsJXl1km9Lcl6SV6+VSwAAAABM77BFUXff3d0fGde/lOT2JGckuTDJNWOxa5K8aFy/MMmbe+EDSR5bVacneX6Sm7r7vu7+4yQ3JTl/Sz8bAAAAAI7ZUR2jqKp2JXlakg8meUJ3350syqQkjx+LnZHkzqWH7R9j642vep09VbW3qvYeOHDgaCICAAAAcIyOuCiqqq9Ncn2SV3b3FzdadMVYbzB+6GD3ld29u7t379y580gjAgAAALAJR1QUVdXDsiiJ3tLdbx/D94xdyjI+3jvG9yc5a+nhZya5a4NxAAAAAGbgSM56VkmuSnJ7d79+6a4bkqydueySJO9cGn/pOPvZM5J8Yeya9utJnldVp46DWD9vjAEAAAAwAzuOYJlnJvm+JB+rqlvG2E8keW2S66rq5Un+MMlLxn03Jrkgyb4kX07ysiTp7vuq6qeSfHgs95Pdfd+WfBYAAAAAbNphi6Lu/t2sPr5Qkjx3xfKd5NJ1nuvqJFcfTUAAAAAAtsdRnfUMAAAAgIcuRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQRFEEAAAAwKAoAgAAACCJoggAAACAQVEEAAAAQBJFEQAAAACDoggAAACAJIoiAAAAAAZFEQAAAABJFEUAAAAADIoiAAAAAJIoigAAAAAYFEUAAAAAJFEUAQAAADAoigAAAABIoigCAAAAYFAUAQAAAJBEUQQAAADAoCgCAAAAIImiCAAAAIDhsEVRVV1dVfdW1W1LY79UVbeMyx1VdcsY31VV/2fpvp9besy3VtXHqmpfVb2xqur4fEoAAAAAHIsdR7DMm5L8pyRvXhvo7u9du15Vr0vyhaXlP9Pd5654niuS7EnygSQ3Jjk/ya8efWQAAAAAjofDblHU3e9Lct+q+8ZWQd+T5NqNnqOqTk/ymO5+f3d3FqXTi44+LgAAAADHy2aPUfSsJPd096eXxp5UVR+tqt+uqmeNsTOS7F9aZv8YW6mq9lTV3qrae+DAgU1GBAAAAOBIbLYoujgP3pro7iRP7O6nJfmxJG+tqsckWXU8ol7vSbv7yu7e3d27d+7cucmIAAAAAByJIzlG0UpVtSPJdyf51rWx7r4/yf3j+s1V9ZkkT85iC6Izlx5+ZpK7jvW1AQAAANh6m9mi6DuSfLK7/2yXsqraWVWnjOvfmOTsJJ/t7ruTfKmqnjGOa/TSJO/cxGsDAAAAsMUOWxRV1bVJ3p/kKVW1v6pePu66KIcexPrZSW6tqt9L8itJfqi71w6E/cNJfiHJviSfiTOeAQAAAMzKYXc96+6L1xn//hVj1ye5fp3l9yZ56lHmAwAAAGCbbPZg1gAAAAA8RCiKAAAAAEiiKAIAAABgUBQBAAAAkERRBAAAAMCgKAIAAAAgiaIIAAAAgEFRBAAAAEASRREAAAAAg6IIAAAAgCSKIgAAAAAGRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQRFEEAAAAwKAoAgAAACCJoggAAACAQVEEAAAAQBJFEQAAAACDoggAAACAJIoiAAAAAAZFEQAAAABJFEUAAAAADIctiqrq6qq6t6puWxp7TVV9vqpuGZcLlu57VVXtq6pPVdXzl8bPH2P7quqyrf9UAAAAANiMI9mi6E1Jzl8x/obuPndcbkySqjonyUVJvmU85j9X1SlVdUqSn03ygiTnJLl4LAsAAADATOw43ALd/b6q2nWEz3dhkrd19/1JPldV+5KcN+7b192fTZKqettY9hNHnRgAAACA42Izxyh6RVXdOnZNO3WMnZHkzqVl9o+x9cZXqqo9VbW3qvYeOHBgExEBAAAAOFLHWhRdkeSbkpyb5O4krxvjtWLZ3mB8pe6+srt3d/funTt3HmNEAAAAAI7GYXc9W6W771m7XlU/n+Rd4+b+JGctLXpmkrvG9fXGAQAAAJiBY9qiqKpOX7r54iRrZ0S7IclFVfWIqnpSkrOTfCjJh5OcXVVPqqqHZ3HA6xuOPTYAAAAAW+2wWxRV1bVJnpPktKran+TVSZ5TVedmsfvYHUl+MEm6++NVdV0WB6l+IMml3f2V8TyvSPLrSU5JcnV3f3zLPxsAAAAAjtmRnPXs4hXDV22w/OVJLl8xfmOSG48qHQAAAADbZjNnPQMAAADgIURRBAAAAEASRREAAAAAg6IIAAAAgCSKIgAAAAAGRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQRFEEAAAAwKAoAgAAACCJoggAAACAQVEEAAAAQBJFEQAAAACDoggAAACAJIoiAAAAAAZFEQAAAABJFEUAAAAADIoiAAAAAJIoigAAAAAYFEUAAAAAJFEUAQAAADAoigAAAABIoigCAAAAYDhsUVRVV1fVvVV129LYv6+qT1bVrVX1jqp67BjfVVX/p6puGZefW3rMt1bVx6pqX1W9sarq+HxKAAAAAByLI9mi6E1Jzj9o7KYkT+3uv5rk95O8aum+z3T3uePyQ0vjVyTZk+TscTn4OQEAAACY0GGLou5+X5L7Dhp7T3c/MG5+IMmZGz1HVZ2e5DHd/f7u7iRvTvKiY4sMAAAAwPGwFcco+oEkv7p0+0lV9dGq+u2qetYYOyPJ/qVl9o+xlapqT1Xtraq9Bw4c2IKIAAAAABzOpoqiqvqXSR5I8pYxdHeSJ3b305L8WJK3VtVjkqw6HlGv97zdfWV37+7u3Tt37txMRAAAAACO0I5jfWBVXZLku5I8d+xOlu6+P8n94/rNVfWZJE/OYgui5d3Tzkxy17G+NgAAAABb75i2KKqq85P8iyQv7O4vL43vrKpTxvVvzOKg1Z/t7ruTfKmqnjHOdvbSJO/cdHoAAAAAtsxhtyiqqmuTPCfJaVW1P8mrszjL2SOS3DTOcv+BcYazZyf5yap6IMlXkvxQd68dCPuHsziD2iOzOKbR8nGNAAAAAJjYYYui7r54xfBV6yx7fZLr17lvb5KnHlU6AAAAALbNVpz1DAAAAICHAEURAAAAAEkURQAAAAAMiiIAAAAAkiiKAAAAABgURQAAAAAkURQBAAAAMCiKAAAAAEiiKAIAAABgUBQBAAAAkERRBAAAAMCgKAIAAAAgiaIIAAAAgEFRBAAAAEASRREAAAAAg6IIAAAAgCSKIgAAAAAGRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQRFEEAAAAwKAoAgAAACCJoggAAACA4YiKoqq6uqrurarblsYeV1U3VdWnx8dTx3hV1Rural9V3VpVT196zCVj+U9X1SVb/+kAAAAAcKyOdIuiNyU5/6Cxy5K8t7vPTvLecTtJXpDk7HHZk+SKZFEsJXl1km9Lcl6SV6+VSwAAAABM74iKou5+X5L7Dhq+MMk14/o1SV60NP7mXvhAksdW1elJnp/kpu6+r7v/OMlNObR8AgAAAGAimzlG0RO6++4kGR8fP8bPSHLn0nL7x9h644eoqj1Vtbeq9h44cGATEQEAAAA4UsfjYNa1Yqw3GD90sPvK7t7d3bt37ty5peEAAAAAWG0zRdE9Y5eyjI/3jvH9Sc5aWu7MJHdtMA4AAADADGymKLohydqZyy5J8s6l8ZeOs589I8kXxq5pv57keVV16jiI9fPGGAAAAAAzsONIFqqqa5M8J8lpVbU/i7OXvTbJdVX18iR/mOQlY/Ebk1yQZF+SLyd5WZJ0931V9VNJPjyW+8nuPvgA2QAAAABM5IiKou6+eJ27nrti2U5y6TrPc3WSq484HQAAAADb5ngczBoAAACAE5CiCAAAAIAkiiIAAAAABkURAAAAAEkURQAAAAAMiiIAAAAAkiiKAAAAABgURQAAAAAkURQBAAAAMCiKAAAAAEiiKAIAAABgUBQBAAAAkERRBAAAAMCgKAIAAAAgiaIIAAAAgEFRBAAAAEASRREAAAAAg6IIAAAAgCSKIgAAAAAGRREAAAAASRRFAAAAAAyKIgAAAACSKIoAAAAAGBRFAAAAACRRFAEAAAAwKIoAAAAASKIoAgAAAGBQFAEAAACQZBNFUVU9papuWbp8sapeWVWvqarPL41fsPSYV1XVvqr6VFU9f2s+BQAAAAC2wo5jfWB3fyrJuUlSVack+XySdyR5WZI3dPfPLC9fVeckuSjJtyT5hiS/UVVP7u6vHGsGAAAAALbOVu169twkn+nuP9hgmQuTvK277+/uzyXZl+S8LXp9AAAAADZpq4qii5Jcu3T7FVV1a1VdXVWnjrEzkty5tMz+MQYAAADADGy6KKqqhyd5YZJfHkNXJPmmLHZLuzvJ69YWXfHwXuc591TV3qrae+DAgc1GBAAAAOAIbMUWRS9I8pHuvidJuvue7v5Kd381yc/nz3cv25/krKXHnZnkrlVP2N1Xdvfu7t69c+fOLYgIAAAAwOFsRVF0cZZ2O6uq05fue3GS28b1G5JcVFWPqKonJTk7yYe24PUBAAAA2ALHfNazJKmqRyX5O0l+cGn431XVuVnsVnbH2n3d/fGqui7JJ5I8kORSZzwDAAAAmI9NFUXd/eUkX3/Q2PdtsPzlSS7fzGsCAAAAcHxs1VnPAAAAADjBKYoAAAAASKIoAgAAAGBQFAEAAACQRFEEAAAAwKAoAgAAACCJoggAAACAQVEEAAAAQBJFEQAAAACDoggAAACAJIoiAAAAAAZFEQAAAABJFEUAAAAADIoiAAAAAJIoigAAAAAYFEUAAAAAJEl2TB0AAAAAYNdl7546ArFFEQAAAACDoggAAACAJIoiAAAAAAZFEQAAAABJFEUAAAAADIoiAAAAAJIoigAAAAAYFEUAAAAAJFEUAQAAADAoigAAAABIoigCAAAAYFAUAQAAAJBkC4qiqrqjqj5WVbdU1d4x9riquqmqPj0+njrGq6reWFX7qurWqnr6Zl8fAAAAgK2xVVsU/e3uPre7d4/blyV5b3efneS943aSvCDJ2eOyJ8kVW/T6AAAAAGzS8dr17MIk14zr1yR50dL4m3vhA0keW1WnH6cMAAAAAByFrSiKOsl7qurmqtozxp7Q3Xcnyfj4+DF+RpI7lx67f4wBAAAAMLEdW/Acz+zuu6rq8UluqqpPbrBsrRjrQxZaFE57kuSJT3ziFkQEAAAA4HA2vUVRd981Pt6b5B1Jzktyz9ouZePjvWPx/UnOWnr4mUnuWvGcV3b37u7evXPnzs1GBAAAAOAIbKooqqpHV9XXrV1P8rwktyW5IcklY7FLkrxzXL8hyUvH2c+ekeQLa7uoAQAAADCtze569oQk76iqted6a3f/WlV9OMl1VfXyJH+Y5CVj+RuTXJBkX5IvJ3nZJl8fAAAAgC2yqaKouz+b5K+tGP+fSZ67YryTXLqZ1wQAAADg+NiKs54BAAAA8BCgKAIAAAAgiaIIAAAAgEFRBAAAAEASRREAAAAAg6IIAAAAgCSKIgAAAACGHVMHAAAAAE48uy5799QROA5sUQQAAABAEkURAAAAAIOiCAAAAIAkiiIAAAAABkURAAAAAEkURQAAAAAMiiIAAAAAkiiKAAAAABgURQAAAAAkSXZMHQAAAAA4/nZd9u6pI3ACsEURAAAAAEkURQAAAAAMiiIAAAAAkiiKAAAAABgURQAAAAAkURQBAAAAMCiKAAAAAEiiKAIAAABg2DF1AAAAAODBdl327qkjcJKyRREAAAAASRRFAAAAAAzHXBRV1VlV9VtVdXtVfbyqfnSMv6aqPl9Vt4zLBUuPeVVV7auqT1XV87fiEwAAAABga2zmGEUPJPmn3f2Rqvq6JDdX1U3jvjd0988sL1xV5yS5KMm3JPmGJL9RVU/u7q9sIgMAAAAAW+SYtyjq7ru7+yPj+peS3J7kjA0ecmGSt3X3/d39uST7kpx3rK8PAAAAwNbakmMUVdWuJE9L8sEx9IqqurWqrq6qU8fYGUnuXHrY/mxcLAEAAACwjTZdFFXV1ya5Pskru/uLSa5I8k1Jzk1yd5LXrS264uG9znPuqaq9VbX3wIEDm40IAAAAwBHYzDGKUlUPy6Ikekt3vz1Juvuepft/Psm7xs39Sc5aeviZSe5a9bzdfWWSK5Nk9+7dK8skAAAAmItdl7176giwJY65KKqqSnJVktu7+/VL46d3993j5ouT3Dau35DkrVX1+iwOZn12kg8d6+sDAADAsVLswGqb2aLomUm+L8nHquqWMfYTSS6uqnOz2K3sjiQ/mCTd/fGqui7JJ7I4Y9qlzngGAADAkVDswPY45qKou383q487dOMGj7k8yeXH+poAAAAAHD+bOkYRAAAAHMzWP3Di2vRZzwAAAAB4aLBFEQAAwAlmq7fYueO137mlzwecuGxRBAAAAEASRREAAAAAg13PAIATgt0sgBPViXBg5xMhI7A9FEUAwHF5g6CI4VicjIXgyfg5AzBfiiIAOAH5z+88ecPPHJyMP4cn4+cMcLwoigBgGyh2YBoKhM3zNQQ4uSiKAICT0olQ3p0IGeFo+bkGmDdFEQAnPMfXAQCAraEoAjhKNsE/OfiP9+b5GvJQ5OcagIc6RREAAMAShSBwMlMUATzE2A0LAAA4VooiAA7L7nYAAHByUBQBs2JrmJODTfoBAGCeFEVwkjkZtwyZeykx93wAAMDJQ1EEbIqSAwAA4KHja6YOAAAAAMA8KIoAAAAASGLXM5g1u3UBAACwnRRFsIUUOwAAAJzI7HoGAAAAQBJbFHGSswUQAAAA/DlbFAEAAACQRFEEAAAAwKAoAgAAACCJYxRxHDn+DwAAAJxYFEX8GcUOAAAAnNy2fdezqjq/qj5VVfuq6rLtfn0AAAAAVtvWoqiqTknys0lekOScJBdX1TnbmQEAAACA1bZ717Pzkuzr7s8mSVW9LcmFST6xzTm2nd26AAAAgLnb7l3Pzkhy59Lt/WMMAAAAgIlt9xZFtWKsD1moak+SPePm/6qqTx3XVNvjtCR/NHWIw5h7xrnnS2TcCnPPl8w/49zzJTJuhbnnS+afce75kvlnnHu+RMatMPd8yfwzzj1fIuNWmHu+ZP4Z554v9dPzz3gU/tKqwe0uivYnOWvp9plJ7jp4oe6+MsmV2xVqO1TV3u7ePXWOjcw949zzJTJuhbnnS+afce75Ehm3wtzzJfPPOPd8yfwzzj1fIuNWmHu+ZP4Z554vkXErzD1fMv+Mc8+XnBgZN2u7dz37cJKzq+pJVfXwJBcluWGbMwAAAACwwrZuUdTdD1TVK5L8epJTklzd3R/fzgwAAAAArLbdu56lu29McuN2v+4MnAi70s0949zzJTJuhbnnS+afce75Ehm3wtzzJfPPOPd8yfwzzj1fIuNWmHu+ZP4Z554vkXErzD1fMv+Mc8+XnBgZN6W6DzmWNAAAAAAnoe0+RhEAAAAAM6UoOs6q6uqqureqbps6y0aq6o6q+lhV3VJVe6fOc7Cq+tGquq2qPl5Vr5w6T7L6e1tVP1VVt46v43uq6htmlu+XRrZbxvf8lqnyjTxnVdVvVdXt43v7o2P8JeP2V6tqsjMKbJDvNVX1+aWv5QUzzPjXqur9Y73+b1X1mIny/YWq+lBV/d7I92/G+FVj7Naq+pWq+top8h0m4+8sfY/vqqr/OlXGkeeUqvpoVb1r3H5FVe2rqq6q06bMtmZFxjdV1eeWvo7nzizft1fVR8b8ck1Vbfsu+QflO2QuntO8skHG2cwtVfXY8Tvlk+P34t+Yy5xymIyzmVc2yDiXeeUpS1+nW6rqi1X1yjmtKxtknNO68k/GenFbVV075sJZzSvrZJzNvLJOvrnNK4e8h5rTurJBxsnWlVr9HupxVXVTVX16fDx1jP+V8Xvx/qr68e3KuC262+U4XpI8O8nTk9w2dZbD5LwjyWlT51gn21OT3JbkUVkcV+s3kpw9g1yHfG+TPGbp+o8k+bk55Tvo/tcl+dcTfw1PT/L0cf3rkvx+knOSfHOSpyT570l2zzDfa5L8+JRfuyPI+OEkf2uM/0CSn5ooXyX52nH9YUk+mOQZB60rr09y2YRfw5UZD1rm+iQvnfh7/WNJ3prkXeP205LsmtPv7xUZ35Tk70+da1W+LP5ZdmeSJ4/7fjLJyyfOd8j3ck7zynoZD7p/0rklyTVJ/tG4/vAkj53LnHKYjLOZVzbIOIt55aCcpyT5H0n+0tzWlVUZDxqfbF1JckaSzyV55Lh9XZLvn9O8skHGWcwr6+T7gTnNK1nnPdSc1pX1Mh60zLauK1n9Hu/fZfytmuSyJD89rj8+yV9PcvmcfodvxcUWRcdZd78vyX1T5zjBfXOSD3T3l7v7gSS/neTFE2da+b3t7i8u3Xx0kskOArbRz15VVZLvSXLttoY6SHff3d0fGde/lOT2JGd09+3d/akps41MK/NNm+rBNsj4lCTvG4vdlOTvTZSvu/t/jZsPG5deW1fGz+IjM+26sjLj2v1V9XVJvj3JZFsUVdWZSb4zyS+sjXX3R7v7jqkyHWxVxjlZke/rk9zf3b8/bk+2nmxkTvPK4Uw9t4wtXJ6d5Kok6e7/191/Mpc5JVk/47SpHmyDjLOYVw7y3CSf6e4/mPG68mcZ1wamXleGHUkeObZ4eVSSu+Y2r2RFxonzHOzgfP8785pXVr6Hmtm6suH7vCnWlXXeQ12YRYGe8fFFY9l7u/vDSf50u/JtF0URazrJe6rq5qraM3WYg9yW5NlV9fVV9agkFyQ5a+JM66qqy6vqziT/IMm/njrPOp6V5J7u/vTUQdZU1a4s/pP1wWmTrLYi3yvGZrtXr21+OrWDMt6W5IXjrpdkwnWmFrv73JLk3iQ3dfcHx/gvZvFf1r+S5D9OlW9kWZlxeHGS9x70h9V2+w9J/nmSr06Y4XDWy3j5WFfeUFWPmCDXmoPz/VGSh9Wf74r09zP93LJyLp7ZvLLR3wtTzy3fmORAkl+sxS6Gv1BVj54oy3o2yjiXeWW9jLOZV5ZclKU3kDNbV9Y8KOMw6brS3Z9P8jNJ/jDJ3Um+0N3vmSLLeg6TcfJ5ZVW+LLYqmtO8su57qBmtK4d7nzf1vLLmCd19d7L4J20WWxI9pCmKWPPM7n56khckubSqnj11oDXdfXuSn86ilf+1JL+X5IFJQ22gu/9ld5+V5C1JXjF1nnVcnIm3JlpWi+PTXJ/klRO/GV9pRb4rknxTknOz+OPgdRPGS7Iy4w9ksS7fnMUuaf9vqmzd/ZXuPjfJmUnOq6qnjvGXJfmGLLaC+t6p8o0sKzMOk64vVfVdSe7t7punynA4G2R8VRZF4F9P8rgk/2K7syWr83V3Z/EG7g1V9aEkX8r0c8vKuXhm88pGfy9MPbfsyGJ3gSu6+2lZ/Hf/sgnzrLJexjnNK+tlnM28kiRV9fAsiqtfXhub2bqyMuMw9bxyahZbSDwpi3n40VX1D6fKs8oGGecyrxySL4vSZTbzykbvoeayrhzB+7yp55WTlqKIJEl33zU+3pvkHUnOmzbRg3X3Vd399O5+dhabAk7dKh+Jt2Yem2U/yNg89ruT/NLUWZKkqh6WRcHxlu5++9R5DrYqX3ffM4qFryb5+Uy8vqyT8ZPd/bzu/tYsJtjPTJlxZPqTLI4Rcv7S2Fey+FmcxbpycMaq+vosvr/vnjDWM5O8sKruSPK2JN9eVf9lwjyrrMw4do3s7r4/yS9munVlvXzv7+5ndfd5WexSM+nccgRz8eTzynoZZzK37E+yf2mLwF/JovCYk5UZZzavrJdxbvPKC5J8pLvvWXHf5OvKcEjGmawr35Hkc919oLv/NMnbk/zNCfOssjLjjOaV9fLNbV453HuoydeV9TLOZF1Zc09VnZ4k4+O9E+c57hRFpKoePY7BkbFp8fOy2AxwNqrq8ePjE7P4hTHLZrmqzl66+cIkn5wqywa+I8knu3v/1EHGfsdXJbm9u18/dZ6DrZdvbaIYXpwJ15cNMq6tM1+T5F8l+bmJ8u2sqseO64/M4ufvU1X1l8dYJfm7mXBdWSfjWp6XZHFg5v87Vb7uflV3n9ndu7L4T+Vvdves/vO7XsalP6oqi/35J1lXNsi3tp48Iov/Sk+ynowMK+fiOc0rh/l7YfK5pbv/R5I7q+opY+i5ST4xVZ5V1ss4p3llg4yzmFeWPGhLgzmtK0tWbQ0x+bqSxXS31y0AAAIDSURBVO5Sz6iqR43fz8/NYuveOVmZcS7zygb5ZjOvjByHvIea27qywfu8Oawra25Icsm4fkmSd06YZVtMerq+k0FVXZvkOUlOq6r9SV7d3VdNm+oQT0jyjsXvuOxI8tbu/rVpIx3i+vGf/T9Ncml3//HUgVZ9b5NcMP6w+mqSP0jyQ3PKN372Vu0rP5VnJvm+JB+rPz/t5U8keUQWx6zZmeTdVXVLdz9/RvkursXpWDuLM4P84ATZ1qyX8eyqunTcfnsW/3WbwulJrqmqU7L458R1WWyd8zu1OGBqZbGZ8Q9PlC9ZkbG73zXuuyjJaydLtoGq+pEsjrnzF5PcWlU3dvc/mjjWwd5SVTuz+D7fkgl/J67jn43d0r4mi91sfnPCLCvn4qq6fi7zynoZx31zmVv+cRY/dw9P8tkkL6uqF2cec8q6GZO8cUbzSrI640tnMq+kFscy+Tt58NfptTNaV9bLmMxgXenuD1bVryT5SBa7+Xw0yZVzmlfWy5jkV+cwr2yQ79/OaF5JVryHqsVxx2azrmT993mTrCvrvMd7bZLrqurlWZSELxnL/sUke5M8JslXq+qVSc7pGR5K42jVYhd9AAAAAE52dj0DAAAAIImiCAAAAIBBUQQAAABAEkURAAAAAIOiCAAAAIAkiiIAAAAABkURAAAAAEkURQAAAAAM/x+hAMAh/AdDGwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams['figure.figsize'] = [20, 6]\n",
    "plt.hist(ratesPerBook['receivedRatings'], bins = 50)\n",
    "plt.xticks(np.arange(min(ratesPerBook['receivedRatings']), max(ratesPerBook['receivedRatings'])+1, 4.0))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = pd.merge(ratings, ratesPerBook, on = ['book_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21710"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Keeping only books with more than a 'minimum' of ratings received\n",
    "\n",
    "minimumReceived = ratings.receivedRatings < 30\n",
    "np.sum(minimumReceived)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "685744"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings = ratings[-minimumReceived]\n",
    "len(ratings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13971"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.user_id.unique())   # total of remaining users after removing the ones with less than the established minimum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8885"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ratings.book_id.unique())    # total of remaining books after removing the ones with less than the established minimum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(685744, 5)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To start off, I have tried to calculate a first type of Top-N recommendation: the top rated books. \n",
    "After having removed the books and users above, the result doesn't change much compared to the what shown right after the  data loading: Bill Watterson and his Calvin and Hobbes is still dominating the chart."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For illustrative purpose, let's first grab author and title for the books dataset:\n",
    "showingTitles = pd.merge(ratings, books[['id', 'authors', 'title']], how = 'left', left_on = ['book_id'], right_on = ['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6631</th>\n",
       "      <td>6920</td>\n",
       "      <td>4.806452</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Indispensable Calvin and Hobbes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8622</th>\n",
       "      <td>9566</td>\n",
       "      <td>4.800000</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>Attack of the Deranged Mutant Killer Monster S...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5094</th>\n",
       "      <td>5207</td>\n",
       "      <td>4.774648</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Days Are Just Packed: A Calvin and Hobbes ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8237</th>\n",
       "      <td>8978</td>\n",
       "      <td>4.767857</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Revenge of the Baby-Sat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4061</th>\n",
       "      <td>4107</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>Andrew Williamson, Andrew Williamson</td>\n",
       "      <td>Harry Potter: A Pop-Up Book: Based on the Film...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6341</th>\n",
       "      <td>6590</td>\n",
       "      <td>4.750000</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Authoritative Calvin and Hobbes: A Calvin ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7602</th>\n",
       "      <td>8109</td>\n",
       "      <td>4.745763</td>\n",
       "      <td>Neil Gaiman, Mike Dringenberg, Chris Bachalo, ...</td>\n",
       "      <td>The Absolute Sandman, Volume One</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6142</th>\n",
       "      <td>6361</td>\n",
       "      <td>4.738095</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>There's Treasure Everywhere: A Calvin and Hobb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4422</th>\n",
       "      <td>4483</td>\n",
       "      <td>4.738095</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>It's a Magical World: A Calvin and Hobbes Coll...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3596</th>\n",
       "      <td>3628</td>\n",
       "      <td>4.734694</td>\n",
       "      <td>Bill Watterson</td>\n",
       "      <td>The Complete Calvin and Hobbes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      book_id    rating                                            authors  \\\n",
       "6631     6920  4.806452                                     Bill Watterson   \n",
       "8622     9566  4.800000                                     Bill Watterson   \n",
       "5094     5207  4.774648                                     Bill Watterson   \n",
       "8237     8978  4.767857                                     Bill Watterson   \n",
       "4061     4107  4.750000               Andrew Williamson, Andrew Williamson   \n",
       "6341     6590  4.750000                                     Bill Watterson   \n",
       "7602     8109  4.745763  Neil Gaiman, Mike Dringenberg, Chris Bachalo, ...   \n",
       "6142     6361  4.738095                                     Bill Watterson   \n",
       "4422     4483  4.738095                                     Bill Watterson   \n",
       "3596     3628  4.734694                                     Bill Watterson   \n",
       "\n",
       "                                                  title  \n",
       "6631                The Indispensable Calvin and Hobbes  \n",
       "8622  Attack of the Deranged Mutant Killer Monster S...  \n",
       "5094  The Days Are Just Packed: A Calvin and Hobbes ...  \n",
       "8237                        The Revenge of the Baby-Sat  \n",
       "4061  Harry Potter: A Pop-Up Book: Based on the Film...  \n",
       "6341  The Authoritative Calvin and Hobbes: A Calvin ...  \n",
       "7602                   The Absolute Sandman, Volume One  \n",
       "6142  There's Treasure Everywhere: A Calvin and Hobb...  \n",
       "4422  It's a Magical World: A Calvin and Hobbes Coll...  \n",
       "3596                     The Complete Calvin and Hobbes  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topRated = showingTitles.groupby(['book_id'], as_index = False) \\\n",
    "                      .agg({'rating' : 'mean', \\\n",
    "                           'authors': 'first', \\\n",
    "                           'title' : 'first'})\n",
    "topRated.sort_values('rating', ascending = False).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Content-Based Filtering (Tag-Based)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_id</th>\n",
       "      <th>count</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>30574</td>\n",
       "      <td>167697</td>\n",
       "      <td>to-read</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258362</th>\n",
       "      <td>1</td>\n",
       "      <td>33165</td>\n",
       "      <td>338</td>\n",
       "      <td>youth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250046</th>\n",
       "      <td>1</td>\n",
       "      <td>17213</td>\n",
       "      <td>347</td>\n",
       "      <td>kindle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246919</th>\n",
       "      <td>1</td>\n",
       "      <td>27535</td>\n",
       "      <td>348</td>\n",
       "      <td>shelfari-favorites</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245888</th>\n",
       "      <td>1</td>\n",
       "      <td>16799</td>\n",
       "      <td>351</td>\n",
       "      <td>juvenile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245375</th>\n",
       "      <td>1</td>\n",
       "      <td>18851</td>\n",
       "      <td>354</td>\n",
       "      <td>made-me-cry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240136</th>\n",
       "      <td>1</td>\n",
       "      <td>25151</td>\n",
       "      <td>364</td>\n",
       "      <td>read-in-2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>238876</th>\n",
       "      <td>1</td>\n",
       "      <td>11550</td>\n",
       "      <td>368</td>\n",
       "      <td>favorite-series</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237950</th>\n",
       "      <td>1</td>\n",
       "      <td>11491</td>\n",
       "      <td>384</td>\n",
       "      <td>faves</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237815</th>\n",
       "      <td>1</td>\n",
       "      <td>32654</td>\n",
       "      <td>386</td>\n",
       "      <td>wizards</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236195</th>\n",
       "      <td>1</td>\n",
       "      <td>26897</td>\n",
       "      <td>392</td>\n",
       "      <td>scifi-fantasy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235846</th>\n",
       "      <td>1</td>\n",
       "      <td>615</td>\n",
       "      <td>399</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>231762</th>\n",
       "      <td>1</td>\n",
       "      <td>21989</td>\n",
       "      <td>410</td>\n",
       "      <td>novel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>230408</th>\n",
       "      <td>1</td>\n",
       "      <td>33124</td>\n",
       "      <td>416</td>\n",
       "      <td>young-adult-fiction</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223860</th>\n",
       "      <td>1</td>\n",
       "      <td>3371</td>\n",
       "      <td>433</td>\n",
       "      <td>audio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214445</th>\n",
       "      <td>1</td>\n",
       "      <td>18045</td>\n",
       "      <td>440</td>\n",
       "      <td>library</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213432</th>\n",
       "      <td>1</td>\n",
       "      <td>6960</td>\n",
       "      <td>445</td>\n",
       "      <td>childrens-books</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207918</th>\n",
       "      <td>1</td>\n",
       "      <td>22689</td>\n",
       "      <td>446</td>\n",
       "      <td>own-it</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206741</th>\n",
       "      <td>1</td>\n",
       "      <td>20824</td>\n",
       "      <td>449</td>\n",
       "      <td>my-favorites</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>206725</th>\n",
       "      <td>1</td>\n",
       "      <td>14024</td>\n",
       "      <td>450</td>\n",
       "      <td>harry-potter-series</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        goodreads_book_id  tag_id   count             tag_name\n",
       "0                       1   30574  167697              to-read\n",
       "258362                  1   33165     338                youth\n",
       "250046                  1   17213     347               kindle\n",
       "246919                  1   27535     348   shelfari-favorites\n",
       "245888                  1   16799     351             juvenile\n",
       "245375                  1   18851     354          made-me-cry\n",
       "240136                  1   25151     364         read-in-2015\n",
       "238876                  1   11550     368      favorite-series\n",
       "237950                  1   11491     384                faves\n",
       "237815                  1   32654     386              wizards\n",
       "236195                  1   26897     392        scifi-fantasy\n",
       "235846                  1     615     399                 2005\n",
       "231762                  1   21989     410                novel\n",
       "230408                  1   33124     416  young-adult-fiction\n",
       "223860                  1    3371     433                audio\n",
       "214445                  1   18045     440              library\n",
       "213432                  1    6960     445      childrens-books\n",
       "207918                  1   22689     446               own-it\n",
       "206741                  1   20824     449         my-favorites\n",
       "206725                  1   14024     450  harry-potter-series"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showingTagName = pd.merge(bookTags, tags, on = 'tag_id')\n",
    "showingTagName.sort_values('goodreads_book_id').head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag_name</th>\n",
       "      <th>number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30574</th>\n",
       "      <td>to-read</td>\n",
       "      <td>9983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11557</th>\n",
       "      <td>favorites</td>\n",
       "      <td>9881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22743</th>\n",
       "      <td>owned</td>\n",
       "      <td>9858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5207</th>\n",
       "      <td>books-i-own</td>\n",
       "      <td>9799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8717</th>\n",
       "      <td>currently-reading</td>\n",
       "      <td>9776</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                tag_name  number\n",
       "30574            to-read    9983\n",
       "11557          favorites    9881\n",
       "22743              owned    9858\n",
       "5207         books-i-own    9799\n",
       "8717   currently-reading    9776"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mostUsedTags = showingTagName.groupby(['tag_name'], as_index = False) \\\n",
    "                      .agg({'goodreads_book_id' : 'count'}) \\\n",
    "                      .rename(columns = {'goodreads_book_id' : 'number'}) \\\n",
    "                      .sort_values('number', ascending = False)\n",
    "mostUsedTags.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I have used the genres.csv files, which was generated by scraping Goodreads website, and it is shown in the other notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>art</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>biography</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>children-s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>christian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>classics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>comics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>cookbooks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>ebooks</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>fantasy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    tag_name\n",
       "0           0         art\n",
       "1           1   biography\n",
       "2           2    business\n",
       "3           3  children-s\n",
       "4           4   christian\n",
       "5           5    classics\n",
       "6           6      comics\n",
       "7           7   cookbooks\n",
       "8           8      ebooks\n",
       "9           9     fantasy"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genres = pd.read_csv('genres.csv')\n",
    "genres.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1242, 2)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genres.shape    # 1235 rows X 2 columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First I convert the genres to a list\n",
    "genreList = genres['tag_name'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1242"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(genreList)    # 1235 genre-tags have been scraped from Goodreads website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "34252"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tags.tag_name.unique())   # the original tag dataset included 34.252 tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "835"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genreTags = tags.loc[tags['tag_name'].isin(genreList)]\n",
    "len(genreTags)    # 832 tags (of out the 1235) scraped from the Genre Section in Goodreads website are included in the original tags table."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then I have merged the bookTags dataset with the genreTags, so that now I have the information of what books in the dataset have been tagged with at least one of the 832 genre-tags used by Goodreads in their genres section.\n",
    "\n",
    "The idea is then to measure the similarity between each book pair, so that the system will be able to recommend Top-N similar books for any selected title from the ones included in the analysis.\n",
    "In order to be able to properly perform content based filtering, first I have to put all the tags related to one single book in a single string: each tag must be separated from each other with a space, to be accounted as a single word.\n",
    "Then I will perform pairwise similarity scores between books."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "mostCommonTags = pd.merge(bookTags, genreTags, on = ['tag_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "stringedTags = mostCommonTags.groupby('goodreads_book_id')['tag_name'].apply(lambda x: \"%s\" % ' '.join(x)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>fantasy young-adult fiction magic children-s n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>fantasy young-adult fiction childrens adventur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>fantasy young-adult fiction adventure classics...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>fantasy young-adult fiction adventure classics...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>21</td>\n",
       "      <td>fiction humor funny adult space science ebooks...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>24</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>25</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>26</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>27</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>28</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>29</td>\n",
       "      <td>contemporary humor funny comedy adult literatu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>30</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>33</td>\n",
       "      <td>fantasy young-adult fiction magic adventure cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>34</td>\n",
       "      <td>fantasy young-adult fiction magic adventure cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>36</td>\n",
       "      <td>fantasy fiction science-fiction-fantasy high-f...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    goodreads_book_id                                           tag_name\n",
       "0                   1  fantasy young-adult fiction magic childrens ad...\n",
       "1                   2  fantasy young-adult fiction magic children-s n...\n",
       "2                   3  fantasy young-adult fiction magic childrens ad...\n",
       "3                   5  fantasy young-adult fiction magic childrens ad...\n",
       "4                   6  fantasy young-adult fiction childrens adventur...\n",
       "5                   8  fantasy young-adult fiction magic childrens ad...\n",
       "6                  10  fantasy young-adult fiction magic childrens ad...\n",
       "7                  11  fantasy young-adult fiction adventure classics...\n",
       "8                  13  fantasy young-adult fiction adventure classics...\n",
       "9                  21  fiction humor funny adult space science ebooks...\n",
       "10                 24  fiction adventure contemporary humor funny com...\n",
       "11                 25  fiction adventure contemporary humor funny com...\n",
       "12                 26  fiction adventure contemporary humor funny com...\n",
       "13                 27  fiction adventure contemporary humor funny com...\n",
       "14                 28  fiction adventure contemporary humor funny com...\n",
       "15                 29  contemporary humor funny comedy adult literatu...\n",
       "16                 30  fantasy young-adult fiction magic childrens ad...\n",
       "17                 33  fantasy young-adult fiction magic adventure cl...\n",
       "18                 34  fantasy young-adult fiction magic adventure cl...\n",
       "19                 36  fantasy fiction science-fiction-fantasy high-f..."
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stringedTags.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I go retrieve the author and book information from the book dataset, so that I will be able to actually see\n",
    "# the title of the book instead of just seeing the id.\n",
    "\n",
    "stringedTags = pd.merge(stringedTags, books[['book_id', 'authors', 'title']], left_on = ['goodreads_book_id'], \\\n",
    "                       right_on = ['book_id']).drop('book_id', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>fantasy young-adult fiction magic children-s n...</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter and the Order of the Phoenix (Har...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (Harry P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré, Rufus Beck</td>\n",
       "      <td>Harry Potter and the Prisoner of Azkaban (Harr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>fantasy young-adult fiction childrens adventur...</td>\n",
       "      <td>J.K. Rowling, Mary GrandPré</td>\n",
       "      <td>Harry Potter and the Goblet of Fire (Harry Pot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id                                           tag_name  \\\n",
       "0                  1  fantasy young-adult fiction magic childrens ad...   \n",
       "1                  2  fantasy young-adult fiction magic children-s n...   \n",
       "2                  3  fantasy young-adult fiction magic childrens ad...   \n",
       "3                  5  fantasy young-adult fiction magic childrens ad...   \n",
       "4                  6  fantasy young-adult fiction childrens adventur...   \n",
       "\n",
       "                                   authors  \\\n",
       "0              J.K. Rowling, Mary GrandPré   \n",
       "1              J.K. Rowling, Mary GrandPré   \n",
       "2              J.K. Rowling, Mary GrandPré   \n",
       "3  J.K. Rowling, Mary GrandPré, Rufus Beck   \n",
       "4              J.K. Rowling, Mary GrandPré   \n",
       "\n",
       "                                               title  \n",
       "0  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "1  Harry Potter and the Order of the Phoenix (Har...  \n",
       "2  Harry Potter and the Sorcerer's Stone (Harry P...  \n",
       "3  Harry Potter and the Prisoner of Azkaban (Harr...  \n",
       "4  Harry Potter and the Goblet of Fire (Harry Pot...  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stringedTags.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Other than the genre-tags,a very important information that we can add to the stringed tag of each book is the author. Including the author will produce a higher similarity score between 2 books written by the same author, which seems reasonable a as someone who likes a book from an author is most likely going to like also other books from the same person."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First of all, I put everything in lowercase and I remove the space between the name and the surname, to make it count\n",
    "# as one single word all the time.\n",
    "stringedTags['authors'] = stringedTags['authors'].astype('str').apply(lambda x: str.lower(x.replace(\" \", \"\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling,marygrandpré</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>fantasy young-adult fiction magic children-s n...</td>\n",
       "      <td>j.k.rowling,marygrandpré</td>\n",
       "      <td>Harry Potter and the Order of the Phoenix (Har...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling,marygrandpré</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (Harry P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling,marygrandpré,rufusbeck</td>\n",
       "      <td>Harry Potter and the Prisoner of Azkaban (Harr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>fantasy young-adult fiction childrens adventur...</td>\n",
       "      <td>j.k.rowling,marygrandpré</td>\n",
       "      <td>Harry Potter and the Goblet of Fire (Harry Pot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   goodreads_book_id                                           tag_name  \\\n",
       "0                  1  fantasy young-adult fiction magic childrens ad...   \n",
       "1                  2  fantasy young-adult fiction magic children-s n...   \n",
       "2                  3  fantasy young-adult fiction magic childrens ad...   \n",
       "3                  5  fantasy young-adult fiction magic childrens ad...   \n",
       "4                  6  fantasy young-adult fiction childrens adventur...   \n",
       "\n",
       "                              authors  \\\n",
       "0            j.k.rowling,marygrandpré   \n",
       "1            j.k.rowling,marygrandpré   \n",
       "2            j.k.rowling,marygrandpré   \n",
       "3  j.k.rowling,marygrandpré,rufusbeck   \n",
       "4            j.k.rowling,marygrandpré   \n",
       "\n",
       "                                               title  \n",
       "0  Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "1  Harry Potter and the Order of the Phoenix (Har...  \n",
       "2  Harry Potter and the Sorcerer's Stone (Harry P...  \n",
       "3  Harry Potter and the Prisoner of Azkaban (Harr...  \n",
       "4  Harry Potter and the Goblet of Fire (Harry Pot...  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stringedTags.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# With the same operation, I remove the comma between the names as well\n",
    "stringedTags['authors'] = stringedTags['authors'].astype('str').apply(lambda x: str.lower(x.replace(\",\", \" \")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>goodreads_book_id</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>authors</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling marygrandpré</td>\n",
       "      <td>Harry Potter and the Half-Blood Prince (Harry ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>fantasy young-adult fiction magic children-s n...</td>\n",
       "      <td>j.k.rowling marygrandpré</td>\n",
       "      <td>Harry Potter and the Order of the Phoenix (Har...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling marygrandpré</td>\n",
       "      <td>Harry Potter and the Sorcerer's Stone (Harry P...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling marygrandpré rufusbeck</td>\n",
       "      <td>Harry Potter and the Prisoner of Azkaban (Harr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>fantasy young-adult fiction childrens adventur...</td>\n",
       "      <td>j.k.rowling marygrandpré</td>\n",
       "      <td>Harry Potter and the Goblet of Fire (Harry Pot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling marygrandpré</td>\n",
       "      <td>Harry Potter Boxed Set, Books 1-5 (Harry Potte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.k.rowling</td>\n",
       "      <td>Harry Potter Collection (Harry Potter, #1-6)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>fantasy young-adult fiction adventure classics...</td>\n",
       "      <td>douglasadams</td>\n",
       "      <td>The Hitchhiker's Guide to the Galaxy (Hitchhik...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>13</td>\n",
       "      <td>fantasy young-adult fiction adventure classics...</td>\n",
       "      <td>douglasadams</td>\n",
       "      <td>The Ultimate Hitchhiker's Guide to the Galaxy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>21</td>\n",
       "      <td>fiction humor funny adult space science ebooks...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>A Short History of Nearly Everything</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>24</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>In a Sunburned Country</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>25</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>I'm a Stranger Here Myself: Notes on Returning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>26</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>The Lost Continent: Travels in Small Town America</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>27</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>Neither Here nor There: Travels in Europe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>28</td>\n",
       "      <td>fiction adventure contemporary humor funny com...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>Notes from a Small Island</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>29</td>\n",
       "      <td>contemporary humor funny comedy adult literatu...</td>\n",
       "      <td>billbryson</td>\n",
       "      <td>The Mother Tongue: English and How It Got That...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>30</td>\n",
       "      <td>fantasy young-adult fiction magic childrens ad...</td>\n",
       "      <td>j.r.r.tolkien</td>\n",
       "      <td>J.R.R. Tolkien 4-Book Boxed Set: The Hobbit an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>33</td>\n",
       "      <td>fantasy young-adult fiction magic adventure cl...</td>\n",
       "      <td>j.r.r.tolkien</td>\n",
       "      <td>The Lord of the Rings (The Lord of the Rings, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>34</td>\n",
       "      <td>fantasy young-adult fiction magic adventure cl...</td>\n",
       "      <td>j.r.r.tolkien</td>\n",
       "      <td>The Fellowship of the Ring (The Lord of the Ri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>36</td>\n",
       "      <td>fantasy fiction science-fiction-fantasy high-f...</td>\n",
       "      <td>chrissmith christopherlee richardtaylor</td>\n",
       "      <td>The Lord of the Rings: Weapons and Warfare</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    goodreads_book_id                                           tag_name  \\\n",
       "0                   1  fantasy young-adult fiction magic childrens ad...   \n",
       "1                   2  fantasy young-adult fiction magic children-s n...   \n",
       "2                   3  fantasy young-adult fiction magic childrens ad...   \n",
       "3                   5  fantasy young-adult fiction magic childrens ad...   \n",
       "4                   6  fantasy young-adult fiction childrens adventur...   \n",
       "5                   8  fantasy young-adult fiction magic childrens ad...   \n",
       "6                  10  fantasy young-adult fiction magic childrens ad...   \n",
       "7                  11  fantasy young-adult fiction adventure classics...   \n",
       "8                  13  fantasy young-adult fiction adventure classics...   \n",
       "9                  21  fiction humor funny adult space science ebooks...   \n",
       "10                 24  fiction adventure contemporary humor funny com...   \n",
       "11                 25  fiction adventure contemporary humor funny com...   \n",
       "12                 26  fiction adventure contemporary humor funny com...   \n",
       "13                 27  fiction adventure contemporary humor funny com...   \n",
       "14                 28  fiction adventure contemporary humor funny com...   \n",
       "15                 29  contemporary humor funny comedy adult literatu...   \n",
       "16                 30  fantasy young-adult fiction magic childrens ad...   \n",
       "17                 33  fantasy young-adult fiction magic adventure cl...   \n",
       "18                 34  fantasy young-adult fiction magic adventure cl...   \n",
       "19                 36  fantasy fiction science-fiction-fantasy high-f...   \n",
       "\n",
       "                                    authors  \\\n",
       "0                  j.k.rowling marygrandpré   \n",
       "1                  j.k.rowling marygrandpré   \n",
       "2                  j.k.rowling marygrandpré   \n",
       "3        j.k.rowling marygrandpré rufusbeck   \n",
       "4                  j.k.rowling marygrandpré   \n",
       "5                  j.k.rowling marygrandpré   \n",
       "6                               j.k.rowling   \n",
       "7                              douglasadams   \n",
       "8                              douglasadams   \n",
       "9                                billbryson   \n",
       "10                               billbryson   \n",
       "11                               billbryson   \n",
       "12                               billbryson   \n",
       "13                               billbryson   \n",
       "14                               billbryson   \n",
       "15                               billbryson   \n",
       "16                            j.r.r.tolkien   \n",
       "17                            j.r.r.tolkien   \n",
       "18                            j.r.r.tolkien   \n",
       "19  chrissmith christopherlee richardtaylor   \n",
       "\n",
       "                                                title  \n",
       "0   Harry Potter and the Half-Blood Prince (Harry ...  \n",
       "1   Harry Potter and the Order of the Phoenix (Har...  \n",
       "2   Harry Potter and the Sorcerer's Stone (Harry P...  \n",
       "3   Harry Potter and the Prisoner of Azkaban (Harr...  \n",
       "4   Harry Potter and the Goblet of Fire (Harry Pot...  \n",
       "5   Harry Potter Boxed Set, Books 1-5 (Harry Potte...  \n",
       "6        Harry Potter Collection (Harry Potter, #1-6)  \n",
       "7   The Hitchhiker's Guide to the Galaxy (Hitchhik...  \n",
       "8       The Ultimate Hitchhiker's Guide to the Galaxy  \n",
       "9                A Short History of Nearly Everything  \n",
       "10                             In a Sunburned Country  \n",
       "11  I'm a Stranger Here Myself: Notes on Returning...  \n",
       "12  The Lost Continent: Travels in Small Town America  \n",
       "13          Neither Here nor There: Travels in Europe  \n",
       "14                          Notes from a Small Island  \n",
       "15  The Mother Tongue: English and How It Got That...  \n",
       "16  J.R.R. Tolkien 4-Book Boxed Set: The Hobbit an...  \n",
       "17  The Lord of the Rings (The Lord of the Rings, ...  \n",
       "18  The Fellowship of the Ring (The Lord of the Ri...  \n",
       "19         The Lord of the Rings: Weapons and Warfare  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stringedTags.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Then I add the author(s) to the stringed genre-tags\n",
    "stringedTags['all_tags'] = stringedTags['tag_name'] + \" \" + stringedTags['authors']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'fantasy young-adult fiction magic children-s novels paranormal teen witches urban-fantasy supernatural mystery wizards juvenile romance coming-of-age science-fiction-fantasy boarding-school action british-literature young-adult-fantasy j.k.rowling marygrandpré'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A example with one of Harry Potter's series book:\n",
    "stringedTags.all_tags[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now I use the 'CountVectorizer' function, that will generate a matrix in which the columns represent all the tag-words that I have included in the analysis and the rows represent the books.\n",
    "\n",
    "From that matrix (the 'tagMatrix'), I have calculated a similarity score between each book pair, choosing the 'cosine' as a metric of distance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "countVec = CountVectorizer(analyzer = 'word', ngram_range = (1, 2), min_df = 0, stop_words = 'english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagMatrix = countVec.fit_transform(stringedTags['all_tags'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<10000x26724 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 501273 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "cosineSim = cosine_similarity(tagMatrix, tagMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 10000)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosineSim.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.7643608 , 0.92288947, ..., 0.27508955, 0.21854335,\n",
       "        0.22239287],\n",
       "       [0.7643608 , 1.        , 0.71674754, ..., 0.26169537, 0.21271497,\n",
       "        0.19981092],\n",
       "       [0.92288947, 0.71674754, 1.        , ..., 0.18654789, 0.1717795 ,\n",
       "        0.17928749],\n",
       "       ...,\n",
       "       [0.27508955, 0.26169537, 0.18654789, ..., 1.        , 0.25668451,\n",
       "        0.27820744],\n",
       "       [0.21854335, 0.21271497, 0.1717795 , ..., 0.25668451, 1.        ,\n",
       "        0.77234329],\n",
       "       [0.22239287, 0.19981092, 0.17928749, ..., 0.27820744, 0.77234329,\n",
       "        1.        ]])"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosineSim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next step is retrieving the titles of the books (from the 'stringedTags' dataset, column 'title') and use tham as index, so that I can create a function, called 'topRecommendations' that takes in a book title and scores all cosine similarity of that title with each book in the dataset; then - after sorting by the highest similarity and after dropping the first observation (that will show the similarity of the book with itself, which is the maximum valu) it grabs only the first 10 most similar books to that title and return them in the form of a top-10 recommendation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "stringedTags = stringedTags.reset_index()\n",
    "bookTitles = stringedTags['title']\n",
    "indices = pd.Series(stringedTags.index, index = bookTitles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def topRecommendations(title):\n",
    "    index = indices[title]\n",
    "    similarityScore = list(enumerate(cosineSim[index]))\n",
    "    similarityScore = sorted(similarityScore, key = lambda x: x[1], reverse = True)\n",
    "    similarityScore = similarityScore[1:10]\n",
    "    bookIndex = [i[0] for i in similarityScore]\n",
    "    return bookTitles.iloc[bookIndex]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Having a look at few books, it looks like this content-based recommender system is able to capture similarity within books based on the tags I have used, giving interesting and relevant recommendations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3890    Harry Potter and the Deathly Hallows (Harry Po...\n",
       "2       Harry Potter and the Sorcerer's Stone (Harry P...\n",
       "1276    Harry Potter and the Chamber of Secrets (Harry...\n",
       "4       Harry Potter and the Goblet of Fire (Harry Pot...\n",
       "5686             Harry Potter Boxset (Harry Potter, #1-7)\n",
       "3       Harry Potter and the Prisoner of Azkaban (Harr...\n",
       "3488    The Harry Potter Collection 1-4 (Harry Potter,...\n",
       "5891    Harry Potter and the Order of the Phoenix (Har...\n",
       "5       Harry Potter Boxed Set, Books 1-5 (Harry Potte...\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topRecommendations('Harry Potter and the Half-Blood Prince (Harry Potter, #6)').head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "405       The Catcher in the Rye\n",
       "87               Of Mice and Men\n",
       "1048         Death of a Salesman\n",
       "440                  Ethan Frome\n",
       "174      The Old Man and the Sea\n",
       "326          The Grapes of Wrath\n",
       "198        To Kill a Mockingbird\n",
       "1381                The Crucible\n",
       "354     The Beautiful and Damned\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topRecommendations('The Great Gatsby').head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Item-Based Collaborative Filtering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another type of recommender that I have decided to implement is based on 'Collaborative Filtering' technique. In this case, I have considered a Item-Based Collaborative Filtering.\n",
    "\n",
    "The starting point of this type of system is a user by book matrix, in which each row indicates a single user, and each columns represents a single book.\n",
    "With the intent of making the table more easy to read and displaying the book titles rather than the book ids, I have used the 'ratingsWithTitles' table.\n",
    "\n",
    "The resulting userByBook table contains an extremely high number of NAs, due to the fact that - as seen before - even the users who have read the highest number of books in our dataset, they have just read roughly 1% of all the books in the analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratingsWithTitles = pd.merge(ratings, books[['id', 'title']], left_on = ['book_id'], right_on = ['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"21\" halign=\"left\">rating</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th>Angels (Walsh Family, #3)</th>\n",
       "      <th>#GIRLBOSS</th>\n",
       "      <th>'Salem's Lot</th>\n",
       "      <th>'Tis (Frank McCourt, #2)</th>\n",
       "      <th>1,000 Places to See Before You Die</th>\n",
       "      <th>10% Happier: How I Tamed the Voice in My Head, Reduced Stress Without Losing My Edge, and Found Self-Help That Actually Works</th>\n",
       "      <th>100 Bullets, Vol. 1: First Shot, Last Call</th>\n",
       "      <th>100 Love Sonnets</th>\n",
       "      <th>100 Selected Poems</th>\n",
       "      <th>10th Anniversary (Women's Murder Club, #10)</th>\n",
       "      <th>...</th>\n",
       "      <th>god is Not Great: How Religion Poisons Everything</th>\n",
       "      <th>number9dream</th>\n",
       "      <th>sTORI Telling</th>\n",
       "      <th>xxxHolic, Vol. 1 (xxxHOLiC, #1)</th>\n",
       "      <th>الفيل الأزرق</th>\n",
       "      <th>تراب الماس</th>\n",
       "      <th>عزازيل</th>\n",
       "      <th>هيبتا</th>\n",
       "      <th>يوتوبيا</th>\n",
       "      <th>ڤيرتيجو</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53279</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53281</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53286</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53288</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53292</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53293</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53295</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53306</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53318</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53331</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53332</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53333</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53337</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53339</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53347</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53348</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53352</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53364</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53366</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53372</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53373</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53378</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53381</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53388</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53401</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53403</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53409</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53411</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53413</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55000</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13971 rows × 8854 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            rating                         \\\n",
       "title    Angels (Walsh Family, #3) #GIRLBOSS 'Salem's Lot   \n",
       "user_id                                                     \n",
       "7                              NaN       NaN          NaN   \n",
       "19                             NaN       NaN          NaN   \n",
       "23                             NaN       NaN          NaN   \n",
       "27                             NaN       NaN          NaN   \n",
       "35                             NaN       NaN          NaN   \n",
       "40                             NaN       NaN          NaN   \n",
       "41                             NaN       NaN          NaN   \n",
       "46                             NaN       NaN          NaN   \n",
       "47                             NaN       NaN          NaN   \n",
       "49                             NaN       NaN          NaN   \n",
       "52                             NaN       NaN          NaN   \n",
       "75                             NaN       NaN          NaN   \n",
       "82                             NaN       NaN          NaN   \n",
       "87                             NaN       NaN          NaN   \n",
       "89                             NaN       NaN          NaN   \n",
       "107                            NaN       NaN          NaN   \n",
       "111                            NaN       NaN          NaN   \n",
       "113                            NaN       NaN          NaN   \n",
       "116                            NaN       NaN          NaN   \n",
       "119                            NaN       NaN          NaN   \n",
       "143                            NaN       NaN          NaN   \n",
       "145                            NaN       NaN          NaN   \n",
       "146                            NaN       NaN          NaN   \n",
       "148                            NaN       NaN          NaN   \n",
       "153                            NaN       NaN          NaN   \n",
       "158                            NaN       NaN          NaN   \n",
       "164                            NaN       NaN          NaN   \n",
       "173                            NaN       NaN          NaN   \n",
       "178                            NaN       NaN          NaN   \n",
       "193                            NaN       NaN          NaN   \n",
       "...                            ...       ...          ...   \n",
       "53279                          NaN       NaN          NaN   \n",
       "53281                          NaN       NaN          NaN   \n",
       "53286                          NaN       NaN          NaN   \n",
       "53288                          NaN       NaN          NaN   \n",
       "53292                          NaN       NaN          NaN   \n",
       "53293                          NaN       NaN          NaN   \n",
       "53295                          NaN       NaN          NaN   \n",
       "53306                          NaN       NaN          NaN   \n",
       "53318                          NaN       NaN          NaN   \n",
       "53331                          NaN       NaN          NaN   \n",
       "53332                          NaN       NaN          NaN   \n",
       "53333                          NaN       NaN          NaN   \n",
       "53337                          NaN       NaN          NaN   \n",
       "53339                          NaN       NaN          NaN   \n",
       "53347                          NaN       NaN          NaN   \n",
       "53348                          NaN       NaN          NaN   \n",
       "53352                          NaN       NaN          NaN   \n",
       "53364                          NaN       NaN          NaN   \n",
       "53366                          NaN       NaN          NaN   \n",
       "53372                          NaN       NaN          NaN   \n",
       "53373                          NaN       NaN          NaN   \n",
       "53378                          NaN       NaN          NaN   \n",
       "53381                          NaN       NaN          NaN   \n",
       "53388                          NaN       NaN          NaN   \n",
       "53401                          NaN       NaN          NaN   \n",
       "53403                          NaN       NaN          NaN   \n",
       "53409                          NaN       NaN          NaN   \n",
       "53411                          NaN       NaN          NaN   \n",
       "53413                          NaN       NaN          NaN   \n",
       "55000                          NaN       NaN          NaN   \n",
       "\n",
       "                                                                     \\\n",
       "title   'Tis (Frank McCourt, #2) 1,000 Places to See Before You Die   \n",
       "user_id                                                               \n",
       "7                            NaN                                NaN   \n",
       "19                           NaN                                NaN   \n",
       "23                           NaN                                NaN   \n",
       "27                           NaN                                NaN   \n",
       "35                           NaN                                NaN   \n",
       "40                           NaN                                NaN   \n",
       "41                           NaN                                NaN   \n",
       "46                           NaN                                NaN   \n",
       "47                           NaN                                NaN   \n",
       "49                           NaN                                NaN   \n",
       "52                           NaN                                NaN   \n",
       "75                           NaN                                NaN   \n",
       "82                           NaN                                NaN   \n",
       "87                           NaN                                NaN   \n",
       "89                           NaN                                NaN   \n",
       "107                          NaN                                NaN   \n",
       "111                          NaN                                NaN   \n",
       "113                          NaN                                NaN   \n",
       "116                          NaN                                NaN   \n",
       "119                          NaN                                NaN   \n",
       "143                          NaN                                NaN   \n",
       "145                          NaN                                NaN   \n",
       "146                          NaN                                NaN   \n",
       "148                          NaN                                NaN   \n",
       "153                          NaN                                NaN   \n",
       "158                          NaN                                NaN   \n",
       "164                          NaN                                NaN   \n",
       "173                          NaN                                NaN   \n",
       "178                          NaN                                NaN   \n",
       "193                          NaN                                NaN   \n",
       "...                          ...                                ...   \n",
       "53279                        NaN                                NaN   \n",
       "53281                        NaN                                NaN   \n",
       "53286                        NaN                                NaN   \n",
       "53288                        NaN                                NaN   \n",
       "53292                        NaN                                NaN   \n",
       "53293                        NaN                                NaN   \n",
       "53295                        NaN                                NaN   \n",
       "53306                        NaN                                NaN   \n",
       "53318                        NaN                                NaN   \n",
       "53331                        NaN                                NaN   \n",
       "53332                        NaN                                NaN   \n",
       "53333                        NaN                                NaN   \n",
       "53337                        NaN                                NaN   \n",
       "53339                        NaN                                NaN   \n",
       "53347                        NaN                                NaN   \n",
       "53348                        NaN                                NaN   \n",
       "53352                        NaN                                NaN   \n",
       "53364                        NaN                                NaN   \n",
       "53366                        NaN                                NaN   \n",
       "53372                        NaN                                NaN   \n",
       "53373                        NaN                                NaN   \n",
       "53378                        NaN                                NaN   \n",
       "53381                        NaN                                NaN   \n",
       "53388                        NaN                                NaN   \n",
       "53401                        NaN                                NaN   \n",
       "53403                        NaN                                NaN   \n",
       "53409                        NaN                                NaN   \n",
       "53411                        NaN                                NaN   \n",
       "53413                        NaN                                NaN   \n",
       "55000                        NaN                                NaN   \n",
       "\n",
       "                                                                                                                                       \\\n",
       "title   10% Happier: How I Tamed the Voice in My Head, Reduced Stress Without Losing My Edge, and Found Self-Help That Actually Works   \n",
       "user_id                                                                                                                                 \n",
       "7                                                      NaN                                                                              \n",
       "19                                                     NaN                                                                              \n",
       "23                                                     NaN                                                                              \n",
       "27                                                     NaN                                                                              \n",
       "35                                                     NaN                                                                              \n",
       "40                                                     NaN                                                                              \n",
       "41                                                     NaN                                                                              \n",
       "46                                                     NaN                                                                              \n",
       "47                                                     NaN                                                                              \n",
       "49                                                     NaN                                                                              \n",
       "52                                                     NaN                                                                              \n",
       "75                                                     NaN                                                                              \n",
       "82                                                     NaN                                                                              \n",
       "87                                                     NaN                                                                              \n",
       "89                                                     NaN                                                                              \n",
       "107                                                    NaN                                                                              \n",
       "111                                                    NaN                                                                              \n",
       "113                                                    NaN                                                                              \n",
       "116                                                    NaN                                                                              \n",
       "119                                                    NaN                                                                              \n",
       "143                                                    NaN                                                                              \n",
       "145                                                    NaN                                                                              \n",
       "146                                                    NaN                                                                              \n",
       "148                                                    NaN                                                                              \n",
       "153                                                    NaN                                                                              \n",
       "158                                                    NaN                                                                              \n",
       "164                                                    NaN                                                                              \n",
       "173                                                    NaN                                                                              \n",
       "178                                                    NaN                                                                              \n",
       "193                                                    NaN                                                                              \n",
       "...                                                    ...                                                                              \n",
       "53279                                                  NaN                                                                              \n",
       "53281                                                  NaN                                                                              \n",
       "53286                                                  NaN                                                                              \n",
       "53288                                                  NaN                                                                              \n",
       "53292                                                  NaN                                                                              \n",
       "53293                                                  NaN                                                                              \n",
       "53295                                                  NaN                                                                              \n",
       "53306                                                  NaN                                                                              \n",
       "53318                                                  NaN                                                                              \n",
       "53331                                                  NaN                                                                              \n",
       "53332                                                  NaN                                                                              \n",
       "53333                                                  NaN                                                                              \n",
       "53337                                                  NaN                                                                              \n",
       "53339                                                  NaN                                                                              \n",
       "53347                                                  NaN                                                                              \n",
       "53348                                                  NaN                                                                              \n",
       "53352                                                  NaN                                                                              \n",
       "53364                                                  NaN                                                                              \n",
       "53366                                                  NaN                                                                              \n",
       "53372                                                  NaN                                                                              \n",
       "53373                                                  NaN                                                                              \n",
       "53378                                                  NaN                                                                              \n",
       "53381                                                  NaN                                                                              \n",
       "53388                                                  NaN                                                                              \n",
       "53401                                                  NaN                                                                              \n",
       "53403                                                  NaN                                                                              \n",
       "53409                                                  NaN                                                                              \n",
       "53411                                                  NaN                                                                              \n",
       "53413                                                  NaN                                                                              \n",
       "55000                                                  NaN                                                                              \n",
       "\n",
       "                                                                     \\\n",
       "title   100 Bullets, Vol. 1: First Shot, Last Call 100 Love Sonnets   \n",
       "user_id                                                               \n",
       "7                                              NaN              NaN   \n",
       "19                                             NaN              NaN   \n",
       "23                                             NaN              NaN   \n",
       "27                                             NaN              NaN   \n",
       "35                                             NaN              NaN   \n",
       "40                                             NaN              NaN   \n",
       "41                                             NaN              NaN   \n",
       "46                                             NaN              NaN   \n",
       "47                                             NaN              NaN   \n",
       "49                                             NaN              NaN   \n",
       "52                                             NaN              NaN   \n",
       "75                                             NaN              NaN   \n",
       "82                                             NaN              NaN   \n",
       "87                                             NaN              NaN   \n",
       "89                                             NaN              NaN   \n",
       "107                                            NaN              NaN   \n",
       "111                                            NaN              NaN   \n",
       "113                                            NaN              NaN   \n",
       "116                                            NaN              NaN   \n",
       "119                                            NaN              NaN   \n",
       "143                                            NaN              NaN   \n",
       "145                                            NaN              NaN   \n",
       "146                                            NaN              NaN   \n",
       "148                                            NaN              NaN   \n",
       "153                                            NaN              NaN   \n",
       "158                                            NaN              NaN   \n",
       "164                                            NaN              NaN   \n",
       "173                                            NaN              NaN   \n",
       "178                                            NaN              NaN   \n",
       "193                                            NaN              NaN   \n",
       "...                                            ...              ...   \n",
       "53279                                          NaN              NaN   \n",
       "53281                                          NaN              NaN   \n",
       "53286                                          NaN              NaN   \n",
       "53288                                          NaN              NaN   \n",
       "53292                                          NaN              NaN   \n",
       "53293                                          NaN              NaN   \n",
       "53295                                          NaN              NaN   \n",
       "53306                                          NaN              NaN   \n",
       "53318                                          NaN              NaN   \n",
       "53331                                          NaN              NaN   \n",
       "53332                                          NaN              NaN   \n",
       "53333                                          NaN              NaN   \n",
       "53337                                          NaN              NaN   \n",
       "53339                                          NaN              NaN   \n",
       "53347                                          NaN              NaN   \n",
       "53348                                          NaN              NaN   \n",
       "53352                                          NaN              NaN   \n",
       "53364                                          NaN              NaN   \n",
       "53366                                          NaN              NaN   \n",
       "53372                                          NaN              NaN   \n",
       "53373                                          NaN              NaN   \n",
       "53378                                          NaN              NaN   \n",
       "53381                                          NaN              NaN   \n",
       "53388                                          NaN              NaN   \n",
       "53401                                          NaN              NaN   \n",
       "53403                                          NaN              NaN   \n",
       "53409                                          NaN              NaN   \n",
       "53411                                          NaN              NaN   \n",
       "53413                                          NaN              NaN   \n",
       "55000                                          NaN              NaN   \n",
       "\n",
       "                                                                        ...  \\\n",
       "title   100 Selected Poems 10th Anniversary (Women's Murder Club, #10)  ...   \n",
       "user_id                                                                 ...   \n",
       "7                      NaN                                         NaN  ...   \n",
       "19                     NaN                                         NaN  ...   \n",
       "23                     NaN                                         NaN  ...   \n",
       "27                     NaN                                         NaN  ...   \n",
       "35                     NaN                                         NaN  ...   \n",
       "40                     NaN                                         NaN  ...   \n",
       "41                     NaN                                         NaN  ...   \n",
       "46                     NaN                                         NaN  ...   \n",
       "47                     NaN                                         NaN  ...   \n",
       "49                     NaN                                         NaN  ...   \n",
       "52                     NaN                                         NaN  ...   \n",
       "75                     NaN                                         NaN  ...   \n",
       "82                     NaN                                         NaN  ...   \n",
       "87                     NaN                                         NaN  ...   \n",
       "89                     NaN                                         NaN  ...   \n",
       "107                    NaN                                         NaN  ...   \n",
       "111                    NaN                                         NaN  ...   \n",
       "113                    NaN                                         NaN  ...   \n",
       "116                    NaN                                         NaN  ...   \n",
       "119                    NaN                                         NaN  ...   \n",
       "143                    NaN                                         NaN  ...   \n",
       "145                    NaN                                         NaN  ...   \n",
       "146                    NaN                                         NaN  ...   \n",
       "148                    NaN                                         NaN  ...   \n",
       "153                    NaN                                         NaN  ...   \n",
       "158                    NaN                                         NaN  ...   \n",
       "164                    NaN                                         NaN  ...   \n",
       "173                    NaN                                         NaN  ...   \n",
       "178                    NaN                                         NaN  ...   \n",
       "193                    NaN                                         NaN  ...   \n",
       "...                    ...                                         ...  ...   \n",
       "53279                  NaN                                         NaN  ...   \n",
       "53281                  NaN                                         NaN  ...   \n",
       "53286                  NaN                                         NaN  ...   \n",
       "53288                  NaN                                         NaN  ...   \n",
       "53292                  NaN                                         NaN  ...   \n",
       "53293                  NaN                                         NaN  ...   \n",
       "53295                  NaN                                         NaN  ...   \n",
       "53306                  NaN                                         NaN  ...   \n",
       "53318                  NaN                                         NaN  ...   \n",
       "53331                  NaN                                         NaN  ...   \n",
       "53332                  NaN                                         NaN  ...   \n",
       "53333                  NaN                                         NaN  ...   \n",
       "53337                  NaN                                         NaN  ...   \n",
       "53339                  NaN                                         NaN  ...   \n",
       "53347                  NaN                                         NaN  ...   \n",
       "53348                  NaN                                         NaN  ...   \n",
       "53352                  NaN                                         NaN  ...   \n",
       "53364                  NaN                                         NaN  ...   \n",
       "53366                  NaN                                         NaN  ...   \n",
       "53372                  NaN                                         NaN  ...   \n",
       "53373                  NaN                                         NaN  ...   \n",
       "53378                  NaN                                         NaN  ...   \n",
       "53381                  NaN                                         NaN  ...   \n",
       "53388                  NaN                                         NaN  ...   \n",
       "53401                  NaN                                         NaN  ...   \n",
       "53403                  NaN                                         NaN  ...   \n",
       "53409                  NaN                                         NaN  ...   \n",
       "53411                  NaN                                         NaN  ...   \n",
       "53413                  NaN                                         NaN  ...   \n",
       "55000                  NaN                                         NaN  ...   \n",
       "\n",
       "                                                                        \\\n",
       "title   god is Not Great: How Religion Poisons Everything number9dream   \n",
       "user_id                                                                  \n",
       "7                                                     NaN          NaN   \n",
       "19                                                    NaN          NaN   \n",
       "23                                                    NaN          NaN   \n",
       "27                                                    NaN          NaN   \n",
       "35                                                    NaN          NaN   \n",
       "40                                                    NaN          NaN   \n",
       "41                                                    NaN          NaN   \n",
       "46                                                    NaN          NaN   \n",
       "47                                                    NaN          NaN   \n",
       "49                                                    NaN          NaN   \n",
       "52                                                    NaN          NaN   \n",
       "75                                                    NaN          NaN   \n",
       "82                                                    NaN          NaN   \n",
       "87                                                    NaN          NaN   \n",
       "89                                                    NaN          NaN   \n",
       "107                                                   NaN          NaN   \n",
       "111                                                   NaN          NaN   \n",
       "113                                                   NaN          NaN   \n",
       "116                                                   NaN          NaN   \n",
       "119                                                   NaN          NaN   \n",
       "143                                                   NaN          NaN   \n",
       "145                                                   NaN          NaN   \n",
       "146                                                   NaN          NaN   \n",
       "148                                                   NaN          NaN   \n",
       "153                                                   NaN          NaN   \n",
       "158                                                   NaN          NaN   \n",
       "164                                                   NaN          NaN   \n",
       "173                                                   NaN          NaN   \n",
       "178                                                   NaN          NaN   \n",
       "193                                                   NaN          NaN   \n",
       "...                                                   ...          ...   \n",
       "53279                                                 NaN          NaN   \n",
       "53281                                                 NaN          NaN   \n",
       "53286                                                 NaN          NaN   \n",
       "53288                                                 NaN          NaN   \n",
       "53292                                                 NaN          NaN   \n",
       "53293                                                 NaN          NaN   \n",
       "53295                                                 NaN          NaN   \n",
       "53306                                                 NaN          NaN   \n",
       "53318                                                 NaN          NaN   \n",
       "53331                                                 NaN          NaN   \n",
       "53332                                                 NaN          NaN   \n",
       "53333                                                 NaN          NaN   \n",
       "53337                                                 NaN          NaN   \n",
       "53339                                                 NaN          NaN   \n",
       "53347                                                 NaN          NaN   \n",
       "53348                                                 NaN          NaN   \n",
       "53352                                                 NaN          NaN   \n",
       "53364                                                 NaN          NaN   \n",
       "53366                                                 NaN          NaN   \n",
       "53372                                                 NaN          NaN   \n",
       "53373                                                 NaN          NaN   \n",
       "53378                                                 NaN          NaN   \n",
       "53381                                                 NaN          NaN   \n",
       "53388                                                 NaN          NaN   \n",
       "53401                                                 NaN          NaN   \n",
       "53403                                                 NaN          NaN   \n",
       "53409                                                 NaN          NaN   \n",
       "53411                                                 NaN          NaN   \n",
       "53413                                                 NaN          NaN   \n",
       "55000                                                 NaN          NaN   \n",
       "\n",
       "                                                                               \\\n",
       "title   sTORI Telling xxxHolic, Vol. 1 (xxxHOLiC, #1) الفيل الأزرق تراب الماس   \n",
       "user_id                                                                         \n",
       "7                 NaN                             NaN          NaN        NaN   \n",
       "19                NaN                             NaN          NaN        NaN   \n",
       "23                NaN                             NaN          NaN        NaN   \n",
       "27                NaN                             NaN          NaN        NaN   \n",
       "35                NaN                             NaN          NaN        NaN   \n",
       "40                NaN                             NaN          NaN        NaN   \n",
       "41                NaN                             NaN          NaN        NaN   \n",
       "46                NaN                             NaN          NaN        NaN   \n",
       "47                NaN                             NaN          NaN        NaN   \n",
       "49                NaN                             NaN          NaN        NaN   \n",
       "52                NaN                             NaN          NaN        NaN   \n",
       "75                NaN                             NaN          NaN        NaN   \n",
       "82                NaN                             NaN          NaN        NaN   \n",
       "87                NaN                             NaN          NaN        NaN   \n",
       "89                NaN                             NaN          NaN        NaN   \n",
       "107               NaN                             NaN          NaN        NaN   \n",
       "111               NaN                             NaN          NaN        NaN   \n",
       "113               NaN                             NaN          NaN        NaN   \n",
       "116               NaN                             NaN          NaN        NaN   \n",
       "119               NaN                             NaN          NaN        NaN   \n",
       "143               NaN                             NaN          NaN        NaN   \n",
       "145               NaN                             NaN          NaN        NaN   \n",
       "146               NaN                             NaN          NaN        NaN   \n",
       "148               NaN                             NaN          NaN        NaN   \n",
       "153               NaN                             NaN          NaN        NaN   \n",
       "158               NaN                             NaN          NaN        NaN   \n",
       "164               NaN                             NaN          NaN        NaN   \n",
       "173               NaN                             NaN          NaN        NaN   \n",
       "178               NaN                             NaN          NaN        NaN   \n",
       "193               NaN                             NaN          NaN        NaN   \n",
       "...               ...                             ...          ...        ...   \n",
       "53279             NaN                             NaN          NaN        NaN   \n",
       "53281             NaN                             NaN          NaN        NaN   \n",
       "53286             NaN                             NaN          NaN        NaN   \n",
       "53288             NaN                             NaN          NaN        NaN   \n",
       "53292             NaN                             NaN          NaN        NaN   \n",
       "53293             NaN                             NaN          NaN        NaN   \n",
       "53295             NaN                             NaN          NaN        NaN   \n",
       "53306             NaN                             NaN          NaN        NaN   \n",
       "53318             NaN                             NaN          NaN        NaN   \n",
       "53331             NaN                             NaN          NaN        NaN   \n",
       "53332             NaN                             NaN          NaN        NaN   \n",
       "53333             NaN                             NaN          NaN        NaN   \n",
       "53337             NaN                             NaN          NaN        NaN   \n",
       "53339             NaN                             NaN          NaN        NaN   \n",
       "53347             NaN                             NaN          NaN        NaN   \n",
       "53348             NaN                             NaN          NaN        NaN   \n",
       "53352             NaN                             NaN          NaN        NaN   \n",
       "53364             NaN                             NaN          NaN        NaN   \n",
       "53366             NaN                             NaN          NaN        NaN   \n",
       "53372             NaN                             NaN          NaN        NaN   \n",
       "53373             NaN                             NaN          NaN        NaN   \n",
       "53378             NaN                             NaN          NaN        NaN   \n",
       "53381             NaN                             NaN          NaN        NaN   \n",
       "53388             NaN                             NaN          NaN        NaN   \n",
       "53401             NaN                             NaN          NaN        NaN   \n",
       "53403             NaN                             NaN          NaN        NaN   \n",
       "53409             NaN                             NaN          NaN        NaN   \n",
       "53411             NaN                             NaN          NaN        NaN   \n",
       "53413             NaN                             NaN          NaN        NaN   \n",
       "55000             NaN                             NaN          NaN        NaN   \n",
       "\n",
       "                                      \n",
       "title   عزازيل هيبتا يوتوبيا ڤيرتيجو  \n",
       "user_id                               \n",
       "7          NaN   NaN     NaN     NaN  \n",
       "19         NaN   NaN     NaN     NaN  \n",
       "23         NaN   NaN     NaN     NaN  \n",
       "27         NaN   NaN     NaN     NaN  \n",
       "35         NaN   NaN     NaN     NaN  \n",
       "40         NaN   NaN     NaN     NaN  \n",
       "41         NaN   NaN     NaN     NaN  \n",
       "46         NaN   NaN     NaN     NaN  \n",
       "47         NaN   NaN     NaN     NaN  \n",
       "49         NaN   NaN     NaN     NaN  \n",
       "52         NaN   NaN     NaN     NaN  \n",
       "75         NaN   NaN     NaN     NaN  \n",
       "82         NaN   NaN     NaN     NaN  \n",
       "87         NaN   NaN     NaN     NaN  \n",
       "89         NaN   NaN     NaN     NaN  \n",
       "107        NaN   NaN     NaN     NaN  \n",
       "111        NaN   NaN     NaN     NaN  \n",
       "113        NaN   NaN     NaN     NaN  \n",
       "116        NaN   NaN     NaN     NaN  \n",
       "119        NaN   NaN     NaN     NaN  \n",
       "143        NaN   NaN     NaN     NaN  \n",
       "145        NaN   NaN     NaN     NaN  \n",
       "146        NaN   NaN     NaN     NaN  \n",
       "148        NaN   NaN     NaN     NaN  \n",
       "153        NaN   NaN     NaN     NaN  \n",
       "158        NaN   NaN     NaN     NaN  \n",
       "164        NaN   NaN     NaN     NaN  \n",
       "173        NaN   NaN     NaN     NaN  \n",
       "178        NaN   NaN     NaN     NaN  \n",
       "193        NaN   NaN     NaN     NaN  \n",
       "...        ...   ...     ...     ...  \n",
       "53279      NaN   NaN     NaN     NaN  \n",
       "53281      NaN   NaN     NaN     NaN  \n",
       "53286      NaN   NaN     NaN     NaN  \n",
       "53288      NaN   NaN     NaN     NaN  \n",
       "53292      NaN   NaN     NaN     NaN  \n",
       "53293      NaN   NaN     NaN     NaN  \n",
       "53295      NaN   NaN     NaN     NaN  \n",
       "53306      NaN   NaN     NaN     NaN  \n",
       "53318      NaN   NaN     NaN     NaN  \n",
       "53331      NaN   NaN     NaN     NaN  \n",
       "53332      NaN   NaN     NaN     NaN  \n",
       "53333      NaN   NaN     NaN     NaN  \n",
       "53337      NaN   NaN     NaN     NaN  \n",
       "53339      NaN   NaN     NaN     NaN  \n",
       "53347      NaN   NaN     NaN     NaN  \n",
       "53348      NaN   NaN     NaN     NaN  \n",
       "53352      NaN   NaN     NaN     NaN  \n",
       "53364      NaN   NaN     NaN     NaN  \n",
       "53366      NaN   NaN     NaN     NaN  \n",
       "53372      NaN   NaN     NaN     NaN  \n",
       "53373      NaN   NaN     NaN     NaN  \n",
       "53378      NaN   NaN     NaN     NaN  \n",
       "53381      NaN   NaN     NaN     NaN  \n",
       "53388      NaN   NaN     NaN     NaN  \n",
       "53401      NaN   NaN     NaN     NaN  \n",
       "53403      NaN   NaN     NaN     NaN  \n",
       "53409      NaN   NaN     NaN     NaN  \n",
       "53411      NaN   NaN     NaN     NaN  \n",
       "53413      NaN   NaN     NaN     NaN  \n",
       "55000      NaN   NaN     NaN     NaN  \n",
       "\n",
       "[13971 rows x 8854 columns]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userByBook = pd.pivot_table(ratingsWithTitles, index = ['user_id'], columns = ['title'], values = ['rating'])\n",
    "userByBook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13971, 8854)"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userByBook.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removing level in columns\n",
    "userByBook.columns = userByBook.columns.droplevel(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The goal in item-based collaborative filtering is to score the similarity between book pairs, based on the ratings they have received by the pool of users. To obtain a matrix in which each rows and columns will represent a single book and the values in the matrix represent the similarity between them, I have used the corr() function, which computes pairwise correlation of columns, excluding NA values, using Pearson as default method."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to get better recommendations and avoid that not very relevant books will pop up in the Top-N list, I have played with the min_periods parameter within the corr() function, which allows to drop from the correlation computation all the books that are not supported by at least a certain number of people that rated both titles.\n",
    "I have set the parameter to 50, 40 and 30 with the intent of having a look at the three of them and see which one looks more reliable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using min_periods = 40\n",
    "corrMatrix40 = userByBook.corr(method = 'pearson', min_periods = 40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "def newRecommendations_40(title):\n",
    "    bookRating = corrMatrix40[[title]][:]\n",
    "    bookRating = bookRating.dropna()\n",
    "    df = pd.DataFrame(bookRating.sort_values(by = title, ascending = False))[1:]\n",
    "    return df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>title</th>\n",
       "      <th>1984</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Animal Farm</th>\n",
       "      <td>0.713404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Odyssey</th>\n",
       "      <td>0.539299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brave New World</th>\n",
       "      <td>0.525559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fahrenheit 451</th>\n",
       "      <td>0.505960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lord of the Flies</th>\n",
       "      <td>0.495693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Catch-22</th>\n",
       "      <td>0.481534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Great Expectations</th>\n",
       "      <td>0.473683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Hitchhiker's Guide to the Galaxy (Hitchhiker's Guide to the Galaxy, #1)</th>\n",
       "      <td>0.414133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Memoirs of a Geisha</th>\n",
       "      <td>0.388141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>The Great Gatsby</th>\n",
       "      <td>0.358294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "title                                                   1984\n",
       "title                                                       \n",
       "Animal Farm                                         0.713404\n",
       "The Odyssey                                         0.539299\n",
       "Brave New World                                     0.525559\n",
       "Fahrenheit 451                                      0.505960\n",
       "Lord of the Flies                                   0.495693\n",
       "Catch-22                                            0.481534\n",
       "Great Expectations                                  0.473683\n",
       "The Hitchhiker's Guide to the Galaxy (Hitchhike...  0.414133\n",
       "Memoirs of a Geisha                                 0.388141\n",
       "The Great Gatsby                                    0.358294"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newRecommendations_40('1984')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Given the above results, I have decided to stick with the min_perios = 40 setting, as long as 50 still includes some not satisfying recommendations and 30 remove too many points, not capturing few interesting recommendations."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 3.1 Using item-based correlation matrix to make recommendations to specific users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "title\n",
       "1984                                                              5.0\n",
       "A Clockwork Orange                                                5.0\n",
       "A Long Way Down                                                   4.0\n",
       "A Midsummer Night's Dream                                         4.0\n",
       "A Scanner Darkly                                                  5.0\n",
       "About a Boy                                                       5.0\n",
       "Animal Farm                                                       5.0\n",
       "Billy Budd, Sailor                                                2.0\n",
       "Cat's Cradle                                                      4.0\n",
       "Choke                                                             5.0\n",
       "Chronicle of a Death Foretold                                     5.0\n",
       "Crash                                                             3.0\n",
       "Do Androids Dream of Electric Sheep?                              5.0\n",
       "Dracula                                                           5.0\n",
       "Fahrenheit 451                                                    5.0\n",
       "Fear and Loathing in Las Vegas                                    5.0\n",
       "Fever Pitch                                                       3.0\n",
       "Fight Club                                                        5.0\n",
       "Flow My Tears, the Policeman Said                                 5.0\n",
       "Frankenstein                                                      4.0\n",
       "Great Expectations                                                5.0\n",
       "Hamlet                                                            4.0\n",
       "Heart of Darkness                                                 5.0\n",
       "High Fidelity                                                     5.0\n",
       "Invisible Monsters                                                3.0\n",
       "Ivanhoe                                                           2.0\n",
       "Journey to the Center of the Earth (Extraordinary Voyages, #3)    4.0\n",
       "Junky                                                             4.0\n",
       "King Lear                                                         3.0\n",
       "Lord of the Flies                                                 4.0\n",
       "                                                                 ... \n",
       "Mother Night                                                      4.0\n",
       "No Country for Old Men                                            5.0\n",
       "Norwegian Wood                                                    5.0\n",
       "One Flew Over the Cuckoo's Nest                                   3.0\n",
       "Othello                                                           4.0\n",
       "Rant                                                              4.0\n",
       "Romeo and Juliet                                                  4.0\n",
       "Six Days of the Condor                                            3.0\n",
       "Slaughterhouse-Five                                               5.0\n",
       "Sleepers                                                          5.0\n",
       "Snuff                                                             3.0\n",
       "The Black Arrow                                                   3.0\n",
       "The Call of the Wild                                              4.0\n",
       "The Catcher in the Rye                                            5.0\n",
       "The Gambler                                                       3.0\n",
       "The Godfather                                                     5.0\n",
       "The Little Prince                                                 3.0\n",
       "The Lovely Bones                                                  4.0\n",
       "The Master and Margarita                                          5.0\n",
       "The Merchant of Venice                                            3.0\n",
       "The Rum Diary                                                     4.0\n",
       "The Talented Mr. Ripley (Ripley, #1)                              2.0\n",
       "The Taming of the Shrew                                           3.0\n",
       "The Unbearable Lightness of Being                                 3.0\n",
       "The Virgin Suicides                                               4.0\n",
       "Tortilla Flat                                                     4.0\n",
       "Trainspotting                                                     4.0\n",
       "Treasure Island                                                   3.0\n",
       "Ubik                                                              5.0\n",
       "White Fang                                                        3.0\n",
       "Name: 55000, Length: 61, dtype: float64"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myRatings = userByBook.loc[55000].dropna()\n",
    "myRatings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1984                        5.0\n",
      "Fahrenheit 451              5.0\n",
      "The Catcher in the Rye      5.0\n",
      "Sleepers                    5.0\n",
      "Slaughterhouse-Five         5.0\n",
      "The Master and Margarita    5.0\n",
      "Norwegian Wood              5.0\n",
      "No Country for Old Men      5.0\n",
      "High Fidelity               5.0\n",
      "Heart of Darkness           5.0\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "simCandidates = pd.Series()\n",
    "\n",
    "for i in range(0, len(myRatings.index)):\n",
    "    sims = corrMatrix40[myRatings.index[i]].dropna()\n",
    "    sims = sims.map(lambda x: x * myRatings[i])\n",
    "    simCandidates =  simCandidates.append(sims)\n",
    "\n",
    "simCandidates.sort_values(inplace = True, ascending = False)\n",
    "print(simCandidates.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1984                                                 19.902126\n",
       "A Christmas Carol                                     6.895688\n",
       "A Clockwork Orange                                    7.842066\n",
       "A Farewell to Arms                                    0.962991\n",
       "A Long Way Down                                       4.000000\n",
       "A Midsummer Night's Dream                            11.404721\n",
       "A Scanner Darkly                                      5.000000\n",
       "A Tale of Two Cities                                 15.283732\n",
       "A Thousand Splendid Suns                              2.069519\n",
       "A Wrinkle in Time (A Wrinkle in Time Quintet, #1)     1.934522\n",
       "dtype: float64"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simCandidates = simCandidates.groupby(simCandidates.index).sum()\n",
    "simCandidates.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Animal Farm            24.526423\n",
       "Fahrenheit 451         23.670873\n",
       "Great Expectations     21.583948\n",
       "Brave New World        21.442749\n",
       "Slaughterhouse-Five    20.935737\n",
       "Lord of the Flies      20.624156\n",
       "1984                   19.902126\n",
       "Macbeth                19.646777\n",
       "Romeo and Juliet       18.721112\n",
       "Of Mice and Men        18.713834\n",
       "dtype: float64"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simCandidates.sort_values(inplace = True, ascending = False)\n",
    "simCandidates.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Brave New World          21.442749\n",
       "Macbeth                  19.646777\n",
       "Of Mice and Men          18.713834\n",
       "The Scarlet Letter       16.610261\n",
       "Catch-22                 16.054034\n",
       "The Great Gatsby         15.697025\n",
       "The Odyssey              15.535719\n",
       "The Hobbit               15.504275\n",
       "A Tale of Two Cities     15.283732\n",
       "Sense and Sensibility    15.255581\n",
       "dtype: float64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filteredSims = simCandidates.drop(myRatings.index)\n",
    "filteredSims.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}