SMM_Recommender_System_prabhj.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DZuJgz7f2IZ3"
   },
   "source": [
    "## Super Mario Maker Dataset - Recommender System"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ErHiyzZVVERF"
   },
   "source": [
    "### 1. Prerequisites"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "moaC0uhpU5i_"
   },
   "source": [
    "Import required libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 138
    },
    "colab_type": "code",
    "id": "hthCYrf-F1bA",
    "outputId": "a0362b56-96b9-4072-ee25-c446087d2a1a"
   },
   "outputs": [],
   "source": [
    "#!pip install surprise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CqfsTYm83mip"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, precision_recall_fscore_support\n",
    "import os\n",
    "import gc\n",
    "from sklearn import model_selection\n",
    "from surprise import NormalPredictor, SVD, KNNBasic, SVDpp\n",
    "from surprise import Dataset\n",
    "from surprise import Reader\n",
    "import surprise\n",
    "from surprise.model_selection import cross_validate, train_test_split\n",
    "from surprise import accuracy\n",
    "from surprise.model_selection import cross_validate\n",
    "\n",
    "import scipy\n",
    "from scipy.sparse import csr_matrix\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pHXQkBC7U-U3"
   },
   "source": [
    "Specify data files location."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6Tm0Ut1d3UY2"
   },
   "outputs": [],
   "source": [
    "data_path = '/Users/pj/Documents/671_Project/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "c-kKbKIKVr1w"
   },
   "source": [
    "Read data of games played."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UMkSzukpVrII"
   },
   "outputs": [],
   "source": [
    "plays_df = pd.read_csv(data_path + 'plays.csv', sep=\"\\t\", usecols=['id', 'player'])\n",
    "plays_df.columns = ['game', 'player']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "3Uq1p0UrVzWq",
    "outputId": "f19fd40b-7c83-48f4-feaa-4b86128b4353"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>game</th>\n",
       "      <th>player</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>darter60k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>davidchofis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>fabioviana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>gorigokky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>groebenzell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  game       player\n",
       "0  0000-0000-0353-3D35    darter60k\n",
       "1  0000-0000-0353-3D35  davidchofis\n",
       "2  0000-0000-0353-3D35   fabioviana\n",
       "3  0000-0000-0353-3D35    gorigokky\n",
       "4  0000-0000-0353-3D35  groebenzell"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plays_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iVkjo039YmG3"
   },
   "source": [
    "Add a column with value of 0 to start with indicating game is played but not cleared by the player."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y7LyYlsEYlR2"
   },
   "outputs": [],
   "source": [
    "plays_df['score'] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zXU4fM38V0o9"
   },
   "source": [
    "Read data of games cleared data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AGJF2Wg9V1Bb"
   },
   "outputs": [],
   "source": [
    "clears_df = pd.read_csv('/Users/pj/Documents/671_Project/clears.csv', sep='\\t', usecols=['id', 'player'])\n",
    "clears_df.columns = ['game', 'player']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "Zqz0n8RQV83l",
    "outputId": "de135e8d-eaa6-49b8-f589-7edc126d0a3e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>game</th>\n",
       "      <th>player</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>darter60k</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>davidchofis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>fabioviana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>groebenzell</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>igatake0229</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  game       player\n",
       "0  0000-0000-0353-3D35    darter60k\n",
       "1  0000-0000-0353-3D35  davidchofis\n",
       "2  0000-0000-0353-3D35   fabioviana\n",
       "3  0000-0000-0353-3D35  groebenzell\n",
       "4  0000-0000-0353-3D35  igatake0229"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clears_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DOgXluWnYvbi"
   },
   "source": [
    "Add a column with value of 1 indicating that the game is cleared by the player."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NTL56Ih2Yv5i"
   },
   "outputs": [],
   "source": [
    "clears_df['score'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>game</th>\n",
       "      <th>player</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>darter60k</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>davidchofis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>fabioviana</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>gorigokky</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000-0000-0353-3D35</td>\n",
       "      <td>groebenzell</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  game       player  score\n",
       "0  0000-0000-0353-3D35    darter60k      0\n",
       "1  0000-0000-0353-3D35  davidchofis      0\n",
       "2  0000-0000-0353-3D35   fabioviana      0\n",
       "3  0000-0000-0353-3D35    gorigokky      0\n",
       "4  0000-0000-0353-3D35  groebenzell      0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plays_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hI4Ps_ccQGUx"
   },
   "source": [
    "Next we merge the 2 dataframes, so it now has information about games cleared as well as games played, but not cleared."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ULogY0x4ZY-l"
   },
   "outputs": [],
   "source": [
    "all_data = clears_df.merge(plays_df, how='outer', on=['game', 'player'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>game</th>\n",
       "      <th>player</th>\n",
       "      <th>score_x</th>\n",
       "      <th>score_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1765852</th>\n",
       "      <td>DC84-0000-032C-80C9</td>\n",
       "      <td>MightyOBERON</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2397648</th>\n",
       "      <td>2BC9-0000-0362-A24D</td>\n",
       "      <td>Noooo15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301831</th>\n",
       "      <td>2679-0000-032A-F74E</td>\n",
       "      <td>Kadezzoyt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>789624</th>\n",
       "      <td>62E9-0000-0352-71C0</td>\n",
       "      <td>goatoo</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3453745</th>\n",
       "      <td>B118-0000-031B-1E68</td>\n",
       "      <td>JOJOGIOGIO1119</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        game          player  score_x  score_y\n",
       "1765852  DC84-0000-032C-80C9    MightyOBERON      1.0      0.0\n",
       "2397648  2BC9-0000-0362-A24D         Noooo15      NaN      0.0\n",
       "301831   2679-0000-032A-F74E       Kadezzoyt      1.0      0.0\n",
       "789624   62E9-0000-0352-71C0          goatoo      1.0      0.0\n",
       "3453745  B118-0000-031B-1E68  JOJOGIOGIO1119      NaN      0.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZLbyJsRsQTSP"
   },
   "source": [
    "Replace *NaN* values with $0$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AgfdQWCf9WFM"
   },
   "outputs": [],
   "source": [
    "all_data.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LSotSseu5ALc"
   },
   "source": [
    "Drop last column *score_y* as it contains only zeroes, all required information is now available in *score_x*. It is $0$ when the player has just played the game, but not cleared. It is $1$ when the player has cleared that game."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OxMuK0ZB4xQZ"
   },
   "outputs": [],
   "source": [
    "all_data.drop(columns='score_y', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Jfn4YcfCRBtT"
   },
   "source": [
    "Presently, $0$ and $1$ labels are segregated, lets shuffle the dataframe to distribute them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aYiFbMmgZu8w"
   },
   "outputs": [],
   "source": [
    "all_data = all_data.sample(frac=1).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "b3Gy7UL94c8m"
   },
   "outputs": [],
   "source": [
    "train_data, test_data = model_selection.train_test_split(all_data, test_size = 0.2)\n",
    "val_data, test_data   = model_selection.train_test_split(test_data, test_size = 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2hZdb4FQXzvR"
   },
   "source": [
    "### 2. Exploratory Data Analysis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZbHe8zKCSBzb"
   },
   "source": [
    "Let us check some data attributes before building the recommendation system."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "Nru5h8w3RypH",
    "outputId": "096b5ae8-d8fc-432a-c8d8-2b929b2f5eed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of players : 861465\n",
      "Total number of games   : 115032\n"
     ]
    }
   ],
   "source": [
    "print('Total number of players : {}'.format(len(all_data['player'].value_counts())))\n",
    "print('Total number of games   : {}'.format(len(all_data['game'].value_counts())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "P1zV6E5fSLro",
    "outputId": "527a6f5c-000c-492e-a4a9-b14237307deb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of games played and cleared     : 2051809\n",
      "Number of games played but not cleared : 2036348\n"
     ]
    }
   ],
   "source": [
    "print('Number of games played and cleared     : {}'.format(len(all_data[all_data['score_x'] == 1])))\n",
    "print('Number of games played but not cleared : {}'.format(len(all_data[all_data['score_x'] == 0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "R-rJP5acddYF"
   },
   "source": [
    "**Comment:** As we can observed from output of above cell, there is approximately $50:50$ split between game levels cleared and games only played."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NJoYH6qdSxWn"
   },
   "source": [
    "### 3. Recommendation system data preparation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gQsFpeI2EEZ6"
   },
   "outputs": [],
   "source": [
    "reader = Reader(rating_scale=(0, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BM2s1M4XS6wI"
   },
   "source": [
    "Load train, validation and test data from dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bgWRJIIqGbz8"
   },
   "outputs": [],
   "source": [
    "rec_data   = Dataset.load_from_df(train_data[['player', 'game', 'score_x']], reader)\n",
    "rec_data_v = Dataset.load_from_df(val_data[['player', 'game', 'score_x']], reader)\n",
    "rec_data_t = Dataset.load_from_df(test_data[['player', 'game', 'score_x']], reader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0ammCC3ls02J"
   },
   "source": [
    "Train/Val/Test split = $80:10:10$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ao5jjmUSszos"
   },
   "outputs": [],
   "source": [
    "train_set , _    = surprise.model_selection.train_test_split(rec_data, test_size=0.0001) # 0.01% data is discarded, bcoz it does not allow test_size to be 0.\n",
    "_ , val_set      = surprise.model_selection.train_test_split(rec_data_v, test_size=1.0)\n",
    "_ , test_set     = surprise.model_selection.train_test_split(rec_data_t, test_size=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_set.split(n_folds=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HxW0jedoTj1b"
   },
   "source": [
    "### 4. Baseline recommendation system."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oVN3MWTxUGem"
   },
   "source": [
    "We will use normal predictor as baseline recommender. It is an algorithm predicting a random rating based on the distribution of the training set, which is assumed to be normal.\n",
    "\n",
    "The prediction $\\hat r_{ui}$ is generated from a normal distribution $N(\\hat \\mu, \\hat \\sigma)$ where $\\hat \\mu$ and $\\hat \\sigma$ are estimated from the training data using Maximum Likelihood Estimation:\n",
    "\n",
    "$ \\hat \\mu=\\frac{1}{R_{train}} \\sum_{r_{ui} \\in R_{train}}\n",
    "{r_{ui}}$\n",
    "\n",
    "$\\hat \\sigma = \\sqrt{\\sum_{r_{ui} \\in R_{train}} \\frac{(r_{ui}-{\\hat \\mu})^2}{|R_{train}|}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E_My3pq5ISoJ"
   },
   "outputs": [],
   "source": [
    "algo_normal = NormalPredictor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm NormalPredictor on 5 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
      "RMSE (testset)    0.6152  0.6159  0.6158  0.6156  0.6159  0.6157  0.0003  \n",
      "MAE (testset)     0.4995  0.5003  0.5003  0.4999  0.5005  0.5001  0.0004  \n",
      "Fit time          5.47    7.50    7.98    9.40    16.48   9.37    3.77    \n",
      "Test time         10.48   7.45    7.65    14.53   18.50   11.72   4.24    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([0.61519401, 0.61590839, 0.61582473, 0.61564001, 0.61593228]),\n",
       " 'test_mae': array([0.49949773, 0.50031523, 0.50034874, 0.49990455, 0.50049746]),\n",
       " 'fit_time': (5.474833965301514,\n",
       "  7.497360944747925,\n",
       "  7.98160195350647,\n",
       "  9.402897119522095,\n",
       "  16.48410701751709),\n",
       " 'test_time': (10.475564002990723,\n",
       "  7.4539079666137695,\n",
       "  7.652374982833862,\n",
       "  14.526904821395874,\n",
       "  18.495610237121582)}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo_normal, rec_data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "a9ZpfXnebxXI",
    "outputId": "0a76c640-a46f-4dd9-d59a-a068cea1d2bc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.random_pred.NormalPredictor at 0x10cad0820>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algo_normal.fit(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5Rfj8s0OeXVR"
   },
   "outputs": [],
   "source": [
    "pred_normal = algo_normal.test(val_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "F-9KN242rXaw",
    "outputId": "a6d8151d-85d8-40ce-af4e-8405da2b5efc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.6160\n",
      "MSE: 0.3795\n"
     ]
    }
   ],
   "source": [
    "rmse_normal = accuracy.rmse(pred_normal)\n",
    "mse_normal = accuracy.mse(pred_normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "zT1xu6zCrlQd",
    "outputId": "477fe5b4-3cfe-4b6c-9259-a918045acb36"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE using Normal Predictor: 0.6160\n",
      "Validation MSE using Normal Predictor : 0.3795\n"
     ]
    }
   ],
   "source": [
    "print(\"Validation RMSE using Normal Predictor: {0:5.4f}\".format(rmse_normal))\n",
    "print(\"Validation MSE using Normal Predictor : {0:5.4f}\".format(mse_normal))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Using kNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#algo_knn = KNNBasic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Takes infinitely long and throws out of memory error.\n",
    "#cross_validate(algo_knn, rec_data, measures=['RMSE', 'MAE'], cv=3, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dvKx_sZ4fAzu"
   },
   "source": [
    "### 5. Recommendation system using SVD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HN_YmvYCJK7b"
   },
   "outputs": [],
   "source": [
    "algo_svd = SVD()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "aExbybVgKG5_",
    "outputId": "5f04819c-e264-4b86-fc29-00ba98155652"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.matrix_factorization.SVD at 0x1b91c9cd0>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algo_svd.fit(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FODWIoK7GhXT"
   },
   "outputs": [],
   "source": [
    "pred_svd = algo_svd.test(val_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "ZAu-IdzUtHc0",
    "outputId": "d24434ce-4a0d-439a-c4fa-47dbd2793fac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.4033\n",
      "MSE: 0.1626\n"
     ]
    }
   ],
   "source": [
    "rmse_svd = accuracy.rmse(pred_svd)\n",
    "mse_svd  = accuracy.mse(pred_svd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "k4xW2xoGHE2U",
    "outputId": "d22449af-8aed-4e20-bd67-0d249c699f41"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation RMSE using SVD : 0.4033\n",
      "Validation MSE using SVD  : 0.1626\n"
     ]
    }
   ],
   "source": [
    "print(\"Validation RMSE using SVD : {0:5.4f}\".format(rmse_svd))\n",
    "print(\"Validation MSE using SVD  : {0:5.4f}\".format(mse_svd))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm SVD on 5 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
      "RMSE (testset)    0.4088  0.4087  0.4084  0.4085  0.4084  0.4086  0.0002  \n",
      "MAE (testset)     0.3453  0.3451  0.3449  0.3447  0.3448  0.3450  0.0002  \n",
      "Fit time          434.69  418.07  419.54  380.73  415.87  413.78  17.81   \n",
      "Test time         20.25   14.72   14.57   14.19   14.30   15.60   2.33    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([0.408804  , 0.4086858 , 0.40839129, 0.40849249, 0.40840941]),\n",
       " 'test_mae': array([0.34528824, 0.3451166 , 0.34487557, 0.34471672, 0.34482165]),\n",
       " 'fit_time': (434.69273495674133,\n",
       "  418.06540083885193,\n",
       "  419.53682708740234,\n",
       "  380.7255029678345,\n",
       "  415.8733410835266),\n",
       " 'test_time': (20.24527096748352,\n",
       "  14.716097831726074,\n",
       "  14.566580057144165,\n",
       "  14.188155889511108,\n",
       "  14.30232286453247)}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo_svd, rec_data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dvKx_sZ4fAzu"
   },
   "source": [
    "### 6. Recommendation system using SVD++"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "algo_svdpp = SVDpp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = algo_svdpp.fit(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_svdpp = algo_svdpp.test(val_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.3935\n",
      "MSE: 0.1549\n"
     ]
    }
   ],
   "source": [
    "rmse_svdpp = accuracy.rmse(pred_svdpp)\n",
    "mse_svdpp  = accuracy.mse(pred_svdpp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm SVDpp on 5 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
      "RMSE (testset)    0.3982  0.3983  0.3987  0.3983  0.3978  0.3982  0.0003  \n",
      "MAE (testset)     0.3308  0.3309  0.3311  0.3308  0.3303  0.3308  0.0003  \n",
      "Fit time          3123.83 3251.89 3000.32 1269.03 1267.79 2382.57 913.18  \n",
      "Test time         94.87   93.31   34.41   38.26   38.33   59.84   28.01   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([0.39815008, 0.39826962, 0.39866955, 0.39828846, 0.3978436 ]),\n",
       " 'test_mae': array([0.33075743, 0.33090524, 0.33114561, 0.33075276, 0.33028428]),\n",
       " 'fit_time': (3123.8310582637787,\n",
       "  3251.8881237506866,\n",
       "  3000.3195321559906,\n",
       "  1269.0346930027008,\n",
       "  1267.7910070419312),\n",
       " 'test_time': (94.87092518806458,\n",
       "  93.31348299980164,\n",
       "  34.41170001029968,\n",
       "  38.260987997055054,\n",
       "  38.33426833152771)}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo_svdpp, rec_data, measures=['RMSE', 'MAE'], cv=5, verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dvKx_sZ4fAzu"
   },
   "source": [
    "#### 6.1 Using Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_svdpp_test = algo_svdpp.test(test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 0.3978\n",
      "MSE: 0.1583\n"
     ]
    }
   ],
   "source": [
    "rmse_svdpp_test = accuracy.rmse(pred_svdpp_test)\n",
    "mse_svdpp_test  = accuracy.mse(pred_svdpp_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x1f36800a0>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEWCAYAAACEz/viAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZbUlEQVR4nO3de5RlZX3m8e9jI5eA3MRV4WqDMppWMgq9gMS1xgYdaMAAswIOxEijCGMEx5nBiTi6FoyKYjIGB+NliJAGNDYEdWgjDOHSFWcSASEoLbCQ5ibdthC5aXtBG3/zx3lLDkVdTlWdunTX97PWWXX2u9+99+/sOnWes6+VqkKSNL+9YLYLkCTNPsNAkmQYSJIMA0kShoEkCcNAkoRhIEnCMNA8k2RhkkqyRRu+JsmyScxnryQbkizof5XSzDMMNCcleTDJz9sH7iNJlifZrt/LqaojquqSHut5Y9d036+q7arqmX7XNGyZQ+tg6PGX07U8zW+GgeayP6iq7YD9gcXAB7tHpmNzfw//QQudoccZI3Ua2tIZr20sE+2vzcvm/oekzUBVrQOuAV6dZDDJuUn+EfgZsE+SHZJclGR9knVJPjK0+ybJgiT/I8mPktwPHNU97za/d3QNn5rk7iQ/SXJXkv2TXAbsBXytfTv/0xF2N+2WZGWSx5OsSXJq1zzPSXJFkkvbfO9Msngq6yTJyUn+Mcn5SR4Dzhml7QVJPpjkoSSPthp2aPMYeg2nJPk+cONUatKmzTDQnJdkT+BI4PbW9FbgNOBFwEPAcmAj8HLgtcBhwNAH/KnAm1r7YuC4MZZzPHAOcBKwPXA08FhVvRX4Ps9+S/+zESZfAawFdmvL+GiSQ7vGH9367AisBPqxu+cg4H5gADh3lLaT2+MQYB9guxGW/Xrgd4DD+1CTNlVV5cPHnHsADwIbgCfpfOB/BtgGGAQ+1NVvAHga2Kar7URgVXt+I/DOrnGHAQVs0YYHgXe059cC7xmjnjd2DS8cmg+wJ/AM8KKu8R8Dlrfn5wDXd41bBPx8gutg6HFqG3cy8P1h/UdquwF4V9fwK4BftbqHXsM+s/379jH7D/cRai47tqqu725IAvBwV9NLgRcC69s46GzxDvXZbVj/h8ZY3p7AfZOoczfg8ar6ybDldO8K+mHX858BWyfZoqo2jjPv562DLg/30LYbz33ND9EJgoFx5qN5xjDQpqj7vusP09ky2GWUD9b1dD7kh+w1xnwfBl7WwzKH+wGwc5IXdQXCXsC6Mabph5FqGt72AzqBOWQvOrvUHgH2GGM+mmc8ZqBNWlWtB/4e+ESS7dsB05cleX3rcgXwH5PskWQn4KwxZvd54L1JDmhnKr08ydAH6SN09rmPVMPDwD8BH0uydZLfBU4BvtCHlzhVXwL+c5K926m5HwUu72GLRPOMYaDNwUnAlsBdwBPAlcCubdxf0TkW8B3gn4GvjDaTqvpbOgdd/wb4CfC/gZ3b6I8BH0zyZJL3jjD5iXT2wf8A+Cpw9hi7dyZi6AymocdXJzj9xcBlwDeAB4BfAO/uQ13azKTKLURJmu/cMpAkeQBZmi1J9qKza2ski6rq+zNZj+Y3dxNJkjbdLYNddtmlFi5cOKlpf/rTn7Ltttv2t6A+sK6Jsa6Jsa6J2Rzruu22235UVS8ZceRsX/U22ccBBxxQk7Vq1apJTzudrGtirGtirGtiNse6gFtrlM9UDyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJIlN+HYU0ly1et1TnHzW1yc17YPnHdXnaqTeuGUgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJCYRBkgVJbk/yd2147yQ3J1mT5PIkW7b2rdrwmjZ+Ydc83t/a70lyeFf70ta2JslZ/Xt5kqReTGTL4D3A3V3DHwfOr6qXA08Ap7T2U4AnWvv5rR9JFgEnAK8ClgKfaQGzAPg0cASwCDix9ZUkzZCewiDJHsBRwOfbcIBDgStbl0uAY9vzY9owbfwbWv9jgBVV9XRVPQCsAQ5sjzVVdX9V/RJY0fpKkmbIFj32+yTwp8CL2vCLgSeramMbXgvs3p7vDjwMUFUbkzzV+u8O3NQ1z+5pHh7WftBIRSQ5DTgNYGBggMHBwR7Lf64NGzZMetrpZF0TM1frGtgGztxv4/gdRzCdr2euri/rmpjpqmvcMEjyJuDRqrotyZK+VzABVXUhcCHA4sWLa8mSyZUzODjIZKedTtY1MXO1rk998So+sbrX71nP9eBblvS3mC5zdX1Z18RMV129vGNfBxyd5Ehga2B74H8COybZom0d7AGsa/3XAXsCa5NsAewAPNbVPqR7mtHaJUkzYNxjBlX1/qrao6oW0jkAfGNVvQVYBRzXui0DrmrPV7Zh2vgbq6pa+wntbKO9gX2BW4BvAfu2s5O2bMtY2ZdXJ0nqyeS2ZTveB6xI8hHgduCi1n4RcFmSNcDjdD7cqao7k1wB3AVsBE6vqmcAkpwBXAssAC6uqjunUJckaYImFAZVNQgMtuf30zkTaHifXwDHjzL9ucC5I7RfDVw9kVokSf3jFciSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgSWJqt6OQNIcsPOvrY44/c7+NnDxGnwfPO6rfJWkT4paBJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJeDsKSc14t7MYi7ey2PS5ZSBJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJLwCWXOY/+BdmjmGgTSHTOWWENJUuJtIkmQYSJIMA0kSPYRBkq2T3JLkO0nuTPLfW/veSW5OsibJ5Um2bO1bteE1bfzCrnm9v7Xfk+TwrvalrW1NkrP6/zIlSWPpZcvgaeDQqvrXwGuApUkOBj4OnF9VLweeAE5p/U8Bnmjt57d+JFkEnAC8ClgKfCbJgiQLgE8DRwCLgBNbX0nSDBk3DKpjQxt8YXsUcChwZWu/BDi2PT+mDdPGvyFJWvuKqnq6qh4A1gAHtseaqrq/qn4JrGh9JUkzJFU1fqfOt/fbgJfT+Rb/58BN7ds/SfYErqmqVyf5LrC0qta2cfcBBwHntGm+0NovAq5pi1haVe9o7W8FDqqqM0ao4zTgNICBgYEDVqxYMakXvWHDBrbbbrtJTTudrOu5Vq97aszxA9vAIz8fffx+u+/Q54p68+jjT41Z12wZb31NxVTWte/7iZlKXYcccshtVbV4pHE9XWdQVc8Ar0myI/BV4JWTqmSKqupC4EKAxYsX15IlSyY1n8HBQSY77XSyruca64Iy6Fx09onVo7+FH3zLkj5X1JtPffGqMeuaLeOtr6mYyrr2fT8x01XXhM4mqqongVXA7wE7Jhl6Z+0BrGvP1wF7ArTxOwCPdbcPm2a0dknSDBn3a0KSlwC/qqonk2wD/Fs6B4VXAcfR2ce/DLiqTbKyDX+zjb+xqirJSuBvkvwFsBuwL3ALEGDfJHvTCYETgD/q30vUbJrNK2r9B+9S73rZZtwVuKQdN3gBcEVV/V2Su4AVST4C3A5c1PpfBFyWZA3wOJ0Pd6rqziRXAHcBG4HT2+4nkpwBXAssAC6uqjv79golSeMaNwyq6g7gtSO030/nTKDh7b8Ajh9lXucC547QfjVwdQ/1SpKmgVcgS5IMA0mSYSBJwjCQJGEYSJIwDCRJ+G8vJfXBVC7wW7502z5Woslyy0CSZBhIkgwDSRKGgSQJw0CShGEgScIwkCRhGEiS8KIz9WD1uqfG/X/EkjZtbhlIkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJLwrqXSiBZO4S6tZ+7Xx0KkGeKWgSTJMJAkGQaSJAwDSRKGgSSJHsIgyZ5JViW5K8mdSd7T2ndOcl2Se9vPnVp7klyQZE2SO5Ls3zWvZa3/vUmWdbUfkGR1m+aCJJmOFytJGlkvWwYbgTOrahFwMHB6kkXAWcANVbUvcEMbBjgC2Lc9TgM+C53wAM4GDgIOBM4eCpDW59Su6ZZO/aVJkno17nUGVbUeWN+e/yTJ3cDuwDHAktbtEmAQeF9rv7SqCrgpyY5Jdm19r6uqxwGSXAcsTTIIbF9VN7X2S4FjgWv68xIFnjcvaWwTuugsyULgtcDNwEALCoAfAgPt+e7Aw12TrW1tY7WvHaF9pOWfRmdrg4GBAQYHBydS/m9s2LBh0tNOp+ms68z9Nk562oFtpjb9dLGuiZmrdc3Hv8epmK66eg6DJNsBXwb+U1X9uHu3flVVkup7dcNU1YXAhQCLFy+uJUuWTGo+g4ODTHba6TSddZ08pS2DjXxi9dy7WN26Jmau1rV86bbz7u9xKqarrp7eGUleSCcIvlhVX2nNjyTZtarWt91Aj7b2dcCeXZPv0drW8exupaH2wda+xwj9Jc0Dq9c9NekvKw+ed1Sfq5m/ejmbKMBFwN1V9Rddo1YCQ2cELQOu6mo/qZ1VdDDwVNuddC1wWJKd2oHjw4Br27gfJzm4LeukrnlJkmZAL1sGrwPeCqxO8u3W9t+A84ArkpwCPAS8uY27GjgSWAP8DHgbQFU9nuTDwLdavw8NHUwG3gUsB7ahc+DYg8eSNIN6OZvo/wGjnff/hhH6F3D6KPO6GLh4hPZbgVePV4skaXp4BbIkyTCQJBkGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9PY/kDUHLDzr67NdgqTNmFsGkiTDQJJkGEiSMAwkSRgGkiQMA0kShoEkCa8zkLQJm+r1Nw+ed1SfKtn0uWUgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSfQQBkkuTvJoku92te2c5Lok97afO7X2JLkgyZokdyTZv2uaZa3/vUmWdbUfkGR1m+aCJOn3i5Qkja2XLYPlwNJhbWcBN1TVvsANbRjgCGDf9jgN+Cx0wgM4GzgIOBA4eyhAWp9Tu6YbvixJ0jQbNwyq6hvA48OajwEuac8vAY7tar+0Om4CdkyyK3A4cF1VPV5VTwDXAUvbuO2r6qaqKuDSrnlJkmbIZO9NNFBV69vzHwID7fnuwMNd/da2trHa147QPqIkp9HZ4mBgYIDBwcFJFb9hw4ZJTzudxqrrzP02zmwxXQa2md3lj8a6Jsa6nm+sz4FN8XNiKqZ8o7qqqiTVj2J6WNaFwIUAixcvriVLlkxqPoODg0x22uk0Vl0nT/GGXFNx5n4b+cTquXdPQ+uaGOt6vgffsmTUcZvi58RUTPZsokfaLh7az0db+zpgz65+e7S2sdr3GKFdkjSDJhsGK4GhM4KWAVd1tZ/Uzio6GHiq7U66FjgsyU7twPFhwLVt3I+THNzOIjqpa16SpBky7rZZki8BS4Bdkqylc1bQecAVSU4BHgLe3LpfDRwJrAF+BrwNoKoeT/Jh4Fut34eqauig9LvonLG0DXBNe2yWxrv3+pn7bZzV3UGS5q9xw6CqThxl1BtG6FvA6aPM52Lg4hHabwVePV4dkqTp4xXIkiTDQJJkGEiSMAwkSRgGkiQMA0kSfbgdhSRtqsa69me8634ePO+o6Shp1rhlIEkyDCRJhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwruWTthYdzmUpE2VWwaSJMNAkuRuIkmalKnsMp6L/xjHLQNJkmEgSTIMJEkYBpIkDANJEoaBJAnDQJLEPL3OYPW6pzjZ20pI0m/MyzCQpNk0lQvWli/dto+VPMvdRJIkw0CSNIfCIMnSJPckWZPkrNmuR5LmkzkRBkkWAJ8GjgAWAScmWTS7VUnS/DEnwgA4EFhTVfdX1S+BFcAxs1yTJM0bqarZroEkxwFLq+odbfitwEFVdcawfqcBp7XBVwD3THKRuwA/muS008m6Jsa6Jsa6JmZzrOulVfWSkUZsUqeWVtWFwIVTnU+SW6tqcR9K6ivrmhjrmhjrmpj5Vtdc2U20Dtiza3iP1iZJmgFzJQy+BeybZO8kWwInACtnuSZJmjfmxG6iqtqY5AzgWmABcHFV3TmNi5zyrqZpYl0TY10TY10TM6/qmhMHkCVJs2uu7CaSJM0iw0CStPmGQZLjk9yZ5NdJRj0Na7TbYLSD2Te39svbge1+1LVzkuuS3Nt+7jRCn0OSfLvr8Yskx7Zxy5M80DXuNTNVV+v3TNeyV3a1z+b6ek2Sb7bf9x1J/n3XuL6ur/Fum5Jkq/b617T1sbBr3Ptb+z1JDp9KHZOo678kuautnxuSvLRr3Ii/0xmq6+Qk/9K1/Hd0jVvWfu/3Jlk2w3Wd31XT95I82TVuWtZXkouTPJrku6OMT5ILWs13JNm/a9zU11VVbZYP4HfoXJg2CCwepc8C4D5gH2BL4DvAojbuCuCE9vxzwJ/0qa4/A85qz88CPj5O/52Bx4HfasPLgeOmYX31VBewYZT2WVtfwL8C9m3PdwPWAzv2e32N9X7p6vMu4HPt+QnA5e35otZ/K2DvNp8FM1jXIV3voT8Zqmus3+kM1XUy8JcjTLszcH/7uVN7vtNM1TWs/7vpnNQy3evr3wD7A98dZfyRwDVAgIOBm/u5rjbbLYOquruqxrtCecTbYCQJcChwZet3CXBsn0o7ps2v1/keB1xTVT/r0/JHM9G6fmO211dVfa+q7m3PfwA8Cox4leUU9XLblO56rwTe0NbPMcCKqnq6qh4A1rT5zUhdVbWq6z10E51reabbVG4zczhwXVU9XlVPANcBS2eprhOBL/Vp2aOqqm/Q+eI3mmOAS6vjJmDHJLvSp3W12YZBj3YHHu4aXtvaXgw8WVUbh7X3w0BVrW/PfwgMjNP/BJ7/Rjy3bSaen2SrGa5r6yS3JrlpaNcVc2h9JTmQzre9+7qa+7W+Rnu/jNinrY+n6KyfXqadzrq6nULnG+aQkX6nM1nXH7bfz5VJhi4+nRPrq+1O2xu4sat5utbXeEaruy/rak5cZzBZSa4HfnuEUR+oqqtmup4hY9XVPVBVlWTUc3tb6u9H5/qLIe+n86G4JZ3zjd8HfGgG63ppVa1Lsg9wY5LVdD7wJq3P6+syYFlV/bo1T3p9bY6S/DGwGHh9V/PzfqdVdd/Ic+i7rwFfqqqnk/wHOltVh87QsntxAnBlVT3T1Tab62vabNJhUFVvnOIsRrsNxmN0NsG2aN/uJnR7jLHqSvJIkl2ran378Hp0jFm9GfhqVf2qa95D35KfTvLXwHtnsq6qWtd+3p9kEHgt8GVmeX0l2R74Op0vAjd1zXvS62sEvdw2ZajP2iRbADvQeT9N5y1Xepp3kjfSCdjXV9XTQ+2j/E778eE2bl1V9VjX4OfpHCMamnbJsGkH+1BTT3V1OQE4vbthGtfXeEaruy/rar7vJhrxNhjVOSqzis7+eoBlQL+2NFa2+fUy3+ftq2wfiEP76Y8FRjzzYDrqSrLT0G6WJLsArwPumu311X53X6WzP/XKYeP6ub56uW1Kd73HATe29bMSOCGds432BvYFbplCLROqK8lrgf8FHF1Vj3a1j/g7ncG6du0aPBq4uz2/Fjis1bcTcBjP3UKe1rpaba+kc0D2m11t07m+xrMSOKmdVXQw8FT7stOfdTUdR8XnwgP4d3T2nT0NPAJc29p3A67u6nck8D06yf6BrvZ96PyxrgH+FtiqT3W9GLgBuBe4Hti5tS8GPt/VbyGdxH/BsOlvBFbT+VD7ArDdTNUF/H5b9nfaz1PmwvoC/hj4FfDtrsdrpmN9jfR+obPb6ej2fOv2+te09bFP17QfaNPdAxzR5/f7eHVd3/4OhtbPyvF+pzNU18eAO9vyVwGv7Jr27W09rgHeNpN1teFzgPOGTTdt64vOF7/17b28ls6xnXcC72zjQ+efgN3Xlr24a9oprytvRyFJmve7iSRJGAaSJAwDSRKGgSQJw0CSxCZ+0ZnUqyTP0Dkdb8iKqjpvmpb1W8BfAb9L53TAJ4GlVbVhOpYn9YNhoPni51U15u2rkyyortsOdF1RPaYR+r0HeKSq9mvjX0Hn3PFJ67UWabLcTaR5LcmDST6e5J+B45MMJvlkkluB9yRZmOTGPPt/APZq0y1P8rkkN/PsLRSG7ErX7Q2q6p5qt39IclKb13eSXNbaelpGkm3Tuef9LUluT9LrHUClcblloPlimyTf7hr+WFVd3p4/VlX7AyR5J7BlVS1uw18DLqmqS5K8HbiAZ2+jvQfw+/Xcm5gBXAz8fZLj6Fw9fUlV3ZvkVcAH2zQ/SrJz6/+pXpaR5KN0bm/x9iQ7Arckub6qftqPFaT5zSuQNS8k2VBV243Q/iCdG7c91IYHgbOr6h/a8I+AXavqV0leCKyvql2SLAdWVdUlw+fZptuOzj1i3gj8EfB77flvV9UHhvXtaRlta2VrYGh30c7A4VV1N9IUuWUgwfBv1r1+0x61XztY/BXgK0l+TedeOL+cYm0B/rDG/6dN0oR5zEAa2z/RuaslwFuA/zveBEle1+4eOXRH1UXAQ3Rumnd8khe3cUO7iXpdxrXAu9sdWIfuRCr1hVsGmi+GHzP4P1X1vH+EPoJ3A3+d5L8C/wK8rYdpXgZ8tn1ov4DO/1n4clVVknOBf2inut5O538A97qMDwOfBO5I8gLgAeBNPdQjjctjBpIkdxNJkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSgP8PmnzgHpeLhNgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Error Plot - Histogram\n",
    "\n",
    "diff = []\n",
    "for i in range(len(test_set)):\n",
    "    d = test_set[i][2] - pred_svdpp_test[i][3]\n",
    "    diff.append(d)\n",
    "    \n",
    "diff_df = pd.DataFrame(diff, columns = ['Prediction_Error'])\n",
    "d_hist = diff_df.hist(bins=20)\n",
    "plt.xlabel('Error Score')\n",
    "d_hist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dvKx_sZ4fAzu"
   },
   "source": [
    "### 7. Recommendation system Latent Factor Model using Keras (Tensorflow Backend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "<frozen importlib._bootstrap>:219: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.8\n",
      "<frozen importlib._bootstrap>:219: RuntimeWarning: builtins.type size changed, may indicate binary incompatibility. Expected 880, got 864\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, precision_recall_fscore_support, mean_squared_error\n",
    "import gc\n",
    "from sklearn import model_selection\n",
    "from tensorflow.keras import layers, models\n",
    "from tensorflow.keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_players = len(all_data['player'].value_counts())\n",
    "n_games = len(all_data['game'].value_counts())\n",
    "k = 100   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>game</th>\n",
       "      <th>player</th>\n",
       "      <th>score_x</th>\n",
       "      <th>player_id</th>\n",
       "      <th>game_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B6BD-0000-0304-9517</td>\n",
       "      <td>sillyputty2829</td>\n",
       "      <td>1.0</td>\n",
       "      <td>649700</td>\n",
       "      <td>27002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B6BD-0000-0304-9517</td>\n",
       "      <td>hiroshi9438</td>\n",
       "      <td>0.0</td>\n",
       "      <td>151388</td>\n",
       "      <td>27002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B6BD-0000-0304-9517</td>\n",
       "      <td>cagemaster</td>\n",
       "      <td>0.0</td>\n",
       "      <td>719608</td>\n",
       "      <td>27002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B6BD-0000-0304-9517</td>\n",
       "      <td>bentrulock</td>\n",
       "      <td>1.0</td>\n",
       "      <td>823196</td>\n",
       "      <td>27002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B6BD-0000-0304-9517</td>\n",
       "      <td>23Juls</td>\n",
       "      <td>0.0</td>\n",
       "      <td>760086</td>\n",
       "      <td>27002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  game          player  score_x  player_id  game_id\n",
       "0  B6BD-0000-0304-9517  sillyputty2829      1.0     649700    27002\n",
       "1  B6BD-0000-0304-9517     hiroshi9438      0.0     151388    27002\n",
       "2  B6BD-0000-0304-9517      cagemaster      0.0     719608    27002\n",
       "3  B6BD-0000-0304-9517      bentrulock      1.0     823196    27002\n",
       "4  B6BD-0000-0304-9517          23Juls      0.0     760086    27002"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "players_list = set(all_data['player'].values)\n",
    "games_list = set(all_data['game'].values)\n",
    "\n",
    "players_df = pd.DataFrame(enumerate(players_list), columns=['player_id', 'player'])\n",
    "games_df = pd.DataFrame(enumerate(games_list), columns=['game_id', 'game'])\n",
    "\n",
    "train_data_1 = pd.DataFrame(train_data)\n",
    "val_data_1   = pd.DataFrame(val_data)\n",
    "test_data_1  = pd.DataFrame(test_data)\n",
    "\n",
    "train_data_1 = train_data_1.merge(players_df, on='player')\n",
    "train_data_1 = train_data_1.merge(games_df, on='game')\n",
    "val_data_1   = val_data_1.merge(players_df, on='player')\n",
    "val_data_1   = val_data_1.merge(games_df, on='game')\n",
    "test_data_1  = test_data_1.merge(players_df, on='player')\n",
    "test_data_1  = test_data_1.merge(games_df, on='game')\n",
    "\n",
    "train_data_1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_1 = train_data_1.sample(frac=1).reset_index(drop=True)\n",
    "val_data_1   = val_data_1.sample(frac=1).reset_index(drop=True)\n",
    "test_data_1  = test_data_1.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "X_p_train = train_data_1['player_id'].values\n",
    "X_g_train = train_data_1['game_id'].values\n",
    "y_train   = train_data_1['score_x'].values\n",
    "\n",
    "X_p_val = val_data_1['player_id'].values\n",
    "X_g_val = val_data_1['game_id'].values\n",
    "y_val   = val_data_1['score_x'].values\n",
    "\n",
    "X_p_test = test_data_1['player_id'].values\n",
    "X_g_test = test_data_1['game_id'].values\n",
    "y_test   = test_data_1['score_x'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# custom metric - RMSE\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "def rmse(y_true, y_pred):\n",
    "        return K.sqrt(K.mean(K.square(y_pred - y_true))) \n",
    "\n",
    "pl_input = layers.Input((1,))\n",
    "gm_input = layers.Input((1,))\n",
    "\n",
    "pl_embedding = layers.Embedding(n_players, k)(pl_input)\n",
    "gm_embedding = layers.Embedding(n_games, k)(gm_input)\n",
    "\n",
    "dot_product = layers.Dot(2)([pl_embedding, gm_embedding])\n",
    "sig_act = layers.Activation('sigmoid')(dot_product)\n",
    "score_output = layers.Reshape((1,))(sig_act)\n",
    "c_f_model = models.Model([pl_input, gm_input], score_output)\n",
    "\n",
    "c_f_model.compile(loss='binary_crossentropy', optimizer='RMSProp', metrics=['acc', rmse])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3270525 samples, validate on 408816 samples\n",
      "Epoch 1/125\n",
      "3270525/3270525 [==============================] - 95s 29us/step - loss: 0.6932 - acc: 0.5004 - rmse: 0.5000 - val_loss: 0.6932 - val_acc: 0.4996 - val_rmse: 0.5000\n",
      "Epoch 2/125\n",
      "3270525/3270525 [==============================] - 93s 28us/step - loss: 0.6894 - acc: 0.7830 - rmse: 0.4981 - val_loss: 0.6931 - val_acc: 0.5010 - val_rmse: 0.5000\n",
      "Epoch 3/125\n",
      "3270525/3270525 [==============================] - 115s 35us/step - loss: 0.6849 - acc: 0.9164 - rmse: 0.4958 - val_loss: 0.6930 - val_acc: 0.5040 - val_rmse: 0.4999\n",
      "Epoch 4/125\n",
      "3270525/3270525 [==============================] - 113s 35us/step - loss: 0.6785 - acc: 0.9642 - rmse: 0.4926 - val_loss: 0.6928 - val_acc: 0.5080 - val_rmse: 0.4998\n",
      "Epoch 5/125\n",
      "3270525/3270525 [==============================] - 108s 33us/step - loss: 0.6696 - acc: 0.9812 - rmse: 0.4881 - val_loss: 0.6922 - val_acc: 0.5120 - val_rmse: 0.4995\n",
      "Epoch 6/125\n",
      "3270525/3270525 [==============================] - 108s 33us/step - loss: 0.6575 - acc: 0.9881 - rmse: 0.4819 - val_loss: 0.6913 - val_acc: 0.5168 - val_rmse: 0.4991\n",
      "Epoch 7/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.6420 - acc: 0.9914 - rmse: 0.4738 - val_loss: 0.6899 - val_acc: 0.5209 - val_rmse: 0.4984\n",
      "Epoch 8/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 0.6229 - acc: 0.9932 - rmse: 0.4637 - val_loss: 0.6881 - val_acc: 0.5250 - val_rmse: 0.4975\n",
      "Epoch 9/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.6003 - acc: 0.9942 - rmse: 0.4517 - val_loss: 0.6860 - val_acc: 0.5296 - val_rmse: 0.4966\n",
      "Epoch 10/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 0.5746 - acc: 0.9949 - rmse: 0.4377 - val_loss: 0.6837 - val_acc: 0.5337 - val_rmse: 0.4955\n",
      "Epoch 11/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 0.5462 - acc: 0.9953 - rmse: 0.4219 - val_loss: 0.6813 - val_acc: 0.5374 - val_rmse: 0.4945\n",
      "Epoch 12/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 0.5155 - acc: 0.9956 - rmse: 0.4046 - val_loss: 0.6790 - val_acc: 0.5405 - val_rmse: 0.4934\n",
      "Epoch 13/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 0.4832 - acc: 0.9960 - rmse: 0.3858 - val_loss: 0.6769 - val_acc: 0.5437 - val_rmse: 0.4925\n",
      "Epoch 14/125\n",
      "3270525/3270525 [==============================] - 87s 27us/step - loss: 0.4497 - acc: 0.9962 - rmse: 0.3660 - val_loss: 0.6750 - val_acc: 0.5467 - val_rmse: 0.4916\n",
      "Epoch 15/125\n",
      "3270525/3270525 [==============================] - 86s 26us/step - loss: 0.4158 - acc: 0.9965 - rmse: 0.3452 - val_loss: 0.6733 - val_acc: 0.5495 - val_rmse: 0.4909\n",
      "Epoch 16/125\n",
      "3270525/3270525 [==============================] - 86s 26us/step - loss: 0.3818 - acc: 0.9968 - rmse: 0.3238 - val_loss: 0.6720 - val_acc: 0.5521 - val_rmse: 0.4903\n",
      "Epoch 17/125\n",
      "3270525/3270525 [==============================] - 86s 26us/step - loss: 0.3483 - acc: 0.9971 - rmse: 0.3022 - val_loss: 0.6711 - val_acc: 0.5544 - val_rmse: 0.4899\n",
      "Epoch 18/125\n",
      "3270525/3270525 [==============================] - 86s 26us/step - loss: 0.3157 - acc: 0.9974 - rmse: 0.2804 - val_loss: 0.6706 - val_acc: 0.5564 - val_rmse: 0.4896\n",
      "Epoch 19/125\n",
      "3270525/3270525 [==============================] - 86s 26us/step - loss: 0.2844 - acc: 0.9976 - rmse: 0.2589 - val_loss: 0.6706 - val_acc: 0.5584 - val_rmse: 0.4895\n",
      "Epoch 20/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.2547 - acc: 0.9979 - rmse: 0.2378 - val_loss: 0.6710 - val_acc: 0.5601 - val_rmse: 0.4895\n",
      "Epoch 21/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.2268 - acc: 0.9981 - rmse: 0.2174 - val_loss: 0.6719 - val_acc: 0.5618 - val_rmse: 0.4897\n",
      "Epoch 22/125\n",
      "3270525/3270525 [==============================] - 87s 27us/step - loss: 0.2010 - acc: 0.9983 - rmse: 0.1978 - val_loss: 0.6733 - val_acc: 0.5632 - val_rmse: 0.4900\n",
      "Epoch 23/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.1773 - acc: 0.9986 - rmse: 0.1793 - val_loss: 0.6751 - val_acc: 0.5645 - val_rmse: 0.4905\n",
      "Epoch 24/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.1557 - acc: 0.9987 - rmse: 0.1618 - val_loss: 0.6774 - val_acc: 0.5657 - val_rmse: 0.4910\n",
      "Epoch 25/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.1362 - acc: 0.9989 - rmse: 0.1454 - val_loss: 0.6801 - val_acc: 0.5668 - val_rmse: 0.4917\n",
      "Epoch 26/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.1188 - acc: 0.9991 - rmse: 0.1303 - val_loss: 0.6833 - val_acc: 0.5678 - val_rmse: 0.4925\n",
      "Epoch 27/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.1033 - acc: 0.9992 - rmse: 0.1165 - val_loss: 0.6868 - val_acc: 0.5687 - val_rmse: 0.4934\n",
      "Epoch 28/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0896 - acc: 0.9993 - rmse: 0.1038 - val_loss: 0.6907 - val_acc: 0.5695 - val_rmse: 0.4944\n",
      "Epoch 29/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0776 - acc: 0.9994 - rmse: 0.0923 - val_loss: 0.6949 - val_acc: 0.5702 - val_rmse: 0.4954\n",
      "Epoch 30/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0672 - acc: 0.9995 - rmse: 0.0820 - val_loss: 0.6993 - val_acc: 0.5708 - val_rmse: 0.4964\n",
      "Epoch 31/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0581 - acc: 0.9996 - rmse: 0.0727 - val_loss: 0.7039 - val_acc: 0.5713 - val_rmse: 0.4975\n",
      "Epoch 32/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0502 - acc: 0.9996 - rmse: 0.0645 - val_loss: 0.7088 - val_acc: 0.5717 - val_rmse: 0.4985\n",
      "Epoch 33/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0435 - acc: 0.9997 - rmse: 0.0571 - val_loss: 0.7138 - val_acc: 0.5721 - val_rmse: 0.4996\n",
      "Epoch 34/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0377 - acc: 0.9997 - rmse: 0.0506 - val_loss: 0.7188 - val_acc: 0.5726 - val_rmse: 0.5007\n",
      "Epoch 35/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0327 - acc: 0.9998 - rmse: 0.0449 - val_loss: 0.7239 - val_acc: 0.5732 - val_rmse: 0.5018\n",
      "Epoch 36/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0285 - acc: 0.9998 - rmse: 0.0398 - val_loss: 0.7291 - val_acc: 0.5735 - val_rmse: 0.5028\n",
      "Epoch 37/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0248 - acc: 0.9999 - rmse: 0.0354 - val_loss: 0.7342 - val_acc: 0.5738 - val_rmse: 0.5038\n",
      "Epoch 38/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0217 - acc: 0.9999 - rmse: 0.0315 - val_loss: 0.7393 - val_acc: 0.5740 - val_rmse: 0.5048\n",
      "Epoch 39/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0190 - acc: 0.9999 - rmse: 0.0280 - val_loss: 0.7443 - val_acc: 0.5743 - val_rmse: 0.5058\n",
      "Epoch 40/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0168 - acc: 0.9999 - rmse: 0.0250 - val_loss: 0.7493 - val_acc: 0.5745 - val_rmse: 0.5067\n",
      "Epoch 41/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0148 - acc: 0.9999 - rmse: 0.0224 - val_loss: 0.7542 - val_acc: 0.5749 - val_rmse: 0.5075\n",
      "Epoch 42/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0131 - acc: 0.9999 - rmse: 0.0201 - val_loss: 0.7589 - val_acc: 0.5750 - val_rmse: 0.5084\n",
      "Epoch 43/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0117 - acc: 0.9999 - rmse: 0.0180 - val_loss: 0.7635 - val_acc: 0.5752 - val_rmse: 0.5092\n",
      "Epoch 44/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0105 - acc: 1.0000 - rmse: 0.0162 - val_loss: 0.7681 - val_acc: 0.5754 - val_rmse: 0.5100\n",
      "Epoch 45/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0094 - acc: 1.0000 - rmse: 0.0146 - val_loss: 0.7725 - val_acc: 0.5755 - val_rmse: 0.5107\n",
      "Epoch 46/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0085 - acc: 1.0000 - rmse: 0.0132 - val_loss: 0.7767 - val_acc: 0.5757 - val_rmse: 0.5114\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0076 - acc: 1.0000 - rmse: 0.0120 - val_loss: 0.7809 - val_acc: 0.5757 - val_rmse: 0.5121\n",
      "Epoch 48/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0069 - acc: 1.0000 - rmse: 0.0109 - val_loss: 0.7849 - val_acc: 0.5758 - val_rmse: 0.5127\n",
      "Epoch 49/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0063 - acc: 1.0000 - rmse: 0.0099 - val_loss: 0.7887 - val_acc: 0.5760 - val_rmse: 0.5133\n",
      "Epoch 50/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0058 - acc: 1.0000 - rmse: 0.0091 - val_loss: 0.7925 - val_acc: 0.5760 - val_rmse: 0.5139\n",
      "Epoch 51/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0053 - acc: 1.0000 - rmse: 0.0083 - val_loss: 0.7961 - val_acc: 0.5761 - val_rmse: 0.5144\n",
      "Epoch 52/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0049 - acc: 1.0000 - rmse: 0.0076 - val_loss: 0.7996 - val_acc: 0.5762 - val_rmse: 0.5149\n",
      "Epoch 53/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0045 - acc: 1.0000 - rmse: 0.0070 - val_loss: 0.8030 - val_acc: 0.5763 - val_rmse: 0.5154\n",
      "Epoch 54/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0042 - acc: 1.0000 - rmse: 0.0064 - val_loss: 0.8063 - val_acc: 0.5764 - val_rmse: 0.5159\n",
      "Epoch 55/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 0.0039 - acc: 1.0000 - rmse: 0.0059 - val_loss: 0.8094 - val_acc: 0.5764 - val_rmse: 0.5164\n",
      "Epoch 56/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0036 - acc: 1.0000 - rmse: 0.0055 - val_loss: 0.8125 - val_acc: 0.5765 - val_rmse: 0.5168\n",
      "Epoch 57/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0034 - acc: 1.0000 - rmse: 0.0051 - val_loss: 0.8154 - val_acc: 0.5766 - val_rmse: 0.5172\n",
      "Epoch 58/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0032 - acc: 1.0000 - rmse: 0.0047 - val_loss: 0.8183 - val_acc: 0.5767 - val_rmse: 0.5176\n",
      "Epoch 59/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0030 - acc: 1.0000 - rmse: 0.0044 - val_loss: 0.8211 - val_acc: 0.5768 - val_rmse: 0.5180\n",
      "Epoch 60/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0028 - acc: 1.0000 - rmse: 0.0041 - val_loss: 0.8238 - val_acc: 0.5769 - val_rmse: 0.5183\n",
      "Epoch 61/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0026 - acc: 1.0000 - rmse: 0.0038 - val_loss: 0.8264 - val_acc: 0.5769 - val_rmse: 0.5187\n",
      "Epoch 62/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0025 - acc: 1.0000 - rmse: 0.0036 - val_loss: 0.8289 - val_acc: 0.5770 - val_rmse: 0.5190\n",
      "Epoch 63/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0024 - acc: 1.0000 - rmse: 0.0034 - val_loss: 0.8313 - val_acc: 0.5770 - val_rmse: 0.5194\n",
      "Epoch 64/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0022 - acc: 1.0000 - rmse: 0.0032 - val_loss: 0.8337 - val_acc: 0.5771 - val_rmse: 0.5197\n",
      "Epoch 65/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0021 - acc: 1.0000 - rmse: 0.0030 - val_loss: 0.8360 - val_acc: 0.5771 - val_rmse: 0.5200\n",
      "Epoch 66/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0020 - acc: 1.0000 - rmse: 0.0029 - val_loss: 0.8382 - val_acc: 0.5772 - val_rmse: 0.5203\n",
      "Epoch 67/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0019 - acc: 1.0000 - rmse: 0.0027 - val_loss: 0.8404 - val_acc: 0.5772 - val_rmse: 0.5205\n",
      "Epoch 68/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0019 - acc: 1.0000 - rmse: 0.0026 - val_loss: 0.8425 - val_acc: 0.5773 - val_rmse: 0.5208\n",
      "Epoch 69/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0018 - acc: 1.0000 - rmse: 0.0024 - val_loss: 0.8446 - val_acc: 0.5774 - val_rmse: 0.5211\n",
      "Epoch 70/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 0.0017 - acc: 1.0000 - rmse: 0.0023 - val_loss: 0.8466 - val_acc: 0.5774 - val_rmse: 0.5213\n",
      "Epoch 71/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 0.0016 - acc: 1.0000 - rmse: 0.0022 - val_loss: 0.8485 - val_acc: 0.5774 - val_rmse: 0.5215\n",
      "Epoch 72/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0016 - acc: 1.0000 - rmse: 0.0021 - val_loss: 0.8504 - val_acc: 0.5775 - val_rmse: 0.5218\n",
      "Epoch 73/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0015 - acc: 1.0000 - rmse: 0.0020 - val_loss: 0.8523 - val_acc: 0.5775 - val_rmse: 0.5220\n",
      "Epoch 74/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0014 - acc: 1.0000 - rmse: 0.0020 - val_loss: 0.8541 - val_acc: 0.5775 - val_rmse: 0.5222\n",
      "Epoch 75/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0014 - acc: 1.0000 - rmse: 0.0019 - val_loss: 0.8558 - val_acc: 0.5776 - val_rmse: 0.5224\n",
      "Epoch 76/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0013 - acc: 1.0000 - rmse: 0.0018 - val_loss: 0.8575 - val_acc: 0.5777 - val_rmse: 0.5226\n",
      "Epoch 77/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0013 - acc: 1.0000 - rmse: 0.0017 - val_loss: 0.8592 - val_acc: 0.5777 - val_rmse: 0.5228\n",
      "Epoch 78/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 0.0013 - acc: 1.0000 - rmse: 0.0017 - val_loss: 0.8608 - val_acc: 0.5777 - val_rmse: 0.5230\n",
      "Epoch 79/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 0.0012 - acc: 1.0000 - rmse: 0.0016 - val_loss: 0.8624 - val_acc: 0.5778 - val_rmse: 0.5232\n",
      "Epoch 80/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 0.0012 - acc: 1.0000 - rmse: 0.0016 - val_loss: 0.8640 - val_acc: 0.5778 - val_rmse: 0.5234\n",
      "Epoch 81/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 0.0011 - acc: 1.0000 - rmse: 0.0015 - val_loss: 0.8655 - val_acc: 0.5779 - val_rmse: 0.5236\n",
      "Epoch 82/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 0.0011 - acc: 1.0000 - rmse: 0.0015 - val_loss: 0.8670 - val_acc: 0.5779 - val_rmse: 0.5238\n",
      "Epoch 83/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 0.0011 - acc: 1.0000 - rmse: 0.0014 - val_loss: 0.8684 - val_acc: 0.5780 - val_rmse: 0.5239\n",
      "Epoch 84/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 0.0010 - acc: 1.0000 - rmse: 0.0014 - val_loss: 0.8699 - val_acc: 0.5780 - val_rmse: 0.5241\n",
      "Epoch 85/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 0.0010 - acc: 1.0000 - rmse: 0.0013 - val_loss: 0.8713 - val_acc: 0.5780 - val_rmse: 0.5243\n",
      "Epoch 86/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 9.7773e-04 - acc: 1.0000 - rmse: 0.0013 - val_loss: 0.8726 - val_acc: 0.5780 - val_rmse: 0.5244\n",
      "Epoch 87/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 9.5088e-04 - acc: 1.0000 - rmse: 0.0013 - val_loss: 0.8740 - val_acc: 0.5781 - val_rmse: 0.5246\n",
      "Epoch 88/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 9.2533e-04 - acc: 1.0000 - rmse: 0.0012 - val_loss: 0.8753 - val_acc: 0.5781 - val_rmse: 0.5247\n",
      "Epoch 89/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 9.0096e-04 - acc: 1.0000 - rmse: 0.0012 - val_loss: 0.8766 - val_acc: 0.5781 - val_rmse: 0.5249\n",
      "Epoch 90/125\n",
      "3270525/3270525 [==============================] - 90s 28us/step - loss: 8.7772e-04 - acc: 1.0000 - rmse: 0.0012 - val_loss: 0.8779 - val_acc: 0.5781 - val_rmse: 0.5250\n",
      "Epoch 91/125\n",
      "3270525/3270525 [==============================] - 91s 28us/step - loss: 8.5553e-04 - acc: 1.0000 - rmse: 0.0011 - val_loss: 0.8791 - val_acc: 0.5782 - val_rmse: 0.5252\n",
      "Epoch 92/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 8.3433e-04 - acc: 1.0000 - rmse: 0.0011 - val_loss: 0.8803 - val_acc: 0.5782 - val_rmse: 0.5253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 8.1404e-04 - acc: 1.0000 - rmse: 0.0011 - val_loss: 0.8815 - val_acc: 0.5782 - val_rmse: 0.5254\n",
      "Epoch 94/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 7.9463e-04 - acc: 1.0000 - rmse: 0.0010 - val_loss: 0.8827 - val_acc: 0.5783 - val_rmse: 0.5256\n",
      "Epoch 95/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 7.7603e-04 - acc: 1.0000 - rmse: 0.0010 - val_loss: 0.8839 - val_acc: 0.5782 - val_rmse: 0.5257\n",
      "Epoch 96/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 7.5819e-04 - acc: 1.0000 - rmse: 9.9315e-04 - val_loss: 0.8850 - val_acc: 0.5782 - val_rmse: 0.5258\n",
      "Epoch 97/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 7.4109e-04 - acc: 1.0000 - rmse: 9.6988e-04 - val_loss: 0.8861 - val_acc: 0.5783 - val_rmse: 0.5259\n",
      "Epoch 98/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 7.2466e-04 - acc: 1.0000 - rmse: 9.4757e-04 - val_loss: 0.8872 - val_acc: 0.5783 - val_rmse: 0.5261\n",
      "Epoch 99/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 7.0888e-04 - acc: 1.0000 - rmse: 9.2617e-04 - val_loss: 0.8883 - val_acc: 0.5784 - val_rmse: 0.5262\n",
      "Epoch 100/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.9371e-04 - acc: 1.0000 - rmse: 9.0562e-04 - val_loss: 0.8893 - val_acc: 0.5784 - val_rmse: 0.5263\n",
      "Epoch 101/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 6.7911e-04 - acc: 1.0000 - rmse: 8.8588e-04 - val_loss: 0.8904 - val_acc: 0.5784 - val_rmse: 0.5264\n",
      "Epoch 102/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.6506e-04 - acc: 1.0000 - rmse: 8.6690e-04 - val_loss: 0.8914 - val_acc: 0.5785 - val_rmse: 0.5265\n",
      "Epoch 103/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.5153e-04 - acc: 1.0000 - rmse: 8.4864e-04 - val_loss: 0.8924 - val_acc: 0.5785 - val_rmse: 0.5266\n",
      "Epoch 104/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 6.3849e-04 - acc: 1.0000 - rmse: 8.3106e-04 - val_loss: 0.8934 - val_acc: 0.5785 - val_rmse: 0.5268\n",
      "Epoch 105/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.2591e-04 - acc: 1.0000 - rmse: 8.1413e-04 - val_loss: 0.8944 - val_acc: 0.5786 - val_rmse: 0.5269\n",
      "Epoch 106/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.1377e-04 - acc: 1.0000 - rmse: 7.9781e-04 - val_loss: 0.8954 - val_acc: 0.5786 - val_rmse: 0.5270\n",
      "Epoch 107/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 6.0206e-04 - acc: 1.0000 - rmse: 7.8206e-04 - val_loss: 0.8963 - val_acc: 0.5786 - val_rmse: 0.5271\n",
      "Epoch 108/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.9074e-04 - acc: 1.0000 - rmse: 7.6688e-04 - val_loss: 0.8973 - val_acc: 0.5786 - val_rmse: 0.5272\n",
      "Epoch 109/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.7980e-04 - acc: 1.0000 - rmse: 7.5221e-04 - val_loss: 0.8982 - val_acc: 0.5786 - val_rmse: 0.5273\n",
      "Epoch 110/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.6923e-04 - acc: 1.0000 - rmse: 7.3805e-04 - val_loss: 0.8991 - val_acc: 0.5786 - val_rmse: 0.5274\n",
      "Epoch 111/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.5900e-04 - acc: 1.0000 - rmse: 7.2436e-04 - val_loss: 0.9000 - val_acc: 0.5786 - val_rmse: 0.5275\n",
      "Epoch 112/125\n",
      "3270525/3270525 [==============================] - 88s 27us/step - loss: 5.4910e-04 - acc: 1.0000 - rmse: 7.1112e-04 - val_loss: 0.9009 - val_acc: 0.5786 - val_rmse: 0.5276\n",
      "Epoch 113/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.3952e-04 - acc: 1.0000 - rmse: 6.9831e-04 - val_loss: 0.9018 - val_acc: 0.5786 - val_rmse: 0.5276\n",
      "Epoch 114/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.3023e-04 - acc: 1.0000 - rmse: 6.8593e-04 - val_loss: 0.9026 - val_acc: 0.5786 - val_rmse: 0.5277\n",
      "Epoch 115/125\n",
      "3270525/3270525 [==============================] - 90s 27us/step - loss: 5.2123e-04 - acc: 1.0000 - rmse: 6.7393e-04 - val_loss: 0.9035 - val_acc: 0.5787 - val_rmse: 0.5278\n",
      "Epoch 116/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.1251e-04 - acc: 1.0000 - rmse: 6.6230e-04 - val_loss: 0.9043 - val_acc: 0.5787 - val_rmse: 0.5279\n",
      "Epoch 117/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 5.0405e-04 - acc: 1.0000 - rmse: 6.5104e-04 - val_loss: 0.9052 - val_acc: 0.5787 - val_rmse: 0.5280\n",
      "Epoch 118/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.9584e-04 - acc: 1.0000 - rmse: 6.4011e-04 - val_loss: 0.9060 - val_acc: 0.5787 - val_rmse: 0.5281\n",
      "Epoch 119/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.8787e-04 - acc: 1.0000 - rmse: 6.2952e-04 - val_loss: 0.9068 - val_acc: 0.5787 - val_rmse: 0.5282\n",
      "Epoch 120/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.8014e-04 - acc: 1.0000 - rmse: 6.1924e-04 - val_loss: 0.9076 - val_acc: 0.5787 - val_rmse: 0.5283\n",
      "Epoch 121/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.7262e-04 - acc: 1.0000 - rmse: 6.0926e-04 - val_loss: 0.9084 - val_acc: 0.5787 - val_rmse: 0.5283\n",
      "Epoch 122/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.6532e-04 - acc: 1.0000 - rmse: 5.9957e-04 - val_loss: 0.9092 - val_acc: 0.5788 - val_rmse: 0.5284\n",
      "Epoch 123/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.5822e-04 - acc: 1.0000 - rmse: 5.9016e-04 - val_loss: 0.9099 - val_acc: 0.5788 - val_rmse: 0.5285\n",
      "Epoch 124/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.5131e-04 - acc: 1.0000 - rmse: 5.8101e-04 - val_loss: 0.9107 - val_acc: 0.5788 - val_rmse: 0.5286\n",
      "Epoch 125/125\n",
      "3270525/3270525 [==============================] - 89s 27us/step - loss: 4.4460e-04 - acc: 1.0000 - rmse: 5.7212e-04 - val_loss: 0.9114 - val_acc: 0.5788 - val_rmse: 0.5287\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x2009630d0>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c_f_model.fit(x=[X_p_train,X_g_train], y=y_train, validation_data=[[X_p_val, X_g_val], y_val], batch_size=32000, epochs=125)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "408816/408816 [==============================] - 1s 2us/step\n",
      "Root Mean Squared Error (RMSE) on test data : 0.5287\n"
     ]
    }
   ],
   "source": [
    "test_pred = c_f_model.predict(x=[X_p_val, X_g_val], verbose=1, batch_size=4096)\n",
    "\n",
    "RMSE = np.sqrt(mean_squared_error(y_val, test_pred))\n",
    "print('Root Mean Squared Error (RMSE) on test data : {0:5.4f}'.format(RMSE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dvKx_sZ4fAzu"
   },
   "source": [
    "### 8. Making Recommendations \n",
    "\n",
    "Predictions !!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115032/115032 [==============================] - 1s 9us/step\n",
      "115032/115032 [==============================] - 1s 7us/step\n",
      "115032/115032 [==============================] - 1s 6us/step\n",
      "115032/115032 [==============================] - 1s 6us/step\n",
      "115032/115032 [==============================] - 1s 6us/step\n",
      "\n",
      "\n",
      "\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Player: AnimateCorpse\n",
      "Top-5 Recommended games with probability of clearing:\n",
      "[['7410-0000-0364-AFCB', 0.9983039], ['7062-0000-0302-FD6F', 0.9979693], ['1F40-0000-032E-FA61', 0.99760604], ['C3F6-0000-0351-6289', 0.9970112], ['1D4D-0000-0354-2B03', 0.9965179]]\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Player: hayatotibi\n",
      "Top-5 Recommended games with probability of clearing:\n",
      "[['8A4C-0000-0300-31F2', 0.99961215], ['FD60-0000-0342-AE39', 0.9995788], ['DA76-0000-0330-0E93', 0.9995389], ['E3BA-0000-034E-9813', 0.999368], ['C22E-0000-035C-33EE', 0.99933213]]\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Player: ravelomananarsmr\n",
      "Top-5 Recommended games with probability of clearing:\n",
      "[['70BE-0000-0336-91E8', 0.9999962], ['410A-0000-0376-DED3', 0.99997985], ['3798-0000-0354-38ED', 0.9999676], ['19FE-0000-035A-0830', 0.9999496], ['8CC2-0000-032A-2336', 0.9999486]]\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Player: Tohshi1022\n",
      "Top-5 Recommended games with probability of clearing:\n",
      "[['61B7-0000-0357-B8F2', 0.99356925], ['3CA2-0000-0324-D18A', 0.99153113], ['4EA6-0000-036B-C98A', 0.9913846], ['2D69-0000-02FF-601C', 0.98432565], ['7793-0000-0340-7D5F', 0.98331416]]\n",
      "-------------------------------------------------------------------------------------------------------------------------------------\n",
      "Player: Candice1412\n",
      "Top-5 Recommended games with probability of clearing:\n",
      "[['67CF-0000-035F-BC1D', 0.7191662], ['4777-0000-038D-B8EE', 0.7008692], ['9C75-0000-0362-33D2', 0.6976036], ['F34E-0000-0310-66E0', 0.6783065], ['EEFB-0000-0349-8A66', 0.67741203]]\n"
     ]
    }
   ],
   "source": [
    "import random \n",
    "\n",
    "player_ids = random.sample(range(n_players), 5)\n",
    "\n",
    "recommendations = dict()\n",
    "\n",
    "game_list = list(range(n_games))\n",
    "\n",
    "for player in player_ids:\n",
    "    player_list = [player]*n_games\n",
    "    scores = c_f_model.predict(x=[player_list, game_list], verbose=1)\n",
    "    temp_df = games_df.copy()\n",
    "    temp_df['score_x'] = scores\n",
    "    temp2_df = all_data[all_data['player'] == players_df.loc[player, 'player']]\n",
    "    temp_df = temp_df.merge(temp2_df, how='left', on='game')\n",
    "    temp_df = temp_df[temp_df['score_x_y']!=0]\n",
    "    temp_df = temp_df[temp_df['score_x_y']!=1]\n",
    "    temp_df.sort_values(by='score_x_x', axis=0, ascending=False, inplace=True)\n",
    "    temp_df.reset_index(drop=True, inplace=True)\n",
    "    lst = []\n",
    "\n",
    "    for m in range(5):\n",
    "        lst.append([temp_df.loc[m, 'game'], temp_df.loc[m, 'score_x_x']])\n",
    "        recommendations[players_df.loc[player, 'player']] = lst\n",
    "\n",
    "print('\\n\\n')\n",
    "for player in recommendations.keys():\n",
    "    print('-------------------------------------------------------------------------------------------------------------------------------------')\n",
    "    print('Player: {}'.format(player))\n",
    "    print('Top-5 Recommended games with probability of clearing:')\n",
    "    print(recommendations[player])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SMM_analysis_2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}