EDA_Phase_(Section_2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "baking-skating",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "substantial-blackberry",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"E:/my_courses/data mining/project/data_genre/\"\n",
    "path = \"C:/Users/User/Downloads/test2/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "stylish-angel",
   "metadata": {},
   "source": [
    "# 3️⃣ EDA Phase : Section 2\n",
    "<hr style=\"border:2px solid gray\"> </hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "assured-miniature",
   "metadata": {},
   "source": [
    "## 📑 PCA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "tested-mortgage",
   "metadata": {},
   "source": [
    "### ✅ PCA on book columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "separate-victory",
   "metadata": {},
   "source": [
    "<div style=\"direction:rtl\", align=\"justify\">\n",
    "<h3>\n",
    "<span style='font-family:\"B Nazanin\"'>\n",
    "با توجه به heatmap بدست آمده در مرحله EDA متوجه شدیم که بسیاری از ستون ها درون دیتاست book حاوی همبستگی قوی با یکدیگر هستند و لذا برای جلوگیری از multicolinearity و  نیز کاهش ابعاد ورودی مدل از pca استفاده کرده و با بررسی میزان پوشش در هر گام بهترین تعداد کامپوننت را استخراج می کنیم. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "partial-vancouver",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>book_id</th>\n",
       "      <th>original_publication_year</th>\n",
       "      <th>average_rating</th>\n",
       "      <th>ratings_count</th>\n",
       "      <th>work_ratings_count</th>\n",
       "      <th>work_text_reviews_count</th>\n",
       "      <th>ratings_1</th>\n",
       "      <th>ratings_2</th>\n",
       "      <th>ratings_3</th>\n",
       "      <th>ratings_4</th>\n",
       "      <th>...</th>\n",
       "      <th>en</th>\n",
       "      <th>fre</th>\n",
       "      <th>ger</th>\n",
       "      <th>ind</th>\n",
       "      <th>jpn</th>\n",
       "      <th>other</th>\n",
       "      <th>per</th>\n",
       "      <th>pol</th>\n",
       "      <th>por</th>\n",
       "      <th>spa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>4.34</td>\n",
       "      <td>4780653</td>\n",
       "      <td>4942365</td>\n",
       "      <td>155254</td>\n",
       "      <td>66715</td>\n",
       "      <td>127936</td>\n",
       "      <td>560092</td>\n",
       "      <td>1481305</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1997.0</td>\n",
       "      <td>4.44</td>\n",
       "      <td>4602479</td>\n",
       "      <td>4800065</td>\n",
       "      <td>75867</td>\n",
       "      <td>75504</td>\n",
       "      <td>101676</td>\n",
       "      <td>455024</td>\n",
       "      <td>1156318</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>3.57</td>\n",
       "      <td>3866839</td>\n",
       "      <td>3916824</td>\n",
       "      <td>95009</td>\n",
       "      <td>456191</td>\n",
       "      <td>436802</td>\n",
       "      <td>793319</td>\n",
       "      <td>875073</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>4.25</td>\n",
       "      <td>3198671</td>\n",
       "      <td>3340896</td>\n",
       "      <td>72586</td>\n",
       "      <td>60427</td>\n",
       "      <td>117415</td>\n",
       "      <td>446835</td>\n",
       "      <td>1001952</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>3.89</td>\n",
       "      <td>2683664</td>\n",
       "      <td>2773745</td>\n",
       "      <td>51992</td>\n",
       "      <td>86236</td>\n",
       "      <td>197621</td>\n",
       "      <td>606158</td>\n",
       "      <td>936012</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   book_id  original_publication_year  average_rating  ratings_count  \\\n",
       "0        1                     2008.0            4.34        4780653   \n",
       "1        2                     1997.0            4.44        4602479   \n",
       "2        3                     2005.0            3.57        3866839   \n",
       "3        4                     1960.0            4.25        3198671   \n",
       "4        5                     1925.0            3.89        2683664   \n",
       "\n",
       "   work_ratings_count  work_text_reviews_count  ratings_1  ratings_2  \\\n",
       "0             4942365                   155254      66715     127936   \n",
       "1             4800065                    75867      75504     101676   \n",
       "2             3916824                    95009     456191     436802   \n",
       "3             3340896                    72586      60427     117415   \n",
       "4             2773745                    51992      86236     197621   \n",
       "\n",
       "   ratings_3  ratings_4  ...  en  fre  ger  ind  jpn  other  per  pol  por  \\\n",
       "0     560092    1481305  ...   1    0    0    0    0      0    0    0    0   \n",
       "1     455024    1156318  ...   1    0    0    0    0      0    0    0    0   \n",
       "2     793319     875073  ...   1    0    0    0    0      0    0    0    0   \n",
       "3     446835    1001952  ...   1    0    0    0    0      0    0    0    0   \n",
       "4     606158     936012  ...   1    0    0    0    0      0    0    0    0   \n",
       "\n",
       "   spa  \n",
       "0    0  \n",
       "1    0  \n",
       "2    0  \n",
       "3    0  \n",
       "4    0  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book = pd.read_csv(path + 'final_book.csv').drop(['Unnamed: 0'],axis=1)\n",
    "book.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "spiritual-samoa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['book_id', 'original_publication_year', 'average_rating', 'ratings_count', 'work_ratings_count', 'work_text_reviews_count', 'ratings_1', 'ratings_2', 'ratings_3', 'ratings_4', 'ratings_5', 'fantasy, paranormal', 'non-fiction', 'mystery, thriller, crime', 'fiction', 'young-adult', 'children', 'poetry', 'comics, graphic', 'romance', 'history, historical fiction, biography', 'ara', 'en', 'fre', 'ger', 'ind', 'jpn', 'other', 'per', 'pol', 'por', 'spa']\n"
     ]
    }
   ],
   "source": [
    "print(list(book.columns))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "destroyed-terry",
   "metadata": {},
   "source": [
    "<div style=\"direction:rtl\", align=\"justify\">\n",
    "<h3>\n",
    "<span style='font-family:\"B Nazanin\"'>\n",
    "پیش از انجام pca ستون های عددی را نرمالایز می کنیم. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "pressed-percentage",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize\n",
    "cols = ['original_publication_year','average_rating','ratings_count','work_ratings_count',\n",
    "        'work_text_reviews_count','ratings_1','ratings_2','ratings_3','ratings_4','ratings_5']\n",
    "\n",
    "features = book[cols]\n",
    "book[cols] = pd.DataFrame(normalize(book[cols]),columns = cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "israeli-adaptation",
   "metadata": {},
   "source": [
    "<div style=\"direction:rtl\", align=\"justify\">\n",
    "<h3>\n",
    "<span style='font-family:\"B Nazanin\"'>\n",
    "در این قسمت میزان پوشش کامپوننت ها ضمن pca با کامپوننت های 1 الی 9 را بررسی می کنیم. اگر به pca با 6 کامپوننت توجه کنیم 99.6 درصد از تغییرات خروجی توسط 6 کامپوننت پوشش داده شده است که بسیار عدد مناسبی می باشد و با انتخاب آن  تعداد ستون های عددی از 10 ستون به 6 ستون کاهش پیدا می کند و هم پوشش خوبی خواهیم داشت."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "speaking-indonesia",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 [0.53697658]\n",
      "____________________________________________________________________\n",
      "2 [0.53697658 0.7811582 ]\n",
      "____________________________________________________________________\n",
      "3 [0.53697658 0.7811582  0.8698168 ]\n",
      "____________________________________________________________________\n",
      "4 [0.53697658 0.7811582  0.8698168  0.94568806]\n",
      "____________________________________________________________________\n",
      "5 [0.53697658 0.7811582  0.8698168  0.94568806 0.98904963]\n",
      "____________________________________________________________________\n",
      "6 [0.53697658 0.7811582  0.8698168  0.94568806 0.98904963 0.99693383]\n",
      "____________________________________________________________________\n",
      "7 [0.53697658 0.7811582  0.8698168  0.94568806 0.98904963 0.99693383\n",
      " 0.99914593]\n",
      "____________________________________________________________________\n",
      "8 [0.53697658 0.7811582  0.8698168  0.94568806 0.98904963 0.99693383\n",
      " 0.99914593 0.99999999]\n",
      "____________________________________________________________________\n",
      "9 [0.53697658 0.7811582  0.8698168  0.94568806 0.98904963 0.99693383\n",
      " 0.99914593 0.99999999 1.        ]\n",
      "____________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "for i in range(1, 10):\n",
    "    pca = PCA(n_components=i)\n",
    "    pca.fit_transform(book[cols])\n",
    "    print(i, np.cumsum(pca.explained_variance_ratio_))\n",
    "    print('____________________________________________________________________')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "suffering-princeton",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.53697658 0.7811582  0.8698168  0.94568806 0.98904963 0.99693383]\n"
     ]
    }
   ],
   "source": [
    "pca = PCA(n_components=6)\n",
    "pca.fit_transform(book[cols])\n",
    "print(np.cumsum(pca.explained_variance_ratio_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "filled-surname",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.141303</td>\n",
       "      <td>-0.032637</td>\n",
       "      <td>-0.045050</td>\n",
       "      <td>0.016251</td>\n",
       "      <td>0.004296</td>\n",
       "      <td>0.001278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.197796</td>\n",
       "      <td>-0.031674</td>\n",
       "      <td>-0.031066</td>\n",
       "      <td>0.045601</td>\n",
       "      <td>-0.003742</td>\n",
       "      <td>-0.005120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.007979</td>\n",
       "      <td>-0.100581</td>\n",
       "      <td>-0.023841</td>\n",
       "      <td>0.086025</td>\n",
       "      <td>0.005656</td>\n",
       "      <td>0.054946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.117254</td>\n",
       "      <td>-0.035646</td>\n",
       "      <td>-0.045628</td>\n",
       "      <td>0.021263</td>\n",
       "      <td>-0.004237</td>\n",
       "      <td>0.004112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.010751</td>\n",
       "      <td>-0.061986</td>\n",
       "      <td>-0.053832</td>\n",
       "      <td>0.017181</td>\n",
       "      <td>-0.012582</td>\n",
       "      <td>0.013082</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          1         2         3         4         5         6\n",
       "0  0.141303 -0.032637 -0.045050  0.016251  0.004296  0.001278\n",
       "1  0.197796 -0.031674 -0.031066  0.045601 -0.003742 -0.005120\n",
       "2  0.007979 -0.100581 -0.023841  0.086025  0.005656  0.054946\n",
       "3  0.117254 -0.035646 -0.045628  0.021263 -0.004237  0.004112\n",
       "4 -0.010751 -0.061986 -0.053832  0.017181 -0.012582  0.013082"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_pca = pd.DataFrame(pca.transform(book[cols]), columns=[x for x in range(1,7)])\n",
    "book_pca.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "liquid-return",
   "metadata": {},
   "source": [
    "<div style=\"direction:rtl\", align=\"justify\">\n",
    "<h3>\n",
    "<span style='font-family:\"B Nazanin\"'>\n",
    "ستون های flag مانند را با 6 ستون حاصل از pca جوین می کنیم."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "enhanced-complexity",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>fantasy, paranormal</th>\n",
       "      <th>non-fiction</th>\n",
       "      <th>mystery, thriller, crime</th>\n",
       "      <th>fiction</th>\n",
       "      <th>...</th>\n",
       "      <th>fre</th>\n",
       "      <th>ger</th>\n",
       "      <th>ind</th>\n",
       "      <th>jpn</th>\n",
       "      <th>other</th>\n",
       "      <th>per</th>\n",
       "      <th>pol</th>\n",
       "      <th>por</th>\n",
       "      <th>spa</th>\n",
       "      <th>book_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.141303</td>\n",
       "      <td>-0.032637</td>\n",
       "      <td>-0.045050</td>\n",
       "      <td>0.016251</td>\n",
       "      <td>0.004296</td>\n",
       "      <td>0.001278</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.197796</td>\n",
       "      <td>-0.031674</td>\n",
       "      <td>-0.031066</td>\n",
       "      <td>0.045601</td>\n",
       "      <td>-0.003742</td>\n",
       "      <td>-0.005120</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.007979</td>\n",
       "      <td>-0.100581</td>\n",
       "      <td>-0.023841</td>\n",
       "      <td>0.086025</td>\n",
       "      <td>0.005656</td>\n",
       "      <td>0.054946</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.117254</td>\n",
       "      <td>-0.035646</td>\n",
       "      <td>-0.045628</td>\n",
       "      <td>0.021263</td>\n",
       "      <td>-0.004237</td>\n",
       "      <td>0.004112</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.010751</td>\n",
       "      <td>-0.061986</td>\n",
       "      <td>-0.053832</td>\n",
       "      <td>0.017181</td>\n",
       "      <td>-0.012582</td>\n",
       "      <td>0.013082</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          1         2         3         4         5         6  \\\n",
       "0  0.141303 -0.032637 -0.045050  0.016251  0.004296  0.001278   \n",
       "1  0.197796 -0.031674 -0.031066  0.045601 -0.003742 -0.005120   \n",
       "2  0.007979 -0.100581 -0.023841  0.086025  0.005656  0.054946   \n",
       "3  0.117254 -0.035646 -0.045628  0.021263 -0.004237  0.004112   \n",
       "4 -0.010751 -0.061986 -0.053832  0.017181 -0.012582  0.013082   \n",
       "\n",
       "   fantasy, paranormal  non-fiction  mystery, thriller, crime  fiction  ...  \\\n",
       "0                    1            0                         1        1  ...   \n",
       "1                    1            0                         1        1  ...   \n",
       "2                    1            0                         0        1  ...   \n",
       "3                    0            0                         1        1  ...   \n",
       "4                    0            0                         0        1  ...   \n",
       "\n",
       "   fre  ger  ind  jpn  other  per  pol  por  spa  book_id  \n",
       "0    0    0    0    0      0    0    0    0    0        1  \n",
       "1    0    0    0    0      0    0    0    0    0        2  \n",
       "2    0    0    0    0      0    0    0    0    0        3  \n",
       "3    0    0    0    0      0    0    0    0    0        4  \n",
       "4    0    0    0    0      0    0    0    0    0        5  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "book_final = pd.concat([book_pca,book.iloc[:,11:]], axis=1)\n",
    "book_final['book_id'] = book.book_id\n",
    "book_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "instant-distinction",
   "metadata": {},
   "source": [
    "### ✅ Merge book and rating dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "assigned-albert",
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = pd.read_csv(path + 'ratings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "italic-perception",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pd.merge(ratings, book_final, how='inner', on='book_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "motivated-course",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>fantasy, paranormal</th>\n",
       "      <th>...</th>\n",
       "      <th>en</th>\n",
       "      <th>fre</th>\n",
       "      <th>ger</th>\n",
       "      <th>ind</th>\n",
       "      <th>jpn</th>\n",
       "      <th>other</th>\n",
       "      <th>per</th>\n",
       "      <th>pol</th>\n",
       "      <th>por</th>\n",
       "      <th>spa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>258</td>\n",
       "      <td>5</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>258</td>\n",
       "      <td>3</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>143</td>\n",
       "      <td>258</td>\n",
       "      <td>4</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>242</td>\n",
       "      <td>258</td>\n",
       "      <td>5</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>325</td>\n",
       "      <td>258</td>\n",
       "      <td>4</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  book_id  rating        1         2         3         4         5  \\\n",
       "0        1      258       5  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "1       11      258       3  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "2      143      258       4  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "3      242      258       5  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "4      325      258       4  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "\n",
       "          6  fantasy, paranormal  ...  en  fre  ger  ind  jpn  other  per  \\\n",
       "0  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "1  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "2  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "3  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "4  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "\n",
       "   pol  por  spa  \n",
       "0    0    0    0  \n",
       "1    0    0    0  \n",
       "2    0    0    0  \n",
       "3    0    0    0  \n",
       "4    0    0    0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cardiovascular-layer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5976479, 30)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "better-fellowship",
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(path + 'classifier_input.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "worse-brighton",
   "metadata": {},
   "source": [
    "# 4️⃣ Setup Phase\n",
    "<hr style=\"border:2px solid gray\"> </hr>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "southwest-yugoslavia",
   "metadata": {},
   "source": [
    "## 📑 Resampling"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "particular-november",
   "metadata": {},
   "source": [
    "### ✅ Check distribution of rating variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "living-choir",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>book_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>fantasy, paranormal</th>\n",
       "      <th>...</th>\n",
       "      <th>en</th>\n",
       "      <th>fre</th>\n",
       "      <th>ger</th>\n",
       "      <th>ind</th>\n",
       "      <th>jpn</th>\n",
       "      <th>other</th>\n",
       "      <th>per</th>\n",
       "      <th>pol</th>\n",
       "      <th>por</th>\n",
       "      <th>spa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>258</td>\n",
       "      <td>5</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>258</td>\n",
       "      <td>3</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>143</td>\n",
       "      <td>258</td>\n",
       "      <td>4</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>242</td>\n",
       "      <td>258</td>\n",
       "      <td>5</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>325</td>\n",
       "      <td>258</td>\n",
       "      <td>4</td>\n",
       "      <td>0.10183</td>\n",
       "      <td>0.039319</td>\n",
       "      <td>-0.046848</td>\n",
       "      <td>0.028769</td>\n",
       "      <td>0.017864</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  book_id  rating        1         2         3         4         5  \\\n",
       "0        1      258       5  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "1       11      258       3  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "2      143      258       4  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "3      242      258       5  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "4      325      258       4  0.10183  0.039319 -0.046848  0.028769  0.017864   \n",
       "\n",
       "          6  fantasy, paranormal  ...  en  fre  ger  ind  jpn  other  per  \\\n",
       "0  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "1  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "2  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "3  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "4  0.000186                    1  ...   1    0    0    0    0      0    0   \n",
       "\n",
       "   pol  por  spa  \n",
       "0    0    0    0  \n",
       "1    0    0    0  \n",
       "2    0    0    0  \n",
       "3    0    0    0  \n",
       "4    0    0    0  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.read_csv(path + 'classifier_input.csv')\n",
    "result.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "experienced-judges",
   "metadata": {},
   "source": [
    "<div style=\"direction:rtl\", align=\"justify\">\n",
    "<h3>\n",
    "<span style='font-family:\"B Nazanin\"'>\n",
    "در این قسمت توزیع متغیر rating درون دیتاست بررسی شده است همانطور که مشخص است بیشترین درصد یعنی 35 درصد از امتیازات امتیاز 4 بوده است و دو امتیاز 1 و 2 درصد بسیار پایینی دارند لذا برای اینکه مدل در یادگیری دچار خطا نشود و پیش بینی امتیازات پایین را نیز به خوبی یاد بگیرد با resample کردن درصد امتیازات 1 و 2 را افزایش می دهیم."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "final-operation",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    2139018\n",
       "5    1983093\n",
       "3    1370916\n",
       "2     359257\n",
       "1     124195\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result['rating'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "overhead-angel",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4    35.790605\n",
      "5    33.181628\n",
      "3    22.938523\n",
      "2     6.011182\n",
      "1     2.078063\n",
      "Name: rating, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(result['rating'].value_counts() / result['rating'].value_counts().sum() * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "greater-finland",
   "metadata": {},
   "source": [
    "### ✅ Make rate distribution balance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "level-swiss",
   "metadata": {},
   "outputs": [],
   "source": [
    "def Resampling(rate, proportion):\n",
    "    global result\n",
    "    n = ( proportion * (len(result)) - len(result[result['rating'] == rate]) ) / (1 - proportion)\n",
    "    samp = result[result['rating'] == rate].sample(n=round(n), replace=True)\n",
    "    balanced_result = pd.concat([result, samp], axis=0)\n",
    "    result = balanced_result.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "sealed-brighton",
   "metadata": {},
   "outputs": [],
   "source": [
    "Resampling(1, 0.12)\n",
    "Resampling(2, 0.12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "laden-death",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4    29.920776\n",
      "5    27.739683\n",
      "3    19.176496\n",
      "2    12.000004\n",
      "1    11.163041\n",
      "Name: rating, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(result['rating'].value_counts() / result['rating'].value_counts().sum() * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "arabic-reality",
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(path + 'balanced_classifier_input.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}