collaborative.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2756101, 5)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('events.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>visitorid</th>\n",
       "      <th>event</th>\n",
       "      <th>itemid</th>\n",
       "      <th>transactionid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1433221332117</td>\n",
       "      <td>257597</td>\n",
       "      <td>view</td>\n",
       "      <td>355908</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1433224214164</td>\n",
       "      <td>992329</td>\n",
       "      <td>view</td>\n",
       "      <td>248676</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1433221999827</td>\n",
       "      <td>111016</td>\n",
       "      <td>view</td>\n",
       "      <td>318965</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1433221955914</td>\n",
       "      <td>483717</td>\n",
       "      <td>view</td>\n",
       "      <td>253185</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1433221337106</td>\n",
       "      <td>951259</td>\n",
       "      <td>view</td>\n",
       "      <td>367447</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       timestamp  visitorid event  itemid  transactionid\n",
       "0  1433221332117     257597  view  355908            NaN\n",
       "1  1433224214164     992329  view  248676            NaN\n",
       "2  1433221999827     111016  view  318965            NaN\n",
       "3  1433221955914     483717  view  253185            NaN\n",
       "4  1433221337106     951259  view  367447            NaN"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['view', 'addtocart', 'transaction'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.event.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22457, 5)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans = df[df['event'] == 'transaction']\n",
    "trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11719,)\n",
      "(12025,)\n"
     ]
    }
   ],
   "source": [
    "visitors = trans['visitorid'].unique()\n",
    "items = trans['itemid'].unique()\n",
    "print(visitors.shape)\n",
    "print(items.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19939, 5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans2 = trans.groupby(['visitorid']).head(50)\n",
    "trans2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\17sus\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "C:\\Users\\17sus\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>visitorid</th>\n",
       "      <th>event</th>\n",
       "      <th>itemid</th>\n",
       "      <th>transactionid</th>\n",
       "      <th>visitors</th>\n",
       "      <th>items</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>130</td>\n",
       "      <td>1433222276276</td>\n",
       "      <td>599528</td>\n",
       "      <td>transaction</td>\n",
       "      <td>356475</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>304</td>\n",
       "      <td>1433193500981</td>\n",
       "      <td>121688</td>\n",
       "      <td>transaction</td>\n",
       "      <td>15335</td>\n",
       "      <td>11117.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>418</td>\n",
       "      <td>1433193915008</td>\n",
       "      <td>552148</td>\n",
       "      <td>transaction</td>\n",
       "      <td>81345</td>\n",
       "      <td>5444.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>814</td>\n",
       "      <td>1433176736375</td>\n",
       "      <td>102019</td>\n",
       "      <td>transaction</td>\n",
       "      <td>150318</td>\n",
       "      <td>13556.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>843</td>\n",
       "      <td>1433174518180</td>\n",
       "      <td>189384</td>\n",
       "      <td>transaction</td>\n",
       "      <td>310791</td>\n",
       "      <td>7244.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2755082</td>\n",
       "      <td>1438388436295</td>\n",
       "      <td>1155978</td>\n",
       "      <td>transaction</td>\n",
       "      <td>430050</td>\n",
       "      <td>4316.0</td>\n",
       "      <td>11716</td>\n",
       "      <td>6280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2755285</td>\n",
       "      <td>1438380441389</td>\n",
       "      <td>218648</td>\n",
       "      <td>transaction</td>\n",
       "      <td>446271</td>\n",
       "      <td>10485.0</td>\n",
       "      <td>3646</td>\n",
       "      <td>12024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2755294</td>\n",
       "      <td>1438377176570</td>\n",
       "      <td>1050575</td>\n",
       "      <td>transaction</td>\n",
       "      <td>31640</td>\n",
       "      <td>8354.0</td>\n",
       "      <td>11717</td>\n",
       "      <td>3246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2755508</td>\n",
       "      <td>1438357730123</td>\n",
       "      <td>855941</td>\n",
       "      <td>transaction</td>\n",
       "      <td>235771</td>\n",
       "      <td>4385.0</td>\n",
       "      <td>11718</td>\n",
       "      <td>2419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2755607</td>\n",
       "      <td>1438358989163</td>\n",
       "      <td>1051054</td>\n",
       "      <td>transaction</td>\n",
       "      <td>312728</td>\n",
       "      <td>17579.0</td>\n",
       "      <td>11659</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>19939 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             timestamp  visitorid        event  itemid  transactionid  \\\n",
       "130      1433222276276     599528  transaction  356475         4000.0   \n",
       "304      1433193500981     121688  transaction   15335        11117.0   \n",
       "418      1433193915008     552148  transaction   81345         5444.0   \n",
       "814      1433176736375     102019  transaction  150318        13556.0   \n",
       "843      1433174518180     189384  transaction  310791         7244.0   \n",
       "...                ...        ...          ...     ...            ...   \n",
       "2755082  1438388436295    1155978  transaction  430050         4316.0   \n",
       "2755285  1438380441389     218648  transaction  446271        10485.0   \n",
       "2755294  1438377176570    1050575  transaction   31640         8354.0   \n",
       "2755508  1438357730123     855941  transaction  235771         4385.0   \n",
       "2755607  1438358989163    1051054  transaction  312728        17579.0   \n",
       "\n",
       "         visitors  items  \n",
       "130             0      0  \n",
       "304             1      1  \n",
       "418             2      2  \n",
       "814             3      3  \n",
       "843             4      4  \n",
       "...           ...    ...  \n",
       "2755082     11716   6280  \n",
       "2755285      3646  12024  \n",
       "2755294     11717   3246  \n",
       "2755508     11718   2419  \n",
       "2755607     11659    188  \n",
       "\n",
       "[19939 rows x 7 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trans2['visitors'] = trans2['visitorid'].apply(lambda x : np.argwhere(visitors == x)[0][0])\n",
    "trans2['items'] = trans2['itemid'].apply(lambda x : np.argwhere(items == x)[0][0])\n",
    "trans2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\17sus\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\_index.py:84: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_intXint(row, col, x.flat[0])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<11719x12025 sparse matrix of type '<class 'numpy.int8'>'\n",
       "\twith 18905 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "occurences = csr_matrix((visitors.shape[0], items.shape[0]), dtype='int8')\n",
    "def set_occurences(visitor, item):\n",
    "    occurences[visitor, item] += 1\n",
    "trans2.apply(lambda row: set_occurences(row['visitors'], row['items']), axis=1)\n",
    "occurences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\17sus\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\_index.py:127: SparseEfficiencyWarning: Changing the sparsity structure of a csc_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n"
     ]
    }
   ],
   "source": [
    "cooc = occurences.transpose().dot(occurences)\n",
    "cooc.setdiag(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xLogX(x):\n",
    "    return x * np.log(x) if x != 0 else 0.0\n",
    "def entropy(x1, x2=0, x3=0, x4=0):\n",
    "    return xLogX(x1 + x2 + x3 + x4) - xLogX(x1) - xLogX(x2) - xLogX(x3) - xLogX(x4)\n",
    "def LLR(k11, k12, k21, k22):\n",
    "    rowEntropy = entropy(k11 + k12, k21 + k22)\n",
    "    columnEntropy = entropy(k11 + k21, k12 + k22)\n",
    "    matrixEntropy = entropy(k11, k12, k21, k22)\n",
    "    if rowEntropy + columnEntropy < matrixEntropy:\n",
    "        return 0.0\n",
    "    return 2.0 * (rowEntropy + columnEntropy - matrixEntropy)\n",
    "def rootLLR(k11, k12, k21, k22):\n",
    "    llr = LLR(k11, k12, k21, k22)\n",
    "    sqrt = np.sqrt(llr)\n",
    "    if k11 * 1.0 / (k11 + k12) < k21 * 1.0 / (k21 + k22):\n",
    "        sqrt = -sqrt\n",
    "    return sqrt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\17sus\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\_index.py:84: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_intXint(row, col, x.flat[0])\n"
     ]
    }
   ],
   "source": [
    "row_sum = np.sum(cooc, axis=0).A.flatten()\n",
    "column_sum = np.sum(cooc, axis=1).A.flatten()\n",
    "total = np.sum(row_sum, axis=0)\n",
    "pp_score = csr_matrix((cooc.shape[0], cooc.shape[1]), dtype='double')\n",
    "cx = cooc.tocoo()\n",
    "for i,j,v in zip(cx.row, cx.col, cx.data):\n",
    "    if v != 0:\n",
    "        k11 = v\n",
    "        k12 = row_sum[i] - k11\n",
    "        k21 = column_sum[j] - k11\n",
    "        k22 = total - k11 - k12 - k21\n",
    "        pp_score[i,j] = rootLLR(k11, k12, k21, k22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = np.flip(np.sort(pp_score.A, axis=1), axis=1)\n",
    "result_indices = np.flip(np.argsort(pp_score.A, axis=1), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([15.33511076, 14.60017668,  3.62091635, ...,  0.        ,\n",
       "        0.        ,  0.        ])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[8456]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}