FinalProject.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "01cc98be",
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm.sklearn import LGBMRanker\n",
    "from datetime import timedelta\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dea1e6c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_features = pd.read_parquet('C:/Users/smexy/Documents/Datasets/CS677/user_features.parquet')\n",
    "item_features = pd.read_parquet('C:/Users/smexy/Documents/Datasets/CS677/item_features.parquet')\n",
    "transactions_df = pd.read_csv('C:/Users/smexy/Documents/Datasets/CS677/transactions_train.csv')\n",
    "transactions_df.t_dat = pd.to_datetime( transactions_df.t_dat )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6eddf735",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_4w = transactions_df[transactions_df['t_dat'] >= pd.to_datetime('2020-08-24')].copy()\n",
    "df_3w = transactions_df[transactions_df['t_dat'] >= pd.to_datetime('2020-08-31')].copy()\n",
    "df_2w = transactions_df[transactions_df['t_dat'] >= pd.to_datetime('2020-09-07')].copy()\n",
    "df_1w = transactions_df[transactions_df['t_dat'] >= pd.to_datetime('2020-09-15')].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2ccbad1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "user_features[['club_member_status', 'fashion_news_frequency']] = (\n",
    "                   user_features[['club_member_status', 'fashion_news_frequency']]\n",
    "                   .apply(lambda x: pd.factorize(x)[0])\n",
    ").astype('int8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5e8708d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "transactions_df = (\n",
    "    transactions_df\n",
    "    .merge(user_features, on = ('customer_id'))\n",
    "    .merge(item_features, on = ('article_id'))\n",
    ")\n",
    "transactions_df.sort_values(['t_dat', 'customer_id'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2b62a7a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#for simplicity let's take only 1M rows\n",
    "N_ROWS = 1_000_000\n",
    "\n",
    "train = transactions_df.loc[ transactions_df.t_dat <= pd.to_datetime('2020-09-15') ].iloc[:N_ROWS]\n",
    "valid = transactions_df.loc[ transactions_df.t_dat >= pd.to_datetime('2020-09-16') ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fb0c21d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete transactions to save memory\n",
    "del transactions_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "282bc19c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((54427, 88), (43, 88))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1ed17b54",
   "metadata": {},
   "outputs": [],
   "source": [
    "purchase_dict_4w = {}\n",
    "\n",
    "for i,x in enumerate(zip(df_4w['customer_id'], df_4w['article_id'])):\n",
    "    cust_id, art_id = x\n",
    "    if cust_id not in purchase_dict_4w:\n",
    "        purchase_dict_4w[cust_id] = {}\n",
    "    \n",
    "    if art_id not in purchase_dict_4w[cust_id]:\n",
    "        purchase_dict_4w[cust_id][art_id] = 0\n",
    "    \n",
    "    purchase_dict_4w[cust_id][art_id] += 1\n",
    "\n",
    "dummy_list_4w = list((df_4w['article_id'].value_counts()).index)[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4669b459",
   "metadata": {},
   "outputs": [],
   "source": [
    "purchase_dict_3w = {}\n",
    "\n",
    "for i,x in enumerate(zip(df_3w['customer_id'], df_3w['article_id'])):\n",
    "    cust_id, art_id = x\n",
    "    if cust_id not in purchase_dict_3w:\n",
    "        purchase_dict_3w[cust_id] = {}\n",
    "    \n",
    "    if art_id not in purchase_dict_3w[cust_id]:\n",
    "        purchase_dict_3w[cust_id][art_id] = 0\n",
    "    \n",
    "    purchase_dict_3w[cust_id][art_id] += 1\n",
    "\n",
    "dummy_list_3w = list((df_3w['article_id'].value_counts()).index)[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "398e84e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "purchase_dict_2w = {}\n",
    "\n",
    "for i,x in enumerate(zip(df_2w['customer_id'], df_2w['article_id'])):\n",
    "    cust_id, art_id = x\n",
    "    if cust_id not in purchase_dict_2w:\n",
    "        purchase_dict_2w[cust_id] = {}\n",
    "    \n",
    "    if art_id not in purchase_dict_2w[cust_id]:\n",
    "        purchase_dict_2w[cust_id][art_id] = 0\n",
    "    \n",
    "    purchase_dict_2w[cust_id][art_id] += 1\n",
    "\n",
    "dummy_list_2w = list((df_2w['article_id'].value_counts()).index)[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1f35f77b",
   "metadata": {},
   "outputs": [],
   "source": [
    "purchase_dict_1w = {}\n",
    "\n",
    "for i,x in enumerate(zip(df_1w['customer_id'], df_1w['article_id'])):\n",
    "    cust_id, art_id = x\n",
    "    if cust_id not in purchase_dict_1w:\n",
    "        purchase_dict_1w[cust_id] = {}\n",
    "    \n",
    "    if art_id not in purchase_dict_1w[cust_id]:\n",
    "        purchase_dict_1w[cust_id][art_id] = 0\n",
    "    \n",
    "    purchase_dict_1w[cust_id][art_id] += 1\n",
    "\n",
    "dummy_list_1w = list((df_1w['article_id'].value_counts()).index)[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61a5f372",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_candidates(customers_id, n_candidates = 12):\n",
    "  \"\"\"\n",
    "  df - basically, dataframe with customers(customers should be unique)\n",
    "  \"\"\"\n",
    "  prediction_dict = {}\n",
    "  dummy_list = list((df_2w['article_id'].value_counts()).index)[:n_candidates]\n",
    "\n",
    "  for i, cust_id in tqdm(enumerate(customers_id)):\n",
    "    # comment this for validation\n",
    "    if cust_id in purchase_dict_1w:\n",
    "        l = sorted((purchase_dict_1w[cust_id]).items(), key=lambda x: x[1], reverse=True)\n",
    "        l = [y[0] for y in l]\n",
    "        if len(l)>n_candidates:\n",
    "            s = l[:n_candidates]\n",
    "        else:\n",
    "            s = l+dummy_list_1w[:(n_candidates-len(l))]\n",
    "    elif cust_id in purchase_dict_2w:\n",
    "        l = sorted((purchase_dict_2w[cust_id]).items(), key=lambda x: x[1], reverse=True)\n",
    "        l = [y[0] for y in l]\n",
    "        if len(l)>n_candidates:\n",
    "            s = l[:n_candidates]\n",
    "        else:\n",
    "            s = l+dummy_list_2w[:(n_candidates-len(l))]\n",
    "    elif cust_id in purchase_dict_3w:\n",
    "        l = sorted((purchase_dict_3w[cust_id]).items(), key=lambda x: x[1], reverse=True)\n",
    "        l = [y[0] for y in l]\n",
    "        if len(l)>n_candidates:\n",
    "            s = l[:n_candidates]\n",
    "        else:\n",
    "            s = l+dummy_list_3w[:(n_candidates-len(l))]\n",
    "    elif cust_id in purchase_dict_4w:\n",
    "        l = sorted((purchase_dict_4w[cust_id]).items(), key=lambda x: x[1], reverse=True)\n",
    "        l = [y[0] for y in l]\n",
    "        if len(l)>n_candidates:\n",
    "            s = l[:n_candidates]\n",
    "        else:\n",
    "            s = l+dummy_list_4w[:(n_candidates-len(l))]\n",
    "    else:\n",
    "        s = dummy_list\n",
    "    prediction_dict[cust_id] = s\n",
    "\n",
    "  k = list(map(lambda x: x[0], prediction_dict.items()))\n",
    "  v = list(map(lambda x: x[1], prediction_dict.items()))\n",
    "  negatives_df = pd.DataFrame({'customer_id': k, 'negatives': v})\n",
    "  negatives_df = (\n",
    "      negatives_df\n",
    "      .explode('negatives')\n",
    "      .rename(columns = {'negatives': 'article_id'})\n",
    "  )\n",
    "  return negatives_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a410caf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#take only last 15 transactions\n",
    "train['rank'] = range(len(train))\n",
    "train = (\n",
    "    train\n",
    "    .assign(\n",
    "        rn = train.groupby(['customer_id'])['rank']\n",
    "                  .rank(method='first', ascending=False))\n",
    "    .query(\"rn <= 15\")\n",
    "    .drop(columns = ['price', 'sales_channel_id'])\n",
    "    .sort_values(['t_dat', 'customer_id'])\n",
    ")\n",
    "train['label'] = 1\n",
    "\n",
    "del train['rank']\n",
    "del train['rn']\n",
    "\n",
    "valid.sort_values(['t_dat', 'customer_id'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1a6e9b41",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2954it [00:00, 590167.38it/s]\n"
     ]
    }
   ],
   "source": [
    "last_dates = (\n",
    "    train\n",
    "    .groupby('customer_id')['t_dat']\n",
    "    .max()\n",
    "    .to_dict()\n",
    ")\n",
    "\n",
    "negatives = prepare_candidates(train['customer_id'].unique(), 15)\n",
    "negatives['t_dat'] = negatives['customer_id'].map(last_dates)\n",
    "\n",
    "negatives = (\n",
    "    negatives\n",
    "    .merge(user_features, on = ('customer_id'))\n",
    "    .merge(item_features, on = ('article_id'))\n",
    ")\n",
    "negatives['label'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "abb53858",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.concat([train, negatives])\n",
    "train.sort_values(['customer_id', 't_dat'], inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1abbd8b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_baskets = train.groupby(['customer_id'])['article_id'].count().values\n",
    "# valid_baskets = valid.groupby(['customer_id'])['article_id'].count().values\n",
    "train_baskets = train.groupby(['customer_id'])['article_id'].count().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f17d5a8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ranker = LGBMRanker(\n",
    "    objective=\"lambdarank\",\n",
    "    metric=\"ndcg\",\n",
    "    boosting_type=\"dart\",\n",
    "    max_depth=7,\n",
    "    n_estimators=300,\n",
    "    importance_type='gain',\n",
    "    verbose=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7d0b9dd5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).\n",
      "[LightGBM] [Debug] Dataset::GetMultiBinFromSparseFeatures: sparse rate 0.847712\n",
      "[LightGBM] [Debug] Dataset::GetMultiBinFromAllFeatures: sparse rate 0.509569\n",
      "[LightGBM] [Debug] init for col-wise cost 0.002576 seconds, init for row-wise cost 0.008634 seconds\n",
      "[LightGBM] [Warning] Auto-choosing row-wise multi-threading, the overhead of testing was 0.003578 seconds.\n",
      "You can set `force_row_wise=true` to remove the overhead.\n",
      "And if memory is not enough, you can set `force_col_wise=true`.\n",
      "[LightGBM] [Debug] Using Sparse Multi-Val Bin\n",
      "[LightGBM] [Info] Total Bins 3507\n",
      "[LightGBM] [Info] Number of data points in the train set: 36079, number of used features: 82\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n",
      "[LightGBM] [Debug] Trained a tree with leaves = 31 and depth = 7\n"
     ]
    }
   ],
   "source": [
    "ranker = ranker.fit(\n",
    "    train.drop(columns = ['t_dat', 'customer_id', 'article_id', 'label']),\n",
    "    train.pop('label'),\n",
    "    group=train_baskets,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c4d3ec8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_sub = pd.read_csv('C:/Users/smexy/Documents/Datasets/CS677/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "accd30dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1371980it [00:01, 697028.31it/s]\n"
     ]
    }
   ],
   "source": [
    "candidates = prepare_candidates(sample_sub.customer_id.unique(), 12)\n",
    "candidates = (\n",
    "    candidates\n",
    "    .merge(user_features, on = ('customer_id'))\n",
    "    .merge(item_features, on = ('article_id'))\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a61d8876",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 76.85it/s]\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "batch_size = 1_000_000\n",
    "for bucket in tqdm(range(0, len(candidates), batch_size)):\n",
    "  outputs = ranker.predict(\n",
    "      candidates.iloc[bucket: bucket+batch_size]\n",
    "      .drop(columns = ['customer_id', 'article_id'])\n",
    "      )\n",
    "  preds.append(outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "54110ae9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\smexy\\AppData\\Local\\Temp/ipykernel_6844/940012020.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  preds.sort_values(['customer_id', 'preds'], ascending=False, inplace = True)\n"
     ]
    }
   ],
   "source": [
    "preds = np.concatenate(preds)\n",
    "candidates['preds'] = preds\n",
    "preds = candidates[['customer_id', 'article_id', 'preds']]\n",
    "preds.sort_values(['customer_id', 'preds'], ascending=False, inplace = True)\n",
    "preds = (\n",
    "    preds\n",
    "    .groupby('customer_id')[['article_id']]\n",
    "    .aggregate(lambda x: x.tolist())\n",
    ")\n",
    "preds['article_id'] = preds['article_id'].apply(lambda x: ' '.join(['0'+str(k) for k in x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6fcd1c5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = sample_sub[['customer_id']].merge(\n",
    "    preds\n",
    "    .reset_index()\n",
    "    .rename(columns = {'article_id': 'prediction'}), how = 'left')\n",
    "preds['prediction'].fillna(' '.join(['0'+str(art) for art in dummy_list_2w]), inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "95407832",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds.to_csv('predictionsheet.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cab5853",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}