assists_feat_stats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # if used pip install package\n",
    "# !pip install xgboost\n",
    "# !pip install lightgbm\n",
    "# !pip install wget\n",
    "# !pip install gensim\n",
    "# !pip install catboost\n",
    "# !pip install cython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import os\n",
    "import sys\n",
    "import logging\n",
    "import gc\n",
    "import wget\n",
    "import time\n",
    "import tarfile\n",
    "import zipfile\n",
    "import functools\n",
    "import random\n",
    "import copy\n",
    "import pickle\n",
    "from tqdm import tqdm_notebook, tqdm\n",
    "from itertools import product, combinations\n",
    "from scipy.special import comb, perm\n",
    "\n",
    "import scipy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import catboost as cbt\n",
    "from glove import *\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold\n",
    "from sklearn.linear_model import LogisticRegression,BayesianRidge,SGDClassifier,PassiveAggressiveClassifier,RidgeClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "from sklearn.ensemble import ExtraTreesClassifier,RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import LinearSVC,NuSVC,SVC\n",
    "from sklearn.metrics import f1_score, roc_auc_score\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, Embedding, Dropout, Activation, Conv1D, LSTM, GRU  #, CuDNNGRU, CuDNNLSTM\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\n",
    "from keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model, Sequential\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.layers import concatenate\n",
    "from keras.callbacks import *\n",
    "from keras.optimizers import *\n",
    "from keras.engine.topology import Layer\n",
    "from keras.layers import *\n",
    "from keras.utils import np_utils\n",
    "\n",
    "import gensim\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import ray\n",
    "# ray.init(object_store_memory=int(100e6))\n",
    "# import modin.pandas as pd\n",
    "# os.environ[\"MODIN_ENGINE\"] = \"ray\"  # Modin will use Ray\n",
    "# os.environ[\"MODIN_ENGINE\"] = \"dask\"  # Modin will use Dask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Logger(object):\n",
    "    def __init__(self, filename='default.log', stream=sys.stdout):\n",
    "        self.terminal = stream\n",
    "        self.log = open(filename, 'a')\n",
    "        \n",
    "    def write(self, message):\n",
    "        self.terminal.write(message)\n",
    "        self.log.write(message)\n",
    "        \n",
    "    def flush(self):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sys.stdout = Logger(\"logs/default.log\", sys.stdout)\n",
    "# sys.stderr = Logger(\"logs/default_err.log\", sys.stderr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DDIR = \"data\"\n",
    "UDDIR = \"user_data\"\n",
    "UFEDIR = \"user_data/feat_data_v05\"\n",
    "UMDIR = \"user_data/model_data\"\n",
    "RESDIR = \"prediction_result\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "UID = \"user_id\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data (Only once)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_fname = wget.download(\"https://tesla-ap-shanghai-1256322946.cos.ap-shanghai.myqcloud.com/cephfs/tesla_common/deeplearning/dataset/algo_contest/train_preliminary.zip\", out=DDIR)\n",
    "# test_fname = wget.download(\"https://tesla-ap-shanghai-1256322946.cos.ap-shanghai.myqcloud.com/cephfs/tesla_common/deeplearning/dataset/algo_contest/test.zip\", out=DDIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def myunzip(filename):\n",
    "#     zFile = zipfile.ZipFile(filename, \"r\")\n",
    "#     for fileM in zFile.namelist(): \n",
    "#         zFile.extract(fileM, DDIR)\n",
    "#         print(fileM)\n",
    "#     zFile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# myunzip(train_fname)\n",
    "# myunzip(test_fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bch_rencol(values, prefix=\"\", suffix=\"\"):\n",
    "    return list(map(lambda x: f\"{prefix}\"+\"_\".join(list(map(lambda y: str(y), x)))+f\"{suffix}\" \n",
    "                    if hasattr(x, \"__iter__\") and not isinstance(x, str) \n",
    "                    else f\"{prefix}\"+str(x)+f\"{suffix}\", values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mynunique(values):\n",
    "    return values.nunique(dropna=False)\n",
    "def getidxmax(x):\n",
    "    return x.idxmax()[1]\n",
    "# for time series\n",
    "def at_len(x):\n",
    "    return len(x)\n",
    "\n",
    "def at_sum(x):\n",
    "    return np.sum(x)\n",
    "\n",
    "def at_max(x):\n",
    "    return np.max(x)\n",
    "\n",
    "def at_min(x):\n",
    "    return np.min(x)\n",
    "\n",
    "def at_mean(x):\n",
    "    return np.mean(x)\n",
    "\n",
    "def at_range(x):\n",
    "    return at_max(x) - at_min(x)\n",
    "\n",
    "def at_nunq(x):\n",
    "    return len(set(x))\n",
    "\n",
    "def at_lenDrange(x):\n",
    "    return at_len(x)/(at_range(x)+1)\n",
    "\n",
    "def at_lenDnunq(x):\n",
    "    return at_len(x)/at_nunq(x)\n",
    "\n",
    "def at_percentile(n):\n",
    "    def at_percentile_(x):\n",
    "        return np.percentile(x, n)\n",
    "    at_percentile_.__name__ = f\"at_percentile_{n}\"\n",
    "    return at_percentile_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "OP_SET = [\"sum\", \"max\", \"min\", \"mean\", \"std\"]\n",
    "OP_SET1 = [\"nunique\", \"sum\", \"max\", \"min\", \"mean\", \"std\", \"median\", \"skew\", at_percentile(25), at_percentile(75)]\n",
    "OP_SET2 = [\"sum\", \"max\", \"min\", \"mean\", \"std\", \"median\", \"skew\", at_percentile(25), at_percentile(75)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_SET = [\"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\", \"time\", \"click_times\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "nesting_level = 0\n",
    "is_start = None\n",
    "class Timer:\n",
    "    def __init__(self):\n",
    "        self.start = time.time()\n",
    "        self.history = [self.start]\n",
    "\n",
    "    def check(self, info):\n",
    "        current = time.time()\n",
    "        print(f\"[{info}] spend {current - self.history[-1]:0.2f} sec\")\n",
    "        self.history.append(current)\n",
    "\n",
    "def log(entry):\n",
    "    global nesting_level\n",
    "    space = \"-\" * (4 * nesting_level)\n",
    "    print(f\"{space}{entry}\")\n",
    "\n",
    "def timeit(method, start_log=None):\n",
    "    @functools.wraps(method)\n",
    "    def timed(*args, **kw):\n",
    "        global is_start\n",
    "        global nesting_level\n",
    "\n",
    "        if not is_start:\n",
    "            print()\n",
    "\n",
    "        is_start = True\n",
    "        log(f\"Start [{method.__name__}]:\" + (start_log if start_log else \"\"))\n",
    "        log(f'Start time: {time.strftime(\"%Y-%m-%d %H:%M:%S\")}')\n",
    "        nesting_level += 1\n",
    "\n",
    "        start_time = time.time()\n",
    "        result = method(*args, **kw)\n",
    "        end_time = time.time()\n",
    "\n",
    "        nesting_level -= 1\n",
    "        log(f\"End   [{method.__name__}]. Time elapsed: {end_time - start_time:0.2f} sec.\")\n",
    "        is_start = False\n",
    "\n",
    "        return result\n",
    "\n",
    "    return timed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stats features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read train data\n",
    "train_click_log = pd.read_csv(f\"{DDIR}/train_preliminary/click_log.csv\")\n",
    "train_ad = pd.read_csv(f\"{DDIR}/train_preliminary/ad.csv\")\n",
    "# tag\n",
    "train_user = pd.read_csv(f\"{DDIR}/train_preliminary/user.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read test data\n",
    "test_click_log = pd.read_csv(f\"{DDIR}/test/click_log.csv\")\n",
    "test_ad = pd.read_csv(f\"{DDIR}/test/ad.csv\")\n",
    "# pd.DataFrame(np.sort(test_click_log[UID].unique()), columns=[UID]).to_csv(f\"{DDIR}/test/user.csv\", index=False)\n",
    "test_user = pd.read_csv(f\"{DDIR}/test/user.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30082771, 4)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_click_log.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2481135, 6)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_ad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(900000, 3)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_user.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(33585512, 4)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_click_log.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2618159, 6)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_ad.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ad[\"product_id\"] = train_ad[\"product_id\"].replace(\"\\\\N\", \"0\").astype(int)\n",
    "train_ad[\"industry\"] = train_ad[\"industry\"].replace(\"\\\\N\", \"0\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ad[\"product_id\"] = test_ad[\"product_id\"].replace(\"\\\\N\", \"0\").astype(int)\n",
    "test_ad[\"industry\"] = test_ad[\"industry\"].replace(\"\\\\N\", \"0\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2481135"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creative id in train (creative id is unique in train_ad)\n",
    "len(train_ad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2618159"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creative id in test (creative id is unique in test_ad)\n",
    "len(test_ad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "tol_train = pd.merge(train_click_log, train_ad, how=\"left\", on=\"creative_id\")\n",
    "tol_test = pd.merge(test_click_log, test_ad, how=\"left\", on=\"creative_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train_click_log, train_ad\n",
    "del test_click_log, test_ad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# comb pairs\n",
    "# ad_id-product_id\n",
    "# product_id-product_category\n",
    "# ad_id-advertiser_id\n",
    "# advertiser_id-industry\n",
    "# product_category-industry\n",
    "# product_category-advertiser_id\n",
    "comb_set = [\"ad_id-product_id\", \"product_id-product_category\", \"ad_id-advertiser_id\", \n",
    "            \"advertiser_id-industry\", \"product_category-industry\", \"product_category-advertiser_id\",\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tol_train[\"ad_id-product_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"ad_id\"], tol_train[\"product_id\"]))\n",
    "# tol_train[\"product_id-product_category\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"product_id\"], tol_train[\"product_category\"]))\n",
    "# tol_train[\"ad_id-advertiser_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"ad_id\"], tol_train[\"advertiser_id\"]))\n",
    "# tol_train[\"advertiser_id-industry\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"advertiser_id\"], tol_train[\"industry\"]))\n",
    "# tol_train[\"product_category-industry\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"product_category\"], tol_train[\"industry\"]))\n",
    "# tol_train[\"product_category-advertiser_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_train[\"product_category\"], tol_train[\"advertiser_id\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tol_test[\"ad_id-product_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"ad_id\"], tol_test[\"product_id\"]))\n",
    "# tol_test[\"product_id-product_category\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"product_id\"], tol_test[\"product_category\"]))\n",
    "# tol_test[\"ad_id-advertiser_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"ad_id\"], tol_test[\"advertiser_id\"]))\n",
    "# tol_test[\"advertiser_id-industry\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"advertiser_id\"], tol_test[\"industry\"]))\n",
    "# tol_test[\"product_category-industry\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"product_category\"], tol_test[\"industry\"]))\n",
    "# tol_test[\"product_category-advertiser_id\"] = list(map(lambda x, y: f\"{x}_{y}\", tol_test[\"product_category\"], tol_test[\"advertiser_id\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Category map to Label (use KFold)\n",
    "\n",
    "__a little overfitting age__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# two kfold\n",
    "@timeit\n",
    "def gen_catemlb(train_data, test_data, tag, col):\n",
    "    train_use = pd.merge(train_data[[UID, col]], tag, how=\"left\", on=UID)\n",
    "    test_use = test_data[[UID, col]]\n",
    "    \n",
    "    nfolds = 5\n",
    "    kfold = KFold(n_splits=5, shuffle=False, random_state=2020)\n",
    "    \n",
    "    kf_map = dict()\n",
    "\n",
    "    re_train_use = pd.DataFrame()\n",
    "    re_test_use = pd.DataFrame()\n",
    "\n",
    "    for kfn, (tr_idx, val_idx) in enumerate(kfold.split(train_use)):\n",
    "        print(\"train current kfn: \", kfn)\n",
    "        tr_ucid, val_ucid = train_use.iloc[tr_idx], train_use.iloc[val_idx]\n",
    "        cur_kf_map = tr_ucid.groupby([col])[[\"age\", \"gender\"]].agg([\"mean\"])\n",
    "        cur_kf_map.columns = bch_rencol(cur_kf_map.columns, prefix=f\"{col}_\")\n",
    "        # # only use intersect between train and test\n",
    "        # cur_kf_map.drop(np.setdiff1d(kf_map.index.unique(), test_use[col].unique()), inplace=True)\n",
    "        kf_map[kfn] = copy.deepcopy(cur_kf_map)\n",
    "        \n",
    "        val_ucid = pd.merge(val_ucid, cur_kf_map, how=\"left\", on=col)\n",
    "        re_train_use = pd.concat([re_train_use, val_ucid])\n",
    "    \n",
    "    \n",
    "    for kfn, (_, val_idx) in enumerate(kfold.split(test_use)):\n",
    "        print(\"test current kfn: \", kfn)\n",
    "        val_ucid = test_use.iloc[val_idx]\n",
    "        val_ucid = pd.merge(val_ucid, kf_map[kfn], how=\"left\", on=col)\n",
    "        re_test_use = pd.concat([re_test_use, val_ucid])\n",
    "\n",
    "    \n",
    "    assert len(train_use) == len(re_train_use)\n",
    "    assert len(test_use) == len(re_test_use)\n",
    "\n",
    "    train_use = None\n",
    "    test_use = None\n",
    "\n",
    "    tmp = re_train_use.groupby([UID], sort=False)[f\"{col}_age_mean\", f\"{col}_gender_mean\"].agg([\"sum\", \"max\", \"mean\", \"median\", \"min\", \"std\", ])\n",
    "    tmp.columns = bch_rencol(tmp.columns)\n",
    "    tmp.to_pickle(f\"{UFEDIR}/train_stats_catemlb_{col}.pkl\")\n",
    "    tmp = None\n",
    "\n",
    "    tmp = re_test_use.groupby([UID], sort=False)[f\"{col}_age_mean\", f\"{col}_gender_mean\"].agg([\"sum\", \"max\", \"mean\", \"median\", \"min\", \"std\", ])\n",
    "    tmp.columns = bch_rencol(tmp.columns)\n",
    "    tmp.to_pickle(f\"{UFEDIR}/test_stats_catemlb_{col}.pkl\")\n",
    "    tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in [\"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\"]:\n",
    "    gen_catemlb(tol_train, tol_test, train_user, col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One key(O1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for train\n",
    "tmp = tol_train.groupby([UID], sort=False).agg(\n",
    "    {\n",
    "        UID: [\"count\"], \n",
    "        \"click_times\": OP_SET1,\n",
    "        \"time\": OP_SET1,\n",
    "        \"creative_id\": [\"nunique\"],\n",
    "        \"ad_id\": [\"nunique\"],\n",
    "        \"product_id\": [\"nunique\"],\n",
    "        \"product_category\": [\"nunique\"],\n",
    "        \"advertiser_id\": [\"nunique\"],\n",
    "        \"industry\": [\"nunique\"],\n",
    "    }\n",
    ")\n",
    "\n",
    "tmp.columns = bch_rencol(tmp.columns)\n",
    "\n",
    "tmp.to_pickle(f\"{UFEDIR}/train_stats_o1.pkl\")\n",
    "tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for test\n",
    "tmp = tol_test.groupby([UID], sort=False).agg(\n",
    "    {\n",
    "        UID: [\"count\"], \n",
    "        \"click_times\": OP_SET1,\n",
    "        \"time\": OP_SET1,\n",
    "        \"creative_id\": [\"nunique\"],\n",
    "        \"ad_id\": [\"nunique\"],\n",
    "        \"product_id\": [\"nunique\"],\n",
    "        \"product_category\": [\"nunique\"],\n",
    "        \"advertiser_id\": [\"nunique\"],\n",
    "        \"industry\": [\"nunique\"],\n",
    "    }\n",
    ")\n",
    "\n",
    "tmp.columns = bch_rencol(tmp.columns)\n",
    "\n",
    "tmp.to_pickle(f\"{UFEDIR}/test_stats_o1.pkl\")\n",
    "tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comb Key(O2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_list = [\"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "for i in tqdm(range(1, 7)):\n",
    "    for cl in combinations(id_list, i):\n",
    "        key_set = [UID] + list(cl)\n",
    "        tmp = tol_train.groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "        tmp.columns = bch_rencol(tmp.columns, prefix=f\"{'C'.join(key_set[1:-1])}_\")\n",
    "        tmp.to_pickle(f\"{UFEDIR}/train_stats_o2_{'C'.join(key_set)}.pkl\")\n",
    "        tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "for i in tqdm(range(1, 7)):\n",
    "    for cl in combinations(id_list, i):\n",
    "        key_set = [UID] + list(cl)\n",
    "        tmp = tol_test.groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "        tmp.columns = bch_rencol(tmp.columns, prefix=f\"{'C'.join(key_set[1:-1])}_\")\n",
    "        tmp.to_pickle(f\"{UFEDIR}/test_stats_o2_{'C'.join(key_set)}.pkl\")\n",
    "        tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time Windows-Time cost(not use)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw = Timer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time Bins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins_set = [3, 7, 10, 30]\n",
    "\n",
    "for bins in bins_set:\n",
    "    tol_bins = pd.cut(pd.concat([tol_train[\"time\"], tol_test[\"time\"]]), bins, labels=range(bins))\n",
    "    tol_train[f\"bins{bins}\"] = tol_bins[:len(tol_train)]\n",
    "    tol_test[f\"bins{bins}\"] = tol_bins[len(tol_train):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Init Bin Set\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### One Key(O1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For train\n",
    "for bins in bins_set:\n",
    "    for cb in tqdm(range(bins)):\n",
    "        tmp = tol_train[tol_train[f\"bins{bins}\"] == cb].groupby([UID], sort=False).agg(\n",
    "            {\n",
    "                UID: [\"count\"], \n",
    "                \"click_times\": OP_SET1,\n",
    "                \"time\": OP_SET1,\n",
    "                \"creative_id\": [\"nunique\"],\n",
    "                \"ad_id\": [\"nunique\"],\n",
    "                \"product_id\": [\"nunique\"],\n",
    "                \"product_category\": [\"nunique\"],\n",
    "                \"advertiser_id\": [\"nunique\"],\n",
    "                \"industry\": [\"nunique\"],\n",
    "            }\n",
    "        )\n",
    "        tmp.columns = bch_rencol(tmp.columns, prefix=f\"bins{bins}_{cb}_\")\n",
    "        tmp.to_pickle(f\"{UFEDIR}/train_stats_o1_bins{bins}_{cb}.pkl\")\n",
    "        tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o1 for train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For test\n",
    "for bins in bins_set:\n",
    "    for cb in tqdm(range(bins)):\n",
    "        tmp = tol_test[tol_test[f\"bins{bins}\"] == cb].groupby([UID], sort=False).agg(\n",
    "            {\n",
    "                UID: [\"count\"], \n",
    "                \"click_times\": OP_SET1,\n",
    "                \"time\": OP_SET1,\n",
    "                \"creative_id\": [\"nunique\"],\n",
    "                \"ad_id\": [\"nunique\"],\n",
    "                \"product_id\": [\"nunique\"],\n",
    "                \"product_category\": [\"nunique\"],\n",
    "                \"advertiser_id\": [\"nunique\"],\n",
    "                \"industry\": [\"nunique\"],\n",
    "            }\n",
    "        )\n",
    "        tmp.columns = bch_rencol(tmp.columns, prefix=f\"bins{bins}_{cb}_\")\n",
    "        tmp.to_pickle(f\"{UFEDIR}/test_stats_o1_bins{bins}_{cb}.pkl\")\n",
    "        tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o1 for test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comb Key(O2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_list = [\"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "for bins in bins_set:\n",
    "    for cb in tqdm(range(bins)):\n",
    "        for i in tqdm(range(1, 7)):\n",
    "            for cl in combinations(id_list, i):\n",
    "                key_set = [UID] + list(cl)\n",
    "                tmp = tol_train[tol_train[f\"bins{bins}\"] == cb].groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "                tmp.columns = bch_rencol(tmp.columns, prefix=f\"bins{bins}_{cb}_{'C'.join(key_set[1:-1])}_\")\n",
    "                tmp.to_pickle(f\"{UFEDIR}/train_stats_o2_bins{bins}_{cb}_{'C'.join(key_set)}.pkl\")\n",
    "                tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o2 bin for train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "for bins in bins_set:\n",
    "    for cb in tqdm(range(bins)):\n",
    "        for i in tqdm(range(1, 7)):\n",
    "            for cl in combinations(id_list, i):\n",
    "                key_set = [UID] + list(cl)\n",
    "                tmp = tol_test[tol_test[f\"bins{bins}\"] == cb].groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "                tmp.columns = bch_rencol(tmp.columns, prefix=f\"bins{bins}_{cb}_{'C'.join(key_set[1:-1])}_\")\n",
    "                tmp.to_pickle(f\"{UFEDIR}/test_stats_o2_bins{bins}_{cb}_{'C'.join(key_set)}.pkl\")\n",
    "                tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o2 bin for test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Time Slides"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "slide_set = [1, 2, 3, 7, 14, 21, 30, 60, 90]\n",
    "\n",
    "tol_train[\"max_time\"] = tol_train.groupby([UID], sort=False)[\"time\"].transform(\"max\")\n",
    "tol_test[\"max_time\"] = tol_test.groupby([UID], sort=False)[\"time\"].transform(\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Get Max time\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### One Key(O1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For train\n",
    "for win in slide_set:\n",
    "    tmp = tol_train[tol_train[\"time\"] >= tol_train[\"max_time\"] - win].groupby([UID], sort=False).agg(\n",
    "        {\n",
    "            UID: [\"count\"], \n",
    "            \"click_times\": OP_SET1,\n",
    "            \"time\": OP_SET1,\n",
    "            \"creative_id\": [\"nunique\"],\n",
    "            \"ad_id\": [\"nunique\"],\n",
    "            \"product_id\": [\"nunique\"],\n",
    "            \"product_category\": [\"nunique\"],\n",
    "            \"advertiser_id\": [\"nunique\"],\n",
    "            \"industry\": [\"nunique\"],\n",
    "        }\n",
    "    )\n",
    "    tmp.columns = bch_rencol(tmp.columns, prefix=f\"slide{win}_\")\n",
    "    tmp.to_pickle(f\"{UFEDIR}/train_stats_o1_slide{win}.pkl\")\n",
    "    tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o1 slide for train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For test\n",
    "for win in slide_set:\n",
    "    tmp = tol_test[tol_test[\"time\"] >= tol_test[\"max_time\"] - win].groupby([UID], sort=False).agg(\n",
    "        {\n",
    "            UID: [\"count\"], \n",
    "            \"click_times\": OP_SET1,\n",
    "            \"time\": OP_SET1,\n",
    "            \"creative_id\": [\"nunique\"],\n",
    "            \"ad_id\": [\"nunique\"],\n",
    "            \"product_id\": [\"nunique\"],\n",
    "            \"product_category\": [\"nunique\"],\n",
    "            \"advertiser_id\": [\"nunique\"],\n",
    "            \"industry\": [\"nunique\"],\n",
    "        }\n",
    "    )\n",
    "    tmp.columns = bch_rencol(tmp.columns, prefix=f\"slide{win}_\")\n",
    "    tmp.to_pickle(f\"{UFEDIR}/test_stats_o1_slide{win}.pkl\")\n",
    "    tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o1 slide for test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Comb Key(O2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_list = [\"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "for win in slide_set:\n",
    "    for i in tqdm(range(1, 7)):\n",
    "        for cl in combinations(id_list, i):\n",
    "            key_set = [UID] + list(cl)\n",
    "            tmp = tol_train[tol_train[\"time\"] >= tol_train[\"max_time\"] - win].groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "            tmp.columns = bch_rencol(tmp.columns, prefix=f\"slide{win}_{'C'.join(key_set[1:-1])}_\")\n",
    "            tmp.to_pickle(f\"{UFEDIR}/train_stats_o2_slide{win}_{'C'.join(key_set)}.pkl\")\n",
    "            tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o2 slide for train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test\n",
    "for win in slide_set:\n",
    "    for i in tqdm(range(1, 7)):\n",
    "        for cl in combinations(id_list, i):\n",
    "            key_set = [UID] + list(cl)\n",
    "            tmp = tol_test[tol_test[\"time\"] >= tol_test[\"max_time\"] - win].groupby(key_set, sort=False)[key_set[-1:]].agg([\"count\"]).groupby([UID]).agg(OP_SET2)\n",
    "            tmp.columns = bch_rencol(tmp.columns, prefix=f\"slide{win}_{'C'.join(key_set[1:-1])}_\")\n",
    "            tmp.to_pickle(f\"{UFEDIR}/test_stats_o2_slide{win}_{'C'.join(key_set)}.pkl\")\n",
    "            tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tm_tw.check(\"Generate o2 slide for test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## (TOP Category) One-Hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tol_data = pd.concat([tol_train, tol_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dum_dic = {\n",
    "    \"creative_id\": 300,\n",
    "    \"ad_id\": 300,\n",
    "    \"product_id\": 100,\n",
    "    \"advertiser_id\": 100,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cid in dum_dic:\n",
    "    use_id = tol_data[cid].value_counts().head(dum_dic[cid]).index.values\n",
    "    tol_data[f\"{cid}_dummy\"] = tol_data[cid]\n",
    "    tol_data[f\"{cid}_dummy\"][~tol_data[f\"{cid}_dummy\"].isin(use_id)] = -999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for cid in  [\"time\", \"click_times\", \"creative_id\", \"ad_id\", \"product_id\", \"product_category\", \"advertiser_id\", \"industry\"]:\n",
    "    ocid = cid\n",
    "    if cid in dum_dic:\n",
    "        cid = f\"{cid}_dummy\"\n",
    "    tmp = tol_data[[UID, cid]].groupby([UID, cid], sort=False)[[cid]].agg([\"count\"]).unstack().fillna(0)\n",
    "    tmp.columns = bch_rencol(tmp.columns)\n",
    "    reduce_mem_usage(tmp).to_pickle(f\"{UFEDIR}/onehot_catecnt_{ocid}.pkl\")\n",
    "    tmp = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time diff (TODO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "355px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}