_2_4_gen_acc_sum_counts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T13:31:56.671234Z",
     "start_time": "2018-04-20T13:31:55.755132Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from utils import load_pickle,dump_pickle, get_nominal_dfal, feats_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T13:31:56.690001Z",
     "start_time": "2018-04-20T13:31:56.673980Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_id_global_sum_count(last_da=23,\n",
    "                            stats_feats=[\n",
    "                                'item_id', 'shop_id', 'user_id',\n",
    "                                'item_brand_id', 'item_city_id', 'hm'\n",
    "                            ]):\n",
    "    dfal = get_nominal_dfal()\n",
    "    dfal = dfal.loc[dfal.da < last_da, stats_feats]\n",
    "    for feat in tqdm(stats_feats):\n",
    "        feat_path = os.path.join(feats_root,'global_count_' + feat + '_lastda' + str(last_da) + '.pkl')\n",
    "        if os.path.exists(feat_path):\n",
    "            print('found ' + feat_path)\n",
    "        else:\n",
    "            print('generating ' + feat_path)\n",
    "            feat_count_sum = pd.DataFrame(\n",
    "                dfal.groupby(feat).size()).reset_index().rename(\n",
    "                    columns={0: 'agg_' + feat + '_sum_count'})\n",
    "            dump_pickle(feat_count_sum, feat_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T13:31:56.705838Z",
     "start_time": "2018-04-20T13:31:56.692918Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_global_count_sum(data,\n",
    "                         last_da=23,\n",
    "                         stats_feats=[\n",
    "                             'item_id', 'shop_id', 'user_id', 'item_brand_id',\n",
    "                             'item_city_id'\n",
    "                         ]):\n",
    "    \"\"\"\n",
    "    添加ID出现次数，根据ID_name拼接\n",
    "    \"\"\"\n",
    "    for feat in tqdm(stats_feats):\n",
    "        feat_path = os.path.join(\n",
    "            feats_root,\n",
    "            'global_count_' + feat + '_lastda' + str(last_da) + '.pkl')\n",
    "        if not os.path.exists(feat_path):\n",
    "            gen_id_global_sum_count(last_da, [feat])\n",
    "        feat_count_sum = load_pickle(feat_path)\n",
    "        data = data.merge(feat_count_sum, 'left', [feat])\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T13:31:58.249467Z",
     "start_time": "2018-04-20T13:31:56.709302Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 32.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda24.pkl\n",
      "generating ./feats/global_count_shop_id_lastda24.pkl\n",
      "generating ./feats/global_count_user_id_lastda24.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda24.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda24.pkl\n",
      "generating ./feats/global_count_hm_lastda24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 39.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda23.pkl\n",
      "generating ./feats/global_count_shop_id_lastda23.pkl\n",
      "generating ./feats/global_count_user_id_lastda23.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda23.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda23.pkl\n",
      "generating ./feats/global_count_hm_lastda23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 43.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda22.pkl\n",
      "generating ./feats/global_count_shop_id_lastda22.pkl\n",
      "generating ./feats/global_count_user_id_lastda22.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda22.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda22.pkl\n",
      "generating ./feats/global_count_hm_lastda22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 48.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda21.pkl\n",
      "generating ./feats/global_count_shop_id_lastda21.pkl\n",
      "generating ./feats/global_count_user_id_lastda21.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda21.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda21.pkl\n",
      "generating ./feats/global_count_hm_lastda21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 44.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda20.pkl\n",
      "generating ./feats/global_count_shop_id_lastda20.pkl\n",
      "generating ./feats/global_count_user_id_lastda20.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda20.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda20.pkl\n",
      "generating ./feats/global_count_hm_lastda20.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 6/6 [00:00<00:00, 82.50it/s]\n",
      "100%|██████████| 6/6 [00:00<00:00, 131.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/global_count_item_id_lastda19.pkl\n",
      "generating ./feats/global_count_shop_id_lastda19.pkl\n",
      "generating ./feats/global_count_user_id_lastda19.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda19.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda19.pkl\n",
      "generating ./feats/global_count_hm_lastda19.pkl\n",
      "generating ./feats/global_count_item_id_lastda18.pkl\n",
      "generating ./feats/global_count_shop_id_lastda18.pkl\n",
      "generating ./feats/global_count_user_id_lastda18.pkl\n",
      "generating ./feats/global_count_item_brand_id_lastda18.pkl\n",
      "generating ./feats/global_count_item_city_id_lastda18.pkl\n",
      "generating ./feats/global_count_hm_lastda18.pkl\n",
      "all done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ =='__main__':\n",
    "    gen_id_global_sum_count(24)\n",
    "    gen_id_global_sum_count(23)\n",
    "    gen_id_global_sum_count(22)\n",
    "    gen_id_global_sum_count(21)\n",
    "    gen_id_global_sum_count(20)\n",
    "    gen_id_global_sum_count(19)\n",
    "    gen_id_global_sum_count(18)\n",
    "    print('all done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}