Data Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "import functools\n",
    "from IPython.display import Image\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import glob\n",
    "%matplotlib inline\n",
    "pd.set_option(\"display.precision\", 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "networks/2019-03_edges.csv\n",
      "      Source  Target  Weight      Type\n",
      "0          1       2       1  Directed\n",
      "1          1       5       5  Directed\n",
      "2          1      10       5  Directed\n",
      "3          1      15       1  Directed\n",
      "4          1      18       6  Directed\n",
      "...      ...     ...     ...       ...\n",
      "4923     203     189       4  Directed\n",
      "4924     203     190       1  Directed\n",
      "4925     203     197       1  Directed\n",
      "4926     203     201      12  Directed\n",
      "4927     203     202       1  Directed\n",
      "\n",
      "[4928 rows x 4 columns]\n",
      "networks/2019-04_edges.csv\n",
      "      Source  Target  Weight      Type\n",
      "0          0     1.0       1  Directed\n",
      "1          0    13.0      21  Directed\n",
      "2          0    20.0       1  Directed\n",
      "3          0    26.0       5  Directed\n",
      "4          0    61.0       5  Directed\n",
      "...      ...     ...     ...       ...\n",
      "7051     203   196.0       5  Directed\n",
      "7052     203   197.0       4  Directed\n",
      "7053     203   200.0       1  Directed\n",
      "7054     203   201.0       9  Directed\n",
      "7055     203   202.0       6  Directed\n",
      "\n",
      "[7056 rows x 4 columns]\n",
      "networks/2019-05_edges.csv\n",
      "      Source  Target  Weight      Type\n",
      "0          0     1.0       2  Directed\n",
      "1          0    13.0      51  Directed\n",
      "2          0    20.0       1  Directed\n",
      "3          0    23.0       1  Directed\n",
      "4          0    26.0       1  Directed\n",
      "...      ...     ...     ...       ...\n",
      "6988     203   195.0       1  Directed\n",
      "6989     203   196.0       5  Directed\n",
      "6990     203   199.0       2  Directed\n",
      "6991     203   201.0      14  Directed\n",
      "6992     203   202.0       3  Directed\n",
      "\n",
      "[6993 rows x 4 columns]\n",
      "networks/2019-06_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0     1.0       5  Directed\n",
      "1           0    13.0      76  Directed\n",
      "2           0     NaN       1  Directed\n",
      "3           0    14.0      74  Directed\n",
      "4           0    16.0       1  Directed\n",
      "...       ...     ...     ...       ...\n",
      "18526     204   188.0       6  Directed\n",
      "18527     204   197.0       1  Directed\n",
      "18528     204   201.0       4  Directed\n",
      "18529     204   202.0       6  Directed\n",
      "18530     204   203.0      50  Directed\n",
      "\n",
      "[18531 rows x 4 columns]\n",
      "networks/2019-07_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0     1.0       7  Directed\n",
      "1           0    13.0     109  Directed\n",
      "2           0    14.0     153  Directed\n",
      "3           0    15.0       1  Directed\n",
      "4           0    18.0       2  Directed\n",
      "...       ...     ...     ...       ...\n",
      "17783     204   188.0      13  Directed\n",
      "17784     204   196.0       2  Directed\n",
      "17785     204   201.0       7  Directed\n",
      "17786     204   202.0       5  Directed\n",
      "17787     204   203.0      58  Directed\n",
      "\n",
      "[17788 rows x 4 columns]\n",
      "networks/2019-08_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0     1.0      10  Directed\n",
      "1           0    13.0     129  Directed\n",
      "2           0    14.0     168  Directed\n",
      "3           0    15.0       1  Directed\n",
      "4           0    16.0       1  Directed\n",
      "...       ...     ...     ...       ...\n",
      "16872     204   187.0       1  Directed\n",
      "16873     204   188.0      18  Directed\n",
      "16874     204   201.0       7  Directed\n",
      "16875     204   202.0       1  Directed\n",
      "16876     204   203.0      57  Directed\n",
      "\n",
      "[16877 rows x 4 columns]\n",
      "networks/2019-09_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0     1.0       1  Directed\n",
      "1           0    13.0      79  Directed\n",
      "2           0    14.0      95  Directed\n",
      "3           0    16.0       1  Directed\n",
      "4           0    18.0       3  Directed\n",
      "...       ...     ...     ...       ...\n",
      "14479     204   181.0       1  Directed\n",
      "14480     204   187.0       1  Directed\n",
      "14481     204   188.0       8  Directed\n",
      "14482     204   201.0       4  Directed\n",
      "14483     204   203.0      49  Directed\n",
      "\n",
      "[14484 rows x 4 columns]\n",
      "networks/2019-10_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0     1.0       1  Directed\n",
      "1           0    13.0      61  Directed\n",
      "2           0    14.0      90  Directed\n",
      "3           0    15.0       1  Directed\n",
      "4           0    20.0       1  Directed\n",
      "...       ...     ...     ...       ...\n",
      "13439     204   178.0       1  Directed\n",
      "13440     204   181.0       2  Directed\n",
      "13441     204   188.0       5  Directed\n",
      "13442     204   201.0       1  Directed\n",
      "13443     204   203.0      33  Directed\n",
      "\n",
      "[13444 rows x 4 columns]\n",
      "networks/2019-11_edges.csv\n",
      "       Source  Target  Weight      Type\n",
      "0           0    13.0      37  Directed\n",
      "1           0    14.0      63  Directed\n",
      "2           0    20.0       1  Directed\n",
      "3           0    23.0       2  Directed\n",
      "4           0    26.0       1  Directed\n",
      "...       ...     ...     ...       ...\n",
      "10523     204   164.0       1  Directed\n",
      "10524     204   170.0       1  Directed\n",
      "10525     204   176.0       1  Directed\n",
      "10526     204   187.0       1  Directed\n",
      "10527     204   203.0      19  Directed\n",
      "\n",
      "[10528 rows x 4 columns]\n",
      "networks/2019-12_edges.csv\n",
      "      Source  Target  Weight      Type\n",
      "0          0      13      30  Directed\n",
      "1          0      14      45  Directed\n",
      "2          0      26       1  Directed\n",
      "3          0      62       6  Directed\n",
      "4          0      67       1  Directed\n",
      "...      ...     ...     ...       ...\n",
      "7700     204     129       5  Directed\n",
      "7701     204     161       3  Directed\n",
      "7702     204     188       1  Directed\n",
      "7703     204     196       1  Directed\n",
      "7704     204     203      12  Directed\n",
      "\n",
      "[7705 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "from processing_helper import extract_data\n",
    "paths = [\n",
    "    r'./data',\n",
    "]\n",
    "\n",
    "result_columns = ['mean', 'std', 'min', 'max', 'leafs', 'time', 'ratio', 'name']\n",
    "results = {}\n",
    "stations = np.array([])\n",
    "for path in paths:\n",
    "    all_files = sorted(glob.glob(path + \"/*.csv\"))\n",
    "    \n",
    "    for idx, filename in enumerate(all_files):\n",
    "        data, station_map = extract_data(filename)\n",
    "        \n",
    "        grouped_data = data.groupby(['rental_place', 'return_place']).size().reset_index()\n",
    "        grouped_data = grouped_data.rename(columns={\n",
    "            'rental_place': 'Source',\n",
    "            'return_place': 'Target',\n",
    "            0: 'Weight'\n",
    "        })\n",
    "        grouped_data['Type'] = 'Directed'\n",
    "        new_filename = (filename.split('_')[-1]).split('.')[0]\n",
    "        results['networks/'+new_filename+'_edges.csv'] = grouped_data\n",
    "        \n",
    "        print(type(station_map))\n",
    "        stations = np.concatenate((stations, station_map))\n",
    "\n",
    "        \n",
    "stations = np.unique(stations)\n",
    "mapping_dict = {stations[i]: i for i in range(0, len(stations))}\n",
    "stations = pd.DataFrame(stations, columns=['name'])\n",
    "stations.index.rename('value')\n",
    "stations.to_csv('networks/nodes.csv', index=True, index_label='value')\n",
    "\n",
    "for filename, data in results.items():\n",
    "    print(filename)\n",
    "\n",
    "    for feature_name in ['Source', 'Target']:\n",
    "        data[feature_name] = data[feature_name].map(mapping_dict)\n",
    "        \n",
    "    data['Source'] = pd.to_numeric(data['Source'], downcast='integer')\n",
    "    data['Target'] = pd.to_numeric(data['Target'], downcast='integer')\n",
    "    data = data.dropna()\n",
    "    data.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}