dob_violations_EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GOAL: Predict violation type via violation type code using other features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./data/dob_violations_data.csv', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bbl</th>\n",
       "      <th>isndobbisviol</th>\n",
       "      <th>boro</th>\n",
       "      <th>bin</th>\n",
       "      <th>block</th>\n",
       "      <th>lot</th>\n",
       "      <th>issuedate</th>\n",
       "      <th>violationtypecode</th>\n",
       "      <th>violationnumber</th>\n",
       "      <th>housenumber</th>\n",
       "      <th>street</th>\n",
       "      <th>dispositiondate</th>\n",
       "      <th>dispositioncomments</th>\n",
       "      <th>devicenumber</th>\n",
       "      <th>description</th>\n",
       "      <th>ecbnumber</th>\n",
       "      <th>number</th>\n",
       "      <th>violationcategory</th>\n",
       "      <th>violationtype</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.014790e+09</td>\n",
       "      <td>1018012</td>\n",
       "      <td>3</td>\n",
       "      <td>3039615.0</td>\n",
       "      <td>01479</td>\n",
       "      <td>00001</td>\n",
       "      <td>2005-01-31</td>\n",
       "      <td>LL6291</td>\n",
       "      <td>15504</td>\n",
       "      <td>880</td>\n",
       "      <td>QUINCY STREET</td>\n",
       "      <td>2005-07-21</td>\n",
       "      <td>IRICIA DISMISSED FULL PENALTY PAID $500       ...</td>\n",
       "      <td>00081537</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V*013105LL629115504</td>\n",
       "      <td>V*-DOB VIOLATION - DISMISSED</td>\n",
       "      <td>LL6291-LOCAL LAW 62/91 - BOILERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.014350e+09</td>\n",
       "      <td>798462</td>\n",
       "      <td>1</td>\n",
       "      <td>1044200.0</td>\n",
       "      <td>01435</td>\n",
       "      <td>00016</td>\n",
       "      <td>2002-03-26</td>\n",
       "      <td>LL6291</td>\n",
       "      <td>04391</td>\n",
       "      <td>330</td>\n",
       "      <td>EAST 61 STREET</td>\n",
       "      <td>2006-06-23</td>\n",
       "      <td>CIAMGZ CANCELLED DEED SUB NEW OWNER AS OF     ...</td>\n",
       "      <td>00111092</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V*032602LL629104391</td>\n",
       "      <td>V*-DOB VIOLATION - DISMISSED</td>\n",
       "      <td>LL6291-LOCAL LAW 62/91 - BOILERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.016710e+09</td>\n",
       "      <td>251606</td>\n",
       "      <td>1</td>\n",
       "      <td>1052648.0</td>\n",
       "      <td>01671</td>\n",
       "      <td>00049</td>\n",
       "      <td>1993-03-18</td>\n",
       "      <td>LL6291</td>\n",
       "      <td>10195</td>\n",
       "      <td>1936</td>\n",
       "      <td>2 AVENUE</td>\n",
       "      <td>2000-08-08</td>\n",
       "      <td>VTUCIA CANCELLED, BLDG HEATED BY 1932         ...</td>\n",
       "      <td>00900563</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V*031893LL629110195</td>\n",
       "      <td>V*-DOB VIOLATION - DISMISSED</td>\n",
       "      <td>LL6291-LOCAL LAW 62/91 - BOILERS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.025230e+09</td>\n",
       "      <td>1071206</td>\n",
       "      <td>2</td>\n",
       "      <td>2097167.0</td>\n",
       "      <td>02523</td>\n",
       "      <td>00061</td>\n",
       "      <td>2006-02-02</td>\n",
       "      <td>E</td>\n",
       "      <td>9444/153715</td>\n",
       "      <td>950</td>\n",
       "      <td>UNIVERSITY AVEN</td>\n",
       "      <td>2007-06-11</td>\n",
       "      <td>PPN203 AOC SUBMITTED ON 06/08/2007 BY DYNAMIC ...</td>\n",
       "      <td>002P1188</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V*020206E9444/153715</td>\n",
       "      <td>V*-DOB VIOLATION - Resolved</td>\n",
       "      <td>E-ELEVATOR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.031440e+09</td>\n",
       "      <td>2027969</td>\n",
       "      <td>2</td>\n",
       "      <td>2013386.0</td>\n",
       "      <td>03144</td>\n",
       "      <td>00066</td>\n",
       "      <td>2016-02-05</td>\n",
       "      <td>IMEGNCY</td>\n",
       "      <td>9246</td>\n",
       "      <td>2094</td>\n",
       "      <td>VALENTINE AVENUE</td>\n",
       "      <td>2016-02-11</td>\n",
       "      <td>BLDING IS NOT DEMOLISHED. FENCE ERECTED IN PLA...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>02/05/16 DOB : DEC 9246 HAS BEEN UPLOADED AND ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>V*020516IMEGNCY9246</td>\n",
       "      <td>V*-DOB VIOLATION - DISMISSED</td>\n",
       "      <td>IMEGNCY-IMMEDIATE EMERGENCY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            bbl  isndobbisviol boro        bin  block    lot   issuedate  \\\n",
       "0  3.014790e+09        1018012    3  3039615.0  01479  00001  2005-01-31   \n",
       "1  1.014350e+09         798462    1  1044200.0  01435  00016  2002-03-26   \n",
       "2  1.016710e+09         251606    1  1052648.0  01671  00049  1993-03-18   \n",
       "3  2.025230e+09        1071206    2  2097167.0  02523  00061  2006-02-02   \n",
       "4  2.031440e+09        2027969    2  2013386.0  03144  00066  2016-02-05   \n",
       "\n",
       "  violationtypecode violationnumber housenumber            street  \\\n",
       "0            LL6291           15504         880     QUINCY STREET   \n",
       "1            LL6291           04391         330    EAST 61 STREET   \n",
       "2            LL6291           10195        1936          2 AVENUE   \n",
       "3                 E     9444/153715         950   UNIVERSITY AVEN   \n",
       "4           IMEGNCY            9246        2094  VALENTINE AVENUE   \n",
       "\n",
       "  dispositiondate                                dispositioncomments  \\\n",
       "0      2005-07-21  IRICIA DISMISSED FULL PENALTY PAID $500       ...   \n",
       "1      2006-06-23  CIAMGZ CANCELLED DEED SUB NEW OWNER AS OF     ...   \n",
       "2      2000-08-08  VTUCIA CANCELLED, BLDG HEATED BY 1932         ...   \n",
       "3      2007-06-11  PPN203 AOC SUBMITTED ON 06/08/2007 BY DYNAMIC ...   \n",
       "4      2016-02-11  BLDING IS NOT DEMOLISHED. FENCE ERECTED IN PLA...   \n",
       "\n",
       "  devicenumber                                        description ecbnumber  \\\n",
       "0     00081537                                                NaN       NaN   \n",
       "1     00111092                                                NaN       NaN   \n",
       "2     00900563                                                NaN       NaN   \n",
       "3     002P1188                                                NaN       NaN   \n",
       "4          NaN  02/05/16 DOB : DEC 9246 HAS BEEN UPLOADED AND ...       NaN   \n",
       "\n",
       "                 number             violationcategory  \\\n",
       "0   V*013105LL629115504  V*-DOB VIOLATION - DISMISSED   \n",
       "1   V*032602LL629104391  V*-DOB VIOLATION - DISMISSED   \n",
       "2   V*031893LL629110195  V*-DOB VIOLATION - DISMISSED   \n",
       "3  V*020206E9444/153715   V*-DOB VIOLATION - Resolved   \n",
       "4   V*020516IMEGNCY9246  V*-DOB VIOLATION - DISMISSED   \n",
       "\n",
       "                      violationtype  \n",
       "0  LL6291-LOCAL LAW 62/91 - BOILERS  \n",
       "1  LL6291-LOCAL LAW 62/91 - BOILERS  \n",
       "2  LL6291-LOCAL LAW 62/91 - BOILERS  \n",
       "3                        E-ELEVATOR  \n",
       "4       IMEGNCY-IMMEDIATE EMERGENCY  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['bbl', 'isndobbisviol', 'boro', 'bin', 'block', 'lot', 'issuedate',\n",
       "       'violationtypecode', 'violationnumber', 'housenumber', 'street',\n",
       "       'dispositiondate', 'dispositioncomments', 'devicenumber', 'description',\n",
       "       'ecbnumber', 'number', 'violationcategory', 'violationtype'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bbl :  215454\n",
      "isndobbisviol :  2020288\n",
      "boro :  12\n",
      "bin :  218799\n",
      "block :  13247\n",
      "lot :  2414\n",
      "issuedate :  12654\n",
      "violationtypecode :  52\n",
      "violationnumber :  799405\n",
      "housenumber :  45077\n",
      "street :  15766\n",
      "dispositiondate :  10078\n",
      "dispositioncomments :  466754\n",
      "devicenumber :  353184\n",
      "description :  100615\n",
      "ecbnumber :  146632\n",
      "number :  2013506\n",
      "violationcategory :  14\n",
      "violationtype :  46\n"
     ]
    }
   ],
   "source": [
    "for column in df.columns:\n",
    "    print(column, ': ', df[column].nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "y = violationcategory (as it reflects violationtype; so drop violationtype column altogether)\n",
    "\n",
    "1. trying categorical approach first\n",
    "    - problem: most features seem non-categorical\n",
    "2. try neural network with concatenated features\n",
    "    - remove irrelevant features like violation number first\n",
    "    - perform concatenation\n",
    "    - train NN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_drop = ['bbl', 'isndobbisviol', 'bin', 'block', 'lot', 'violationnumber',\\\n",
    "                'devicenumber', 'dispositioncomments', 'ecbnumber', 'number',\\\n",
    "                'violationtype', 'violationcategory']\n",
    "processed_df = df.drop(cols_to_drop, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['boro', 'issuedate', 'violationtypecode', 'housenumber', 'street',\n",
       "       'dispositiondate', 'description'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.Series(processed_df.loc[:, processed_df.columns != 'violationtypecode'].fillna('').values.tolist()).str.join('')\n",
    "y = pd.Series(processed_df['violationtypecode'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                32005-01-31880QUINCY STREET2005-07-21\n",
       "1               12002-03-26330EAST 61 STREET2006-06-23\n",
       "2                    11993-03-1819362 AVENUE2000-08-08\n",
       "3              22006-02-02950UNIVERSITY AVEN2007-06-11\n",
       "4    22016-02-052094VALENTINE AVENUE2016-02-1102/05...\n",
       "dtype: object"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     LL6291\n",
       "1     LL6291\n",
       "2     LL6291\n",
       "3          E\n",
       "4    IMEGNCY\n",
       "Name: violationtypecode, dtype: object"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'LL6291'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Constructing dict for looking up actual violationtypecode\n",
    "y_keys = []\n",
    "y_vals = []\n",
    "for key, val in enumerate(y.unique()):\n",
    "    y_keys.append(key)\n",
    "    y_vals.append(val)\n",
    "y_dict = dict(zip(y_keys, y_vals))\n",
    "y_dict[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    1\n",
       "4    2\n",
       "Name: violationtypecode, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for key in y_dict:\n",
    "    y.replace(to_replace=y_dict[key], value=key, inplace=True)\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([14,  5,  0,  1, 10,  4, 11, 27,  6,  7, 39, 25,  3, 18, 16, 13, 23,\n",
       "       12, 15, 17,  9,  2, 28, 30, 29, 26, 51,  8, 35, 32, 20, 19, 31, 21,\n",
       "       24, 40, 33, 36, 22, 37, 34, 42, 38, 43, 46, 49, 47, 48, 50, 44, 41,\n",
       "       45], dtype=int64)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.unique() # A classification with 52 categories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "from keras.models import Model\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_list = X_train.tolist()\n",
    "X_test_list = X_test.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvec = CountVectorizer()\n",
    "cvec.fit(X_train_list)\n",
    "X_train_vec = cvec.transform(X_train_list)\n",
    "X_test_vec = cvec.transform(X_test_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<20202x25805 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 165586 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Embedding to reduce dimensionality of feature vectors\n",
    "def simpleNN(input_dim, output_dim):\n",
    "    model = Sequential()\n",
    "    model.add(layers.Dense(8, input_dim=input_dim, activation='relu', use_bias=True))\n",
    "    model.add(layers.Dense(8, activation='relu', use_bias=True))\n",
    "#     model.add(layers.Dropout(rate=0.6))\n",
    "    model.add(layers.Dense(output_dim, activation='sigmoid'))\n",
    "    model.compile(loss='sparse_categorical_crossentropy',\n",
    "                   optimizer='adam',\n",
    "                   metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2020288,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "20202/20202 [==============================] - 7s 332us/step - loss: 3.7700 - acc: 0.3102\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_25 (Dense)             (None, 8)                 206448    \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 8)                 72        \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 52)                468       \n",
      "=================================================================\n",
      "Total params: 206,988\n",
      "Trainable params: 206,988\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n",
      "Training Accuracy: 0.3173\n",
      "T esting Accuracy: 0.3217\n"
     ]
    }
   ],
   "source": [
    "input_dim = X_train_vec.shape[1]\n",
    "output_dim = y.nunique()\n",
    "\n",
    "model = simpleNN(input_dim, output_dim)\n",
    "model.fit(X_train_vec, y_train,\n",
    "          epochs=1,\n",
    "          verbose=1,\n",
    "          validation_data=None,\n",
    "          batch_size=256)\n",
    "print(model.summary())\n",
    "\n",
    "loss, train_accuracy = model.evaluate(X_train_vec, y_train, verbose=False)\n",
    "print('Training Accuracy: {:.4f}'.format(train_accuracy))\n",
    "# loss, val_accuracy = model.evaluate(X_val_vec, y_val, verbose=False)\n",
    "# print('Validation Accuracy: {:.4f}'.format(val_accuracy))\n",
    "loss, test_accuracy = model.evaluate(X_test_vec, y_test, verbose=False)\n",
    "print('Testing Accuracy: {:.4f}'.format(test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}