Training_data_preparation_for_spaCy_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "least-colonial",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:21.778082Z",
     "start_time": "2021-03-18T15:00:20.928046Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import re\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "rotary-needle",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:22.885393Z",
     "start_time": "2021-03-18T15:00:22.359704Z"
    }
   },
   "outputs": [],
   "source": [
    "raw_df = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "checked-survey",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:23.716364Z",
     "start_time": "2021-03-18T15:00:23.677468Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>raw_address</th>\n",
       "      <th>POI/street</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>jl kapuk timur delta sili iii lippo cika 11 a ...</td>\n",
       "      <td>/jl kapuk timur delta sili iii lippo cika</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>aye, jati sampurna</td>\n",
       "      <td>/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>setu siung 119 rt 5 1 13880 cipayung</td>\n",
       "      <td>/siung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>toko dita, kertosono</td>\n",
       "      <td>toko dita/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>jl. orde baru</td>\n",
       "      <td>/jl. orde baru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299995</th>\n",
       "      <td>299995</td>\n",
       "      <td>jend ahmad yani 331 kertasari ciamis</td>\n",
       "      <td>/jend ahmad yani</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299996</th>\n",
       "      <td>299996</td>\n",
       "      <td>raya cila kko, cilandak timur kel.</td>\n",
       "      <td>/raya cila kko</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299997</th>\n",
       "      <td>299997</td>\n",
       "      <td>tanjung gusta jl. yaya 2 no 17</td>\n",
       "      <td>/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299998</th>\n",
       "      <td>299998</td>\n",
       "      <td>jalan cipadu jaya taman asri gang bijaksana 3 ...</td>\n",
       "      <td>taman asri/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299999</th>\n",
       "      <td>299999</td>\n",
       "      <td>jaya maspion permata blok beryl b2  no.58</td>\n",
       "      <td>jaya maspion permata/</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id                                        raw_address  \\\n",
       "0            0  jl kapuk timur delta sili iii lippo cika 11 a ...   \n",
       "1            1                                 aye, jati sampurna   \n",
       "2            2               setu siung 119 rt 5 1 13880 cipayung   \n",
       "3            3                               toko dita, kertosono   \n",
       "4            4                                      jl. orde baru   \n",
       "...        ...                                                ...   \n",
       "299995  299995               jend ahmad yani 331 kertasari ciamis   \n",
       "299996  299996                 raya cila kko, cilandak timur kel.   \n",
       "299997  299997                     tanjung gusta jl. yaya 2 no 17   \n",
       "299998  299998  jalan cipadu jaya taman asri gang bijaksana 3 ...   \n",
       "299999  299999          jaya maspion permata blok beryl b2  no.58   \n",
       "\n",
       "                                       POI/street  \n",
       "0       /jl kapuk timur delta sili iii lippo cika  \n",
       "1                                               /  \n",
       "2                                          /siung  \n",
       "3                                      toko dita/  \n",
       "4                                  /jl. orde baru  \n",
       "...                                           ...  \n",
       "299995                           /jend ahmad yani  \n",
       "299996                             /raya cila kko  \n",
       "299997                                          /  \n",
       "299998                                taman asri/  \n",
       "299999                      jaya maspion permata/  \n",
       "\n",
       "[300000 rows x 3 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = raw_df.copy()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "authentic-transport",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:28.942041Z",
     "start_time": "2021-03-18T15:00:28.692010Z"
    }
   },
   "outputs": [],
   "source": [
    "# split columns for POI and street\n",
    "df['POI'] = df['POI/street'].apply(lambda x: x.split('/')[0])\n",
    "df['street'] = df['POI/street'].apply(lambda x: x.split('/')[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "second-court",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:32.509020Z",
     "start_time": "2021-03-18T15:00:32.488111Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>raw_address</th>\n",
       "      <th>POI/street</th>\n",
       "      <th>POI</th>\n",
       "      <th>street</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>jl kapuk timur delta sili iii lippo cika 11 a ...</td>\n",
       "      <td>/jl kapuk timur delta sili iii lippo cika</td>\n",
       "      <td></td>\n",
       "      <td>jl kapuk timur delta sili iii lippo cika</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>aye, jati sampurna</td>\n",
       "      <td>/</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>setu siung 119 rt 5 1 13880 cipayung</td>\n",
       "      <td>/siung</td>\n",
       "      <td></td>\n",
       "      <td>siung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>toko dita, kertosono</td>\n",
       "      <td>toko dita/</td>\n",
       "      <td>toko dita</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>jl. orde baru</td>\n",
       "      <td>/jl. orde baru</td>\n",
       "      <td></td>\n",
       "      <td>jl. orde baru</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299995</th>\n",
       "      <td>299995</td>\n",
       "      <td>jend ahmad yani 331 kertasari ciamis</td>\n",
       "      <td>/jend ahmad yani</td>\n",
       "      <td></td>\n",
       "      <td>jend ahmad yani</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299996</th>\n",
       "      <td>299996</td>\n",
       "      <td>raya cila kko, cilandak timur kel.</td>\n",
       "      <td>/raya cila kko</td>\n",
       "      <td></td>\n",
       "      <td>raya cila kko</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299997</th>\n",
       "      <td>299997</td>\n",
       "      <td>tanjung gusta jl. yaya 2 no 17</td>\n",
       "      <td>/</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299998</th>\n",
       "      <td>299998</td>\n",
       "      <td>jalan cipadu jaya taman asri gang bijaksana 3 ...</td>\n",
       "      <td>taman asri/</td>\n",
       "      <td>taman asri</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299999</th>\n",
       "      <td>299999</td>\n",
       "      <td>jaya maspion permata blok beryl b2  no.58</td>\n",
       "      <td>jaya maspion permata/</td>\n",
       "      <td>jaya maspion permata</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id                                        raw_address  \\\n",
       "0            0  jl kapuk timur delta sili iii lippo cika 11 a ...   \n",
       "1            1                                 aye, jati sampurna   \n",
       "2            2               setu siung 119 rt 5 1 13880 cipayung   \n",
       "3            3                               toko dita, kertosono   \n",
       "4            4                                      jl. orde baru   \n",
       "...        ...                                                ...   \n",
       "299995  299995               jend ahmad yani 331 kertasari ciamis   \n",
       "299996  299996                 raya cila kko, cilandak timur kel.   \n",
       "299997  299997                     tanjung gusta jl. yaya 2 no 17   \n",
       "299998  299998  jalan cipadu jaya taman asri gang bijaksana 3 ...   \n",
       "299999  299999          jaya maspion permata blok beryl b2  no.58   \n",
       "\n",
       "                                       POI/street                   POI  \\\n",
       "0       /jl kapuk timur delta sili iii lippo cika                         \n",
       "1                                               /                         \n",
       "2                                          /siung                         \n",
       "3                                      toko dita/             toko dita   \n",
       "4                                  /jl. orde baru                         \n",
       "...                                           ...                   ...   \n",
       "299995                           /jend ahmad yani                         \n",
       "299996                             /raya cila kko                         \n",
       "299997                                          /                         \n",
       "299998                                taman asri/            taman asri   \n",
       "299999                      jaya maspion permata/  jaya maspion permata   \n",
       "\n",
       "                                          street  \n",
       "0       jl kapuk timur delta sili iii lippo cika  \n",
       "1                                                 \n",
       "2                                          siung  \n",
       "3                                                 \n",
       "4                                  jl. orde baru  \n",
       "...                                          ...  \n",
       "299995                           jend ahmad yani  \n",
       "299996                             raya cila kko  \n",
       "299997                                            \n",
       "299998                                            \n",
       "299999                                            \n",
       "\n",
       "[300000 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "painted-member",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T15:00:54.668874Z",
     "start_time": "2021-03-18T15:00:54.650940Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                            5\n",
       "raw_address    raya samb gede, 299 toko bb kids\n",
       "POI/street          toko bb kids/raya samb gede\n",
       "POI                                toko bb kids\n",
       "street                           raya samb gede\n",
       "Name: 5, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "metric-cleaning",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-15T07:43:01.583115Z",
     "start_time": "2021-03-15T07:43:01.575136Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                                      6230\n",
       "raw_address    indo alam (da pemb, tegalsawah karawang timur\n",
       "POI/street                               /indo alam (da pemb\n",
       "POI                                                         \n",
       "street                                    indo alam (da pemb\n",
       "Name: 6230, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[6230]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "vocal-prison",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-15T03:44:14.482289Z",
     "start_time": "2021-03-15T03:44:14.466357Z"
    }
   },
   "outputs": [],
   "source": [
    "# example for data format to train on spaCy\n",
    "train = [ # entities exact start_letter, but +1 at end_letter like range()\n",
    "    (\"jl kapuk timur delta sili iii lippo cika 11 a cicau cikarang pusat\", {\"entities\": [(0,40,\"STREET\")]}),\n",
    "    (\"raya samb gede, 299 toko bb kids\", {\"entities\": [(20,31,\"POI\"), (0,14,\"STREET\")]})\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "general-glory",
   "metadata": {},
   "source": [
    "## Try using Regular Expression to preprocess and extract (doesn't work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "played-traffic",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T16:47:02.024742Z",
     "start_time": "2021-03-18T16:46:39.151380Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████| 300000/300000 [00:22<00:00, 13136.49it/s]\n"
     ]
    }
   ],
   "source": [
    "train_data = []\n",
    "entities = {\"entities\": []}\n",
    "for row in tqdm(df.itertuples(index = False), total = len(df)):\n",
    "    if row[3] != '': # POI\n",
    "        try:\n",
    "            match = re.search(row[3], row[1])\n",
    "            if match != None:\n",
    "                entities['entities'].append((match.start(), match.end(), \"POI\"))\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    if row[4] != '': # street\n",
    "        try:\n",
    "            match = re.search(row[4], row[1])\n",
    "            if match != None:\n",
    "                entities['entities'].append((match.start(), match.end(), \"STREET\"))\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    # in case of, poi/street doesn't match the raw_address\n",
    "    # need to improve this later because this has to fix/fill the missing POI/stress\n",
    "#     for index, element in enumerate(entities['entities']):\n",
    "#         if element[0] == -1:\n",
    "#             entities['entities'].pop(index)\n",
    "    \n",
    "#     #in case of, both POI and street are uncomplete\n",
    "#     for index, element in enumerate(entities['entities']):\n",
    "#         if element[0] == -1:\n",
    "#             entities['entities'].pop(index)\n",
    "    \n",
    "    train_data.append((row[1], entities))\n",
    "    entities = {\"entities\": []}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "regulated-kennedy",
   "metadata": {},
   "source": [
    "## A better way to preprocess data to be ready to train on spaCy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "speaking-momentum",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T17:57:02.349342Z",
     "start_time": "2021-03-18T17:56:59.796624Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 300000/300000 [00:02<00:00, 120394.02it/s]\n"
     ]
    }
   ],
   "source": [
    "# in case of using .find and there is a overlap text, we can use .find and specify start argument to be\n",
    "# an index of '/', this will do!!!\n",
    "train_data_final = []\n",
    "entities = {\"entities\": []}\n",
    "for row in tqdm(df.itertuples(index = False), total = len(df)):\n",
    "    if row[3] != '': # POI\n",
    "        entities['entities'].append((row[1].find(row[3]), row[1].find(row[3]) + len(row[3]), 'POI'))\n",
    "    \n",
    "    # in case of, poi doesn't match the raw_address\n",
    "    if entities['entities'] != [] and entities['entities'][0][0] == -1:\n",
    "        entities['entities'].pop()\n",
    "    \n",
    "    if row[4] != '': # street\n",
    "        # in case of overlapping\n",
    "        if entities['entities'] != [] and ((row[1].find(row[4]) >= entities['entities'][0][0]) and (row[1].find(row[4]) + len(row[4]) <= entities['entities'][0][1])):\n",
    "            # start find after the last char of POI\n",
    "            entities['entities'].append((row[1].find(row[4], row[1].find(row[3]) + len(row[3])), \n",
    "                                         row[1].find(row[4], row[1].find(row[3]) + len(row[3])) + len(row[4]), 'STREET'))\n",
    "        else:\n",
    "            entities['entities'].append((row[1].find(row[4]), row[1].find(row[4]) + len(row[4]), 'STREET'))\n",
    "\n",
    "    #in case of, both POI and street are uncomplete\n",
    "#     for index, element in enumerate(entities['entities']):\n",
    "#         if element[0] == -1:\n",
    "#             entities['entities'].pop(index)\n",
    "    \n",
    "    #in case of, both POI and street are uncomplete\n",
    "    for index, element in enumerate(entities['entities']):\n",
    "        if element[0] == -1:\n",
    "            entities['entities'].pop(index)\n",
    "    \n",
    "    train_data_final.append((row[1], entities))\n",
    "    entities = {\"entities\": []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "angry-minority",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T17:57:05.005595Z",
     "start_time": "2021-03-18T17:57:04.732301Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame(train_data)\n",
    "result_final = pd.DataFrame(train_data_final)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "legislative-narrative",
   "metadata": {},
   "source": [
    "## Compare to make sure the final version is better than the first one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "original-superior",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T17:23:16.509812Z",
     "start_time": "2021-03-18T17:23:16.488868Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jl kapuk timur delta sili iii lippo cika 11 a ...</td>\n",
       "      <td>{'entities': [(0, 40, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aye, jati sampurna</td>\n",
       "      <td>{'entities': []}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>setu siung 119 rt 5 1 13880 cipayung</td>\n",
       "      <td>{'entities': [(5, 10, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>toko dita, kertosono</td>\n",
       "      <td>{'entities': [(0, 9, 'POI')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jl. orde baru</td>\n",
       "      <td>{'entities': [(0, 13, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>cau terr, gal,</td>\n",
       "      <td>{'entities': []}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>taman kota kedaung kali angke gg.h.musanif no ...</td>\n",
       "      <td>{'entities': [(0, 10, 'POI')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>aren jaya sumb iv 319 rt 3 10 17111 bekasi timur</td>\n",
       "      <td>{'entities': [(10, 17, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>shi mel, nanggalo</td>\n",
       "      <td>{'entities': [(0, 7, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>pamoyanan town house blok g no 1</td>\n",
       "      <td>{'entities': []}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    0  \\\n",
       "0   jl kapuk timur delta sili iii lippo cika 11 a ...   \n",
       "1                                  aye, jati sampurna   \n",
       "2                setu siung 119 rt 5 1 13880 cipayung   \n",
       "3                                toko dita, kertosono   \n",
       "4                                       jl. orde baru   \n",
       "..                                                ...   \n",
       "95                                     cau terr, gal,   \n",
       "96  taman kota kedaung kali angke gg.h.musanif no ...   \n",
       "97   aren jaya sumb iv 319 rt 3 10 17111 bekasi timur   \n",
       "98                                  shi mel, nanggalo   \n",
       "99                   pamoyanan town house blok g no 1   \n",
       "\n",
       "                                     1  \n",
       "0    {'entities': [(0, 40, 'STREET')]}  \n",
       "1                     {'entities': []}  \n",
       "2    {'entities': [(5, 10, 'STREET')]}  \n",
       "3        {'entities': [(0, 9, 'POI')]}  \n",
       "4    {'entities': [(0, 13, 'STREET')]}  \n",
       "..                                 ...  \n",
       "95                    {'entities': []}  \n",
       "96      {'entities': [(0, 10, 'POI')]}  \n",
       "97  {'entities': [(10, 17, 'STREET')]}  \n",
       "98    {'entities': [(0, 7, 'STREET')]}  \n",
       "99                    {'entities': []}  \n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_final.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "stone-chain",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T17:58:07.529204Z",
     "start_time": "2021-03-18T17:58:07.416513Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>983</th>\n",
       "      <td>knalpot putra mahakam, maha 11 65111 klojen</td>\n",
       "      <td>{'entities': [(0, 21, 'POI'), (14, 18, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2233</th>\n",
       "      <td>kintawani car wash, kinta, no c 16</td>\n",
       "      <td>{'entities': [(0, 18, 'POI'), (0, 5, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2921</th>\n",
       "      <td>angsa, 26 cntrl + p,</td>\n",
       "      <td>{'entities': [(0, 5, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3256</th>\n",
       "      <td>dr. gema nazri yanni, m.ked(ped), sp.a bunga l...</td>\n",
       "      <td>{'entities': [(39, 48, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4382</th>\n",
       "      <td>bank rakyat indonesia (persero) tbk. pt, haurw...</td>\n",
       "      <td>{'entities': []}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297759</th>\n",
       "      <td>mekar jaya griya asri utara vii griya asri uta...</td>\n",
       "      <td>{'entities': [(11, 31, 'POI'), (11, 31, 'STREE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298010</th>\n",
       "      <td>lya &amp; nurul (lyn), m h tham boule, tanah abang</td>\n",
       "      <td>{'entities': [(19, 33, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298366</th>\n",
       "      <td>dip, 12 natasha diponegoro surabaya, rw 1 wono...</td>\n",
       "      <td>{'entities': [(8, 35, 'POI'), (16, 22, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298818</th>\n",
       "      <td>sekolah dasar negeri (sdn) 190, r kemu, ario k...</td>\n",
       "      <td>{'entities': [(32, 38, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299936</th>\n",
       "      <td>pfm bibis karah karah</td>\n",
       "      <td>{'entities': [(0, 15, 'POI'), (10, 15, 'STREET...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>403 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        0  \\\n",
       "983           knalpot putra mahakam, maha 11 65111 klojen   \n",
       "2233                   kintawani car wash, kinta, no c 16   \n",
       "2921                                 angsa, 26 cntrl + p,   \n",
       "3256    dr. gema nazri yanni, m.ked(ped), sp.a bunga l...   \n",
       "4382    bank rakyat indonesia (persero) tbk. pt, haurw...   \n",
       "...                                                   ...   \n",
       "297759  mekar jaya griya asri utara vii griya asri uta...   \n",
       "298010     lya & nurul (lyn), m h tham boule, tanah abang   \n",
       "298366  dip, 12 natasha diponegoro surabaya, rw 1 wono...   \n",
       "298818  sekolah dasar negeri (sdn) 190, r kemu, ario k...   \n",
       "299936                              pfm bibis karah karah   \n",
       "\n",
       "                                                        1  \n",
       "983     {'entities': [(0, 21, 'POI'), (14, 18, 'STREET...  \n",
       "2233     {'entities': [(0, 18, 'POI'), (0, 5, 'STREET')]}  \n",
       "2921                     {'entities': [(0, 5, 'STREET')]}  \n",
       "3256                   {'entities': [(39, 48, 'STREET')]}  \n",
       "4382                                     {'entities': []}  \n",
       "...                                                   ...  \n",
       "297759  {'entities': [(11, 31, 'POI'), (11, 31, 'STREE...  \n",
       "298010                 {'entities': [(19, 33, 'STREET')]}  \n",
       "298366  {'entities': [(8, 35, 'POI'), (16, 22, 'STREET...  \n",
       "298818                 {'entities': [(32, 38, 'STREET')]}  \n",
       "299936  {'entities': [(0, 15, 'POI'), (10, 15, 'STREET...  \n",
       "\n",
       "[403 rows x 2 columns]"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[result[1] != result_final[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "molecular-evaluation",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T17:58:10.818295Z",
     "start_time": "2021-03-18T17:58:10.701181Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>983</th>\n",
       "      <td>knalpot putra mahakam, maha 11 65111 klojen</td>\n",
       "      <td>{'entities': [(0, 21, 'POI'), (23, 27, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2233</th>\n",
       "      <td>kintawani car wash, kinta, no c 16</td>\n",
       "      <td>{'entities': [(0, 18, 'POI'), (20, 25, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2921</th>\n",
       "      <td>angsa, 26 cntrl + p,</td>\n",
       "      <td>{'entities': [(10, 19, 'POI'), (0, 5, 'STREET')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3256</th>\n",
       "      <td>dr. gema nazri yanni, m.ked(ped), sp.a bunga l...</td>\n",
       "      <td>{'entities': [(0, 38, 'POI'), (39, 48, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4382</th>\n",
       "      <td>bank rakyat indonesia (persero) tbk. pt, haurw...</td>\n",
       "      <td>{'entities': [(0, 39, 'POI')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297759</th>\n",
       "      <td>mekar jaya griya asri utara vii griya asri uta...</td>\n",
       "      <td>{'entities': [(11, 31, 'POI'), (32, 52, 'STREE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298010</th>\n",
       "      <td>lya &amp; nurul (lyn), m h tham boule, tanah abang</td>\n",
       "      <td>{'entities': [(0, 17, 'POI'), (19, 33, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298366</th>\n",
       "      <td>dip, 12 natasha diponegoro surabaya, rw 1 wono...</td>\n",
       "      <td>{'entities': [(8, 35, 'POI')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298818</th>\n",
       "      <td>sekolah dasar negeri (sdn) 190, r kemu, ario k...</td>\n",
       "      <td>{'entities': [(0, 30, 'POI'), (32, 38, 'STREET...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299936</th>\n",
       "      <td>pfm bibis karah karah</td>\n",
       "      <td>{'entities': [(0, 15, 'POI'), (16, 21, 'STREET...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>403 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                        0  \\\n",
       "983           knalpot putra mahakam, maha 11 65111 klojen   \n",
       "2233                   kintawani car wash, kinta, no c 16   \n",
       "2921                                 angsa, 26 cntrl + p,   \n",
       "3256    dr. gema nazri yanni, m.ked(ped), sp.a bunga l...   \n",
       "4382    bank rakyat indonesia (persero) tbk. pt, haurw...   \n",
       "...                                                   ...   \n",
       "297759  mekar jaya griya asri utara vii griya asri uta...   \n",
       "298010     lya & nurul (lyn), m h tham boule, tanah abang   \n",
       "298366  dip, 12 natasha diponegoro surabaya, rw 1 wono...   \n",
       "298818  sekolah dasar negeri (sdn) 190, r kemu, ario k...   \n",
       "299936                              pfm bibis karah karah   \n",
       "\n",
       "                                                        1  \n",
       "983     {'entities': [(0, 21, 'POI'), (23, 27, 'STREET...  \n",
       "2233    {'entities': [(0, 18, 'POI'), (20, 25, 'STREET...  \n",
       "2921    {'entities': [(10, 19, 'POI'), (0, 5, 'STREET')]}  \n",
       "3256    {'entities': [(0, 38, 'POI'), (39, 48, 'STREET...  \n",
       "4382                       {'entities': [(0, 39, 'POI')]}  \n",
       "...                                                   ...  \n",
       "297759  {'entities': [(11, 31, 'POI'), (32, 52, 'STREE...  \n",
       "298010  {'entities': [(0, 17, 'POI'), (19, 33, 'STREET...  \n",
       "298366                     {'entities': [(8, 35, 'POI')]}  \n",
       "298818  {'entities': [(0, 30, 'POI'), (32, 38, 'STREET...  \n",
       "299936  {'entities': [(0, 15, 'POI'), (16, 21, 'STREET...  \n",
       "\n",
       "[403 rows x 2 columns]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_final[result[1] != result_final[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "cathedral-gregory",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T18:09:35.538461Z",
     "start_time": "2021-03-18T18:09:35.519512Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('jl. pasar senen dlm. iv no. 37 rt rw 007 04 kelurahan senen kecamatan senen 10410',\n",
       " {'entities': [(4, 15, 'POI'), (0, 15, 'STREET')]})"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_final.iloc[3091][0], result_final.iloc[3091][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "cellular-workplace",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T18:09:37.165399Z",
     "start_time": "2021-03-18T18:09:37.148420Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                                                          3091\n",
       "raw_address    jl. pasar senen dlm. iv no. 37 rt rw 007 04 ke...\n",
       "POI/street                           pasar senen/jl. pasar senen\n",
       "POI                                                  pasar senen\n",
       "street                                           jl. pasar senen\n",
       "Name: 3091, dtype: object"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[3091]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "congressional-affiliate",
   "metadata": {},
   "source": [
    "## Save the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "still-rally",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-15T08:12:14.415455Z",
     "start_time": "2021-03-15T08:12:13.975673Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('pickle/spacy_train.pickle','wb') as file:\n",
    "    pickle.dump(train_data, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "reflected-cover",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-03-18T18:08:11.478675Z",
     "start_time": "2021-03-18T18:08:10.955158Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('pickle/spacy_train_final.pickle','wb') as file:\n",
    "    pickle.dump(train_data_final, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}