notebooks/2a. Create basic network.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "5f034096",
      "metadata": {},
      "source": [
        "# 2a. Create basic network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "071158b6",
      "metadata": {
        "gather": {
          "logged": 1708957796604
        }
      },
      "outputs": [],
      "source": [
        "# Standard library and path imports\n",
        "import set_path\n",
        "import warnings\n",
        "\n",
        "# Third-party library imports\n",
        "import pandas as pd\n",
        "import shapely.ops as so\n",
        "import geopandas as gpd\n",
        "from geopandas import GeoDataFrame\n",
        "from centerline.geometry import Centerline\n",
        "from tqdm.notebook import tqdm_notebook\n",
        "tqdm_notebook.pandas()\n",
        "import folium\n",
        "\n",
        "# Local or project-specific imports\n",
        "import plot_utils\n",
        "import poly_utils\n",
        "import bgt_utils\n",
        "import settings as st\n",
        "\n",
        "if st.my_run == \"azure\":\n",
        "    import config_azure as cf\n",
        "elif st.my_run == \"local\":\n",
        "    import config as cf\n",
        "\n",
        "from shapely.errors import ShapelyDeprecationWarning\n",
        "warnings.filterwarnings(\"ignore\", category=ShapelyDeprecationWarning)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "50c6b9b1",
      "metadata": {},
      "source": [
        "## Import BGT data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0d67ccd2-f06b-408a-bb19-535fba848a6c",
      "metadata": {
        "gather": {
          "logged": 1708957863719
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Get BGT data\n",
        "df_bgt_full = bgt_utils.get_bgt_data_for_bbox(st.bbox, st.bgt_layers)\n",
        "df_bgt_full['naam'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8096e21a-ff68-4c06-aac5-3cf1c854e26d",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Select pilot areas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "32ea3892-d7d9-4bb9-adfb-631e6ddf7331",
      "metadata": {
        "gather": {
          "logged": 1708957864026
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Import areas\n",
        "df_areas = gpd.read_file(cf.output_pilot_area)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "410c68a5-d781-4185-b61b-ca21cf041e81",
      "metadata": {
        "gather": {
          "logged": 1708957864692
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Only keep BGT data within pilot areas\n",
        "df_bgt = df_bgt_full.sjoin(df_areas, how='inner', predicate='within') # note: only sidewalk polygons fully inside area are included\n",
        "df_bgt['naam_left'].value_counts()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "d3171ef4-cc72-4189-9e98-1d9c9f19c523",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Visualize BGT data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c0fa3061-4d1b-407a-a544-e259e3b81d07",
      "metadata": {
        "gather": {
          "logged": 1705498917864
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# set True for satellite background, False for standard background\n",
        "satellite = False\n",
        "\n",
        "# Create Folium map\n",
        "map = folium.Map(\n",
        "    location=[52.350547922223434, 4.794019242371844], tiles=plot_utils.generate_map_params(satellite=satellite),\n",
        "    min_zoom=10, max_zoom=25, zoom_start=17,\n",
        "    zoom_control=True, control_scale=True, control=False\n",
        "    )\n",
        "\n",
        "# Add sidewalks\n",
        "folium.GeoJson(data=df_bgt[df_bgt['naam_left'] == 'voetpad'], style_function=lambda x: {\"fillColor\": \"orange\"}).add_to(map)\n",
        "folium.GeoJson(data=df_bgt[df_bgt['naam_left'] == 'inrit'], style_function=lambda x: {\"fillColor\": \"blue\"}).add_to(map)\n",
        "folium.GeoJson(data=df_bgt[df_bgt['naam_left'] == 'voetgangersgebied'], style_function=lambda x: {\"fillColor\": \"purple\"}).add_to(map)\n",
        "\n",
        "map"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "4b040a19-bee5-434e-9672-2c4b9081692b",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Pre-process BGT data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5c507598-d05e-4a26-920c-05d6dd01f3cd",
      "metadata": {
        "gather": {
          "logged": 1708957885829
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Remove obstacles (small interiors)\n",
        "df_bgt['geometry_no_holes'] = df_bgt.progress_apply(  \n",
        "   lambda row: poly_utils.remove_interiors(row.geometry, st.min_interior_size), axis=1)\n",
        "df_bgt = df_bgt.set_geometry('geometry_no_holes')\n",
        "\n",
        "# Merge sidewalk polygons\n",
        "df_bgt = GeoDataFrame(geometry=gpd.GeoSeries(df_bgt['geometry_no_holes'].unary_union))\n",
        "df_bgt = gpd.GeoDataFrame(df_bgt.geometry.explode()) \n",
        "\n",
        "# Ignore sidewalk polygons that are too small\n",
        "df_bgt['area'] = df_bgt['geometry'].area\n",
        "df_bgt = df_bgt[df_bgt.area > st.min_area_size]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e0e9203a-5397-4e3d-96e2-d62b98ff84f4",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "## Create centerlines"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "adc8a0fd-29da-4db0-b40d-67a035f2e15c",
      "metadata": {
        "gather": {
          "logged": 1708958155979
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Calculate centerlines\n",
        "df_bgt['centerlines'] = df_bgt.progress_apply(\n",
        "   lambda row: Centerline(row.geometry, interpolation_distance=0.5), axis=1)\n",
        "df_bgt = df_bgt.set_geometry('centerlines')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4ca6a2f3-73ce-4b10-b7fb-a32db5f4d773",
      "metadata": {
        "gather": {
          "logged": 1705479417241
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Merge lines\n",
        "df_bgt['centerlines'] = df_bgt['centerlines'].progress_apply(so.linemerge)\n",
        "\n",
        "# Add sidewalk ID\n",
        "df_bgt['sidewalk_id'] = range(0, len(df_bgt))\n",
        "\n",
        "# Get each centerline in separate row\n",
        "df_bgt_exp = df_bgt.explode(ignore_index=True)\n",
        "df_bgt_exp.reset_index(inplace=True, names='cl_id')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6e464e84",
      "metadata": {},
      "source": [
        "## Cut lines that are too long"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8f704b19",
      "metadata": {
        "gather": {
          "logged": 1705480011902
        }
      },
      "outputs": [],
      "source": [
        "# Remove geometry column and add length column\n",
        "df_bgt_cut = df_bgt_exp.drop('geometry', axis=1)\n",
        "df_bgt_cut['length'] = df_bgt_cut['centerlines'].length\n",
        "\n",
        "# Cut linestrings longer than a max length (this takes a while)\n",
        "df_bgt_cut = poly_utils.shorten_linestrings(df_bgt_cut, st.max_ls_length)\n",
        "\n",
        "# Add shortened linestrings to previous dataframe\n",
        "df_bgt_exp = pd.merge(df_bgt_exp.drop(['centerlines'], axis=1), df_bgt_cut.drop(['area', 'sidewalk_id'], axis=1), \n",
        "                                                            how = 'outer', on = 'cl_id')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "10991d1e",
      "metadata": {},
      "source": [
        "## Store"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "228c5bf9-1d9a-4f6f-9b1b-2da65632a71e",
      "metadata": {
        "gather": {
          "logged": 1705480012312
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Finalize dataframe\n",
        "df_bgt_exp = df_bgt_exp.set_crs(st.CRS)\n",
        "\n",
        "df_bgt_final = df_bgt_exp[['cl_id', 'centerlines', 'length', 'sidewalk_id']]  \n",
        "df_bgt_final.head(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a2d416f4",
      "metadata": {
        "gather": {
          "logged": 1705480256935
        }
      },
      "outputs": [],
      "source": [
        "# Write basic network to file\n",
        "df_bgt_final.to_file(cf.output_basic_network, driver='GPKG')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9f7929c8-02ca-427c-9d99-88072698d6df",
      "metadata": {
        "gather": {
          "logged": 1705480262380
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Create dataframe related sidewalks (to be used for widths)\n",
        "df_bgt_sidewalks = df_bgt_exp[['sidewalk_id', 'geometry', 'area']]\n",
        "df_bgt_sidewalks = df_bgt_sidewalks.drop_duplicates().reset_index(drop=True)\n",
        "df_bgt_sidewalks = df_bgt_sidewalks.set_geometry('geometry')\n",
        "df_bgt_sidewalks = df_bgt_sidewalks.set_crs(st.CRS)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e39f8074-85a2-4fc7-bc72-bcb45884a43f",
      "metadata": {
        "gather": {
          "logged": 1705480808213
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Write sidewalks related to basic network to file\n",
        "df_bgt_sidewalks.to_file(cf.output_sidewalks_basic_network, driver='GPKG')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "bff0944b",
      "metadata": {},
      "source": [
        "## Visualize BGT data and basic network"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "df5f63ae-d6a5-4ea0-8b5d-215569553abe",
      "metadata": {
        "gather": {
          "logged": 1705500098963
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# set True for satellite background, False for standard background\n",
        "satellite = False\n",
        "\n",
        "# Create Folium map\n",
        "map = folium.Map(\n",
        "    location=[52.350547922223434, 4.794019242371844], tiles=plot_utils.generate_map_params(satellite=satellite),\n",
        "    min_zoom=10, max_zoom=25, zoom_start=17,\n",
        "    zoom_control=True, control_scale=True, control=False\n",
        "    )\n",
        "\n",
        "# Add areas\n",
        "folium.GeoJson(data=df_areas, style_function=lambda x: {\"fillColor\": \"orange\"}).add_to(map)    \n",
        "\n",
        "# Add sidewalks\n",
        "folium.GeoJson(data=df_bgt_sidewalks, style_function=lambda x: {\"fillColor\": \"grey\"}).add_to(map)\n",
        "\n",
        "# Add basic network\n",
        "folium.GeoJson(data=df_bgt_final, style_function=lambda x: {\"color\": \"black\"}).add_to(map)\n",
        "\n",
        "map"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bb1e9c78-946f-42ea-bffb-b8cc589fb839",
      "metadata": {
        "gather": {
          "logged": 1699615565749
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Add title\n",
        "loc = 'Basic network'\n",
        "title_html = '''\n",
        "             <h3 align=\"center\" style=\"font-size:16px\"><b>{}</b></h3>\n",
        "             '''.format(loc)\n",
        "\n",
        "map.get_root().html.add_child(folium.Element(title_html))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e5329c7f-19c7-4cea-9395-700f99464ada",
      "metadata": {
        "gather": {
          "logged": 1699615686206
        },
        "jupyter": {
          "outputs_hidden": false,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Store map\n",
        "map.save(cf.basic_network_map)"
      ]
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "venv_rp"
    },
    "kernelspec": {
      "display_name": "social_mobility_network",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.0"
    },
    "microsoft": {
      "host": {
        "AzureML": {
          "notebookHasBeenCompleted": true
        }
      },
      "ms_spell_check": {
        "ms_spell_check_language": "en"
      }
    },
    "nteract": {
      "version": "nteract-front-end@1.0.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}