From 4e860eca056595b4fd808ed6bbddd872477f428a Mon Sep 17 00:00:00 2001 From: miyake Date: Thu, 25 Jul 2024 17:39:57 +0900 Subject: [PATCH 01/10] add bash script and sql file to import Overture Maps and OpenStreetMap --- SQL/buildings/create_tables.sql | 166 +++++++++++++++++++++++++++ SQL/buildings/download_import_ovt.sh | 51 ++++++++ SQL/buildings/import_osm_tsv.sh | 28 +++++ SQL/readme.md | 24 ++++ 4 files changed, 269 insertions(+) create mode 100644 SQL/buildings/create_tables.sql create mode 100644 SQL/buildings/download_import_ovt.sh create mode 100644 SQL/buildings/import_osm_tsv.sh create mode 100644 SQL/readme.md diff --git a/SQL/buildings/create_tables.sql b/SQL/buildings/create_tables.sql new file mode 100644 index 0000000..915c382 --- /dev/null +++ b/SQL/buildings/create_tables.sql @@ -0,0 +1,166 @@ +drop table ovt.tokyo_bld; +create table ovt.tokyo_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +drop table ovt.hamamatsu_bld; +create table ovt.hamamatsu_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50)) +; + +drop table ovt.tateyama_bld; +create table ovt.tateyama_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +drop table ovt.kumamoto_bld; +create table ovt.kumamoto_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +drop table ovt.higashi_hiroshima_bld; +create table ovt.higashi_hiroshima_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +drop table ovt.morioka_bld; +create table ovt.morioka_bld ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; diff --git a/SQL/buildings/download_import_ovt.sh b/SQL/buildings/download_import_ovt.sh new file mode 100644 index 0000000..f7e7efd --- /dev/null +++ b/SQL/buildings/download_import_ovt.sh @@ -0,0 +1,51 @@ +# It will takes 25 minutes to process 5 areas +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +mkdir ./data/202405/overturemaps/ + +psql -U postgres -d postgres -c "create schema ovt;" + +psql -U postgres -d postgres -c "drop table if exists ovt.hamamatsu_bld;" +psql -U postgres -d postgres -c "drop table if exists ovt.tokyo_bld;" +psql -U postgres -d postgres -c "drop table if exists ovt.tateyama_bld;" +psql -U postgres -d postgres -c "drop table if exists ovt.kumamoto_bld;" +psql -U postgres -d postgres -c "drop table if exists ovt.higashi_hiroshima_bld;" +psql -U postgres -d postgres -c "drop table if exists ovt.morioka_bld;" + +psql -U postgres -d postgres -f ./create_tables.sql + +# Tokyo +overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=building -o ./data/202405/overturemaps/tokyo.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.tokyo_bld -nlt multipolygon ./data/202405/overturemaps/tokyo.geojson + +# Hamamatsu +overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=building -o ./data/202405/overturemaps/hamamatsu.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.hamamatsu_bld -nlt multipolygon ./data/202405/overturemaps/hamamatsu.geojson + +# Tateyama +overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=building -o ./data/202405/overturemaps/tateyama.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.tateyama_bld -nlt multipolygon ./data/202405/overturemaps/tateyama.geojson + +# Kumamoto +overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=building -o ./data/202405/overturemaps/kumamoto.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.kumamoto_bld -nlt multipolygon ./data/202405/overturemaps/kumamoto.geojson + +# Higashi_hiroshima +overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=building -o ./data/202405/overturemaps/higashi_hiroshima.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.higashi_hiroshima_bld -nlt multipolygon ./data/202405/overturemaps/higashi_hiroshima.geojson + +# Morioka +overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=building -o ./data/202405/overturemaps/morioka.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.morioka_bld -nlt multipolygon ./data/202405/overturemaps/morioka.geojson + + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" diff --git a/SQL/buildings/import_osm_tsv.sh b/SQL/buildings/import_osm_tsv.sh new file mode 100644 index 0000000..94aa6a5 --- /dev/null +++ b/SQL/buildings/import_osm_tsv.sh @@ -0,0 +1,28 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +psql -U postgres -d postgres -c "create schema osm;" + +psql -U postgres -d postgres -c "drop table if exists osm.tokyo_bld;" +psql -U postgres -d postgres -c "drop table if exists osm.tateyama_bld;" +psql -U postgres -d postgres -c "drop table if exists osm.hamamatsu_bld;" +psql -U postgres -d postgres -c "drop table if exists osm.kumamoto_bld;" +psql -U postgres -d postgres -c "drop table if exists osm.higashi_hiroshima_bld;" +psql -U postgres -d postgres -c "drop table if exists osm.morioka_bld;" + +psql -U postgres -d postgres -c "create table osm.tokyo_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" +psql -U postgres -d postgres -c "create table osm.tateyama_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" +psql -U postgres -d postgres -c "create table osm.hamamatsu_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" +psql -U postgres -d postgres -c "create table osm.kumamoto_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" +psql -U postgres -d postgres -c "create table osm.higashi_hiroshima_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" +psql -U postgres -d postgres -c "create table osm.morioka_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" + +psql -U postgres -d postgres -c "\copy osm.tokyo_bld from './data/202405/osm/osm_building_tokyo.tsv' delimiter E'\t' csv;" +psql -U postgres -d postgres -c "\copy osm.tateyama_bld from './data/202405/osm/osm_building_tateyama.tsv' delimiter E'\t' csv;" +psql -U postgres -d postgres -c "\copy osm.hamamatsu_bld from './data/202405/osm/osm_building_hamamatsu.tsv' delimiter E'\t' csv;" +psql -U postgres -d postgres -c "\copy osm.kumamoto_bld from './data/202405/osm/osm_building_kumamoto.tsv' delimiter E'\t' csv;" +psql -U postgres -d postgres -c "\copy osm.higashi_hiroshima_bld from './data/202405/osm/osm_building_higashi_hiroshima.tsv' delimiter E'\t' csv;" +psql -U postgres -d postgres -c "\copy osm.morioka_bld from './data/202405/osm/osm_building_morioka.tsv' delimiter E'\t' csv;" + diff --git a/SQL/readme.md b/SQL/readme.md new file mode 100644 index 0000000..311091b --- /dev/null +++ b/SQL/readme.md @@ -0,0 +1,24 @@ +# Buildings + +## Overture Maps + +Install a CLI tool to download Overture Maps data. + +``` +pip install overturemaps +``` + +Run the following code and it will download buildings in target areas from Overture Maps and import them into PostgreSQL + +``` +bash buildings/download_import_ovt.sql +``` + +## OpenStreetMap + +This script just import tsv files into PostgreSQL. +You need to extract data before running this script. + +``` +bash building/import_osm_tsv.shcd . +``` \ No newline at end of file From 6210313d50595b21d6e89baa385da649198402ad Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 10:39:30 +0900 Subject: [PATCH 02/10] add settings.json --- .vscode/settings.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..421ed4c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,19 @@ +{ + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "python.testing.unittestArgs": ["-v", "-s", ".", "-p", "test_*.py"], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true, + "black-formatter.args": ["--line-length", "120"], + "flake8.args": ["--max-line-length", "120"], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "[js]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[sql]": { + "editor.defaultFormatter": "inferrinizzard.prettier-sql-vscode" + } +} From 6fd389cf3f2b9dd84550058e9936d00975f17ede Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 10:40:09 +0900 Subject: [PATCH 03/10] change dir name from SQL to Bash, and change Abbreviation from ovt to omf --- {SQL => Bash}/buildings/create_tables.sql | 24 ++++++++--------- .../buildings/download_import_omf.sh | 27 ++++++++++--------- {SQL => Bash}/buildings/import_osm_tsv.sh | 0 {SQL => Bash}/readme.md | 0 4 files changed, 26 insertions(+), 25 deletions(-) rename {SQL => Bash}/buildings/create_tables.sql (92%) rename SQL/buildings/download_import_ovt.sh => Bash/buildings/download_import_omf.sh (71%) rename {SQL => Bash}/buildings/import_osm_tsv.sh (100%) rename {SQL => Bash}/readme.md (100%) diff --git a/SQL/buildings/create_tables.sql b/Bash/buildings/create_tables.sql similarity index 92% rename from SQL/buildings/create_tables.sql rename to Bash/buildings/create_tables.sql index 915c382..e38470f 100644 --- a/SQL/buildings/create_tables.sql +++ b/Bash/buildings/create_tables.sql @@ -1,5 +1,5 @@ -drop table ovt.tokyo_bld; -create table ovt.tokyo_bld ( +drop table omf.tokyo_bld; +create table omf.tokyo_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, @@ -26,8 +26,8 @@ create table ovt.tokyo_bld ( ) ; -drop table ovt.hamamatsu_bld; -create table ovt.hamamatsu_bld ( +drop table omf.hamamatsu_bld; +create table omf.hamamatsu_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, @@ -53,8 +53,8 @@ create table ovt.hamamatsu_bld ( area_name char(50)) ; -drop table ovt.tateyama_bld; -create table ovt.tateyama_bld ( +drop table omf.tateyama_bld; +create table omf.tateyama_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, @@ -81,8 +81,8 @@ create table ovt.tateyama_bld ( ) ; -drop table ovt.kumamoto_bld; -create table ovt.kumamoto_bld ( +drop table omf.kumamoto_bld; +create table omf.kumamoto_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, @@ -109,8 +109,8 @@ create table ovt.kumamoto_bld ( ) ; -drop table ovt.higashi_hiroshima_bld; -create table ovt.higashi_hiroshima_bld ( +drop table omf.higashi_hiroshima_bld; +create table omf.higashi_hiroshima_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, @@ -137,8 +137,8 @@ create table ovt.higashi_hiroshima_bld ( ) ; -drop table ovt.morioka_bld; -create table ovt.morioka_bld ( +drop table omf.morioka_bld; +create table omf.morioka_bld ( ogc_fid integer, wkb_geometry geometry(MULTIPOLYGON, 4326), id varchar, diff --git a/SQL/buildings/download_import_ovt.sh b/Bash/buildings/download_import_omf.sh similarity index 71% rename from SQL/buildings/download_import_ovt.sh rename to Bash/buildings/download_import_omf.sh index f7e7efd..f10c0b7 100644 --- a/SQL/buildings/download_import_ovt.sh +++ b/Bash/buildings/download_import_omf.sh @@ -1,3 +1,4 @@ +#!/bin/bash # It will takes 25 minutes to process 5 areas # Initialize the command in seconds. SECONDS=0 @@ -8,40 +9,40 @@ cd "${SCRIPT_DIR}" mkdir ./data/202405/overturemaps/ -psql -U postgres -d postgres -c "create schema ovt;" +psql -U postgres -d postgres -c "create schema omf;" -psql -U postgres -d postgres -c "drop table if exists ovt.hamamatsu_bld;" -psql -U postgres -d postgres -c "drop table if exists ovt.tokyo_bld;" -psql -U postgres -d postgres -c "drop table if exists ovt.tateyama_bld;" -psql -U postgres -d postgres -c "drop table if exists ovt.kumamoto_bld;" -psql -U postgres -d postgres -c "drop table if exists ovt.higashi_hiroshima_bld;" -psql -U postgres -d postgres -c "drop table if exists ovt.morioka_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.hamamatsu_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.tokyo_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.tateyama_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.kumamoto_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.higashi_hiroshima_bld;" +psql -U postgres -d postgres -c "drop table if exists omf.morioka_bld;" psql -U postgres -d postgres -f ./create_tables.sql # Tokyo overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=building -o ./data/202405/overturemaps/tokyo.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.tokyo_bld -nlt multipolygon ./data/202405/overturemaps/tokyo.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_bld -nlt multipolygon ./data/202405/overturemaps/tokyo.geojson # Hamamatsu overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=building -o ./data/202405/overturemaps/hamamatsu.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.hamamatsu_bld -nlt multipolygon ./data/202405/overturemaps/hamamatsu.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.hamamatsu_bld -nlt multipolygon ./data/202405/overturemaps/hamamatsu.geojson # Tateyama overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=building -o ./data/202405/overturemaps/tateyama.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.tateyama_bld -nlt multipolygon ./data/202405/overturemaps/tateyama.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tateyama_bld -nlt multipolygon ./data/202405/overturemaps/tateyama.geojson # Kumamoto overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=building -o ./data/202405/overturemaps/kumamoto.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.kumamoto_bld -nlt multipolygon ./data/202405/overturemaps/kumamoto.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.kumamoto_bld -nlt multipolygon ./data/202405/overturemaps/kumamoto.geojson # Higashi_hiroshima overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=building -o ./data/202405/overturemaps/higashi_hiroshima.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.higashi_hiroshima_bld -nlt multipolygon ./data/202405/overturemaps/higashi_hiroshima.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.higashi_hiroshima_bld -nlt multipolygon ./data/202405/overturemaps/higashi_hiroshima.geojson # Morioka overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=building -o ./data/202405/overturemaps/morioka.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln ovt.morioka_bld -nlt multipolygon ./data/202405/overturemaps/morioka.geojson +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_bld -nlt multipolygon ./data/202405/overturemaps/morioka.geojson # Display the measurement time. diff --git a/SQL/buildings/import_osm_tsv.sh b/Bash/buildings/import_osm_tsv.sh similarity index 100% rename from SQL/buildings/import_osm_tsv.sh rename to Bash/buildings/import_osm_tsv.sh diff --git a/SQL/readme.md b/Bash/readme.md similarity index 100% rename from SQL/readme.md rename to Bash/readme.md From 57bd04b0a798da71ededc97bd29475e5bb6eab60 Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 14:30:49 +0900 Subject: [PATCH 04/10] fix dowload and import script --- .gitignore | 6 +- Bash/OMF/ETL/create_tables_building.sql | 54 ++++++++ Bash/OMF/ETL/create_tables_place.sql | 46 +++++++ Bash/OMF/ETL/omf_download.sh | 57 ++++++++ Bash/OMF/ETL/omf_import.sh | 55 ++++++++ Bash/OMF/ETL/requirements.txt | 1 + Bash/OMF/ETL/setup.sh | 19 +++ Bash/buildings/create_tables.sql | 166 ------------------------ Bash/buildings/download_import_omf.sh | 52 -------- Bash/buildings/import_osm_tsv.sh | 28 ---- 10 files changed, 237 insertions(+), 247 deletions(-) create mode 100644 Bash/OMF/ETL/create_tables_building.sql create mode 100644 Bash/OMF/ETL/create_tables_place.sql create mode 100644 Bash/OMF/ETL/omf_download.sh create mode 100644 Bash/OMF/ETL/omf_import.sh create mode 100644 Bash/OMF/ETL/requirements.txt create mode 100644 Bash/OMF/ETL/setup.sh delete mode 100644 Bash/buildings/create_tables.sql delete mode 100644 Bash/buildings/download_import_omf.sh delete mode 100644 Bash/buildings/import_osm_tsv.sh diff --git a/.gitignore b/.gitignore index 14a6b30..cc0394e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,8 @@ __pycache__/ cache/ .streamlit/ tiles -other.py \ No newline at end of file +other.py + +# ignore data dir +Bash/**/data/** +Bash/**/overturemaps-py/** \ No newline at end of file diff --git a/Bash/OMF/ETL/create_tables_building.sql b/Bash/OMF/ETL/create_tables_building.sql new file mode 100644 index 0000000..9de1e6b --- /dev/null +++ b/Bash/OMF/ETL/create_tables_building.sql @@ -0,0 +1,54 @@ +-- drop table +drop table if exists omf.tokyo_building; +drop table if exists omf.hamamatsu_building; +drop table if exists omf.tateyama_building; +drop table if exists omf.kumamoto_building; +drop table if exists omf.higashi_hiroshima_building; +drop table if exists omf.morioka_building; +drop table if exists omf.building; + +-- create table +create table omf.building ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +create table + omf.tokyo_building (like omf.building including all); + +create table + omf.hamamatsu_building (like omf.building including all); + +create table + omf.tateyama_building (like omf.building including all); + +create table + omf.kumamoto_building (like omf.building including all); + +create table + omf.higashi_hiroshima_building (like omf.building including all); + +create table + omf.morioka_building (like omf.building including all); diff --git a/Bash/OMF/ETL/create_tables_place.sql b/Bash/OMF/ETL/create_tables_place.sql new file mode 100644 index 0000000..8a96e79 --- /dev/null +++ b/Bash/OMF/ETL/create_tables_place.sql @@ -0,0 +1,46 @@ +-- drop table +drop table if exists omf.tokyo_place; +drop table if exists omf.hamamatsu_place; +drop table if exists omf.tateyama_place; +drop table if exists omf.kumamoto_place; +drop table if exists omf.higashi_hiroshima_place; +drop table if exists omf.morioka_place; +drop table if exists omf.place; + +-- create table +create table + omf.place ( + ogc_fid serial primary key, + wkb_geometry geometry (point, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources jsonb, + names jsonb, + categories jsonb, + confidence double precision, + websites text, + emails text, + socials text, + phones text, + addresses jsonb, + brand jsonb + ); + +create table + omf.tokyo_place (like omf.place including all); + +create table + omf.hamamatsu_place (like omf.place including all); + +create table + omf.tateyama_place (like omf.place including all); + +create table + omf.kumamoto_place (like omf.place including all); + +create table + omf.higashi_hiroshima_place (like omf.place including all); + +create table + omf.morioka_place (like omf.place including all); \ No newline at end of file diff --git a/Bash/OMF/ETL/omf_download.sh b/Bash/OMF/ETL/omf_download.sh new file mode 100644 index 0000000..ed302cc --- /dev/null +++ b/Bash/OMF/ETL/omf_download.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, place]" + exit 1 +fi +if [ -z "$2" ]; then + echo -e "Write Release Version in second argument.\nYou could check list of versions:" + aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + exit 1 +fi +TYPE=$1 # e.g. [building, place] +RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1] + +# create data directory +DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" +mkdir -p $DATA_DIR + +# change resource in "git@github.com:OvertureMaps/overturemaps-py.git" to change +cd "${SCRIPT_DIR}/overturemaps-py" +before="overturemaps-us-west-2/release/.*/theme" +after="overturemaps-us-west-2/release/$RELEASE_VERSION/theme" +sed -i '' "s|$before|$after|g" "${SCRIPT_DIR}/overturemaps-py/overturemaps/core.py" + +# Tokyo +poetry run overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=$TYPE -o "$DATA_DIR/tokyo_$TYPE.geojson" + +# Hamamatsu +poetry run overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=$TYPE -o "$DATA_DIR/hamamatsu_$TYPE.geojson" + +# Tateyama +poetry run overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=$TYPE -o "$DATA_DIR/tateyama_$TYPE.geojson" + +# Kumamoto +poetry run overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=$TYPE -o "$DATA_DIR/kumamoto_$TYPE.geojson" + +# Higashi_hiroshima +poetry run overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=$TYPE -o "$DATA_DIR/higashi_hiroshima_$TYPE.geojson" + +# Morioka +poetry run overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=$TYPE -o "$DATA_DIR/morioka_$TYPE.geojson" + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/omf_import.sh b/Bash/OMF/ETL/omf_import.sh new file mode 100644 index 0000000..474c10a --- /dev/null +++ b/Bash/OMF/ETL/omf_import.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, place]" + exit 1 +fi +if [ -z "$2" ]; then + echo -e "Write Release Version in second argument.\nYou could check list of versions:" + aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + exit 1 +fi +TYPE=$1 # e.g. [building, place] +RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1] + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" + +# create schema +psql -U postgres -d postgres -c "create schema omf;" + +# create table +psql -U postgres -d postgres -f ./create_tables_$TYPE.sql + +# Tokyo +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_$TYPE -nlt multipolygon $DATA_DIR/tokyo_$TYPE.geojson + +# Hamamatsu +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.hamamatsu_$TYPE -nlt multipolygon $DATA_DIR/hamamatsu_$TYPE.geojson + +# Tateyama +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tateyama_$TYPE -nlt multipolygon $DATA_DIR/tateyama_$TYPE.geojson + +# Kumamoto +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.kumamoto_$TYPE -nlt multipolygon $DATA_DIR/kumamoto_$TYPE.geojson + +# Higashi_hiroshima +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.higashi_hiroshima_$TYPE -nlt multipolygon $DATA_DIR/higashi_hiroshima_$TYPE.geojson + +# Morioka +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_$TYPE -nlt multipolygon $DATA_DIR/morioka_$TYPE.geojson + + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" diff --git a/Bash/OMF/ETL/requirements.txt b/Bash/OMF/ETL/requirements.txt new file mode 100644 index 0000000..2bd745f --- /dev/null +++ b/Bash/OMF/ETL/requirements.txt @@ -0,0 +1 @@ +poetry \ No newline at end of file diff --git a/Bash/OMF/ETL/setup.sh b/Bash/OMF/ETL/setup.sh new file mode 100644 index 0000000..8570bb2 --- /dev/null +++ b/Bash/OMF/ETL/setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# pip install +pip install -r "$SCRIPT_DIR/requirements.txt" + +# get latest resource +OMF_DIR="$SCRIPT_DIR/overturemaps-py" +if [ -d "$OMF_DIR" ]; then + cd "$OMF_DIR" + git checkout . + git pull origin main + cd - +else + git clone git@github.com:OvertureMaps/overturemaps-py.git +fi \ No newline at end of file diff --git a/Bash/buildings/create_tables.sql b/Bash/buildings/create_tables.sql deleted file mode 100644 index e38470f..0000000 --- a/Bash/buildings/create_tables.sql +++ /dev/null @@ -1,166 +0,0 @@ -drop table omf.tokyo_bld; -create table omf.tokyo_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50) -) -; - -drop table omf.hamamatsu_bld; -create table omf.hamamatsu_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50)) -; - -drop table omf.tateyama_bld; -create table omf.tateyama_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50) -) -; - -drop table omf.kumamoto_bld; -create table omf.kumamoto_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50) -) -; - -drop table omf.higashi_hiroshima_bld; -create table omf.higashi_hiroshima_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50) -) -; - -drop table omf.morioka_bld; -create table omf.morioka_bld ( - ogc_fid integer, - wkb_geometry geometry(MULTIPOLYGON, 4326), - id varchar, - version integer, - update_time timestamp with time zone, - sources json, - subtype varchar, - names json, - class varchar, - level integer, - has_parts boolean, - height double precision, - min_height double precision, - num_floors integer, - facade_color varchar, - facade_material varchar, - min_floor integer, - roof_material varchar, - roof_shape varchar, - roof_direction double precision, - roof_orientation varchar, - roof_color varchar, - area_name char(50) -) -; diff --git a/Bash/buildings/download_import_omf.sh b/Bash/buildings/download_import_omf.sh deleted file mode 100644 index f10c0b7..0000000 --- a/Bash/buildings/download_import_omf.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# It will takes 25 minutes to process 5 areas -# Initialize the command in seconds. -SECONDS=0 - -# move to working directory -SCRIPT_DIR=$(cd $(dirname $0); pwd) -cd "${SCRIPT_DIR}" - -mkdir ./data/202405/overturemaps/ - -psql -U postgres -d postgres -c "create schema omf;" - -psql -U postgres -d postgres -c "drop table if exists omf.hamamatsu_bld;" -psql -U postgres -d postgres -c "drop table if exists omf.tokyo_bld;" -psql -U postgres -d postgres -c "drop table if exists omf.tateyama_bld;" -psql -U postgres -d postgres -c "drop table if exists omf.kumamoto_bld;" -psql -U postgres -d postgres -c "drop table if exists omf.higashi_hiroshima_bld;" -psql -U postgres -d postgres -c "drop table if exists omf.morioka_bld;" - -psql -U postgres -d postgres -f ./create_tables.sql - -# Tokyo -overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=building -o ./data/202405/overturemaps/tokyo.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_bld -nlt multipolygon ./data/202405/overturemaps/tokyo.geojson - -# Hamamatsu -overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=building -o ./data/202405/overturemaps/hamamatsu.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.hamamatsu_bld -nlt multipolygon ./data/202405/overturemaps/hamamatsu.geojson - -# Tateyama -overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=building -o ./data/202405/overturemaps/tateyama.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tateyama_bld -nlt multipolygon ./data/202405/overturemaps/tateyama.geojson - -# Kumamoto -overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=building -o ./data/202405/overturemaps/kumamoto.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.kumamoto_bld -nlt multipolygon ./data/202405/overturemaps/kumamoto.geojson - -# Higashi_hiroshima -overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=building -o ./data/202405/overturemaps/higashi_hiroshima.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.higashi_hiroshima_bld -nlt multipolygon ./data/202405/overturemaps/higashi_hiroshima.geojson - -# Morioka -overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=building -o ./data/202405/overturemaps/morioka.geojson -ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_bld -nlt multipolygon ./data/202405/overturemaps/morioka.geojson - - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" diff --git a/Bash/buildings/import_osm_tsv.sh b/Bash/buildings/import_osm_tsv.sh deleted file mode 100644 index 94aa6a5..0000000 --- a/Bash/buildings/import_osm_tsv.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -psql -U postgres -d postgres -c "create schema osm;" - -psql -U postgres -d postgres -c "drop table if exists osm.tokyo_bld;" -psql -U postgres -d postgres -c "drop table if exists osm.tateyama_bld;" -psql -U postgres -d postgres -c "drop table if exists osm.hamamatsu_bld;" -psql -U postgres -d postgres -c "drop table if exists osm.kumamoto_bld;" -psql -U postgres -d postgres -c "drop table if exists osm.higashi_hiroshima_bld;" -psql -U postgres -d postgres -c "drop table if exists osm.morioka_bld;" - -psql -U postgres -d postgres -c "create table osm.tokyo_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" -psql -U postgres -d postgres -c "create table osm.tateyama_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" -psql -U postgres -d postgres -c "create table osm.hamamatsu_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" -psql -U postgres -d postgres -c "create table osm.kumamoto_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" -psql -U postgres -d postgres -c "create table osm.higashi_hiroshima_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" -psql -U postgres -d postgres -c "create table osm.morioka_bld(area_id bigint, id bigint, geom geometry(MultiPolygon,4326));" - -psql -U postgres -d postgres -c "\copy osm.tokyo_bld from './data/202405/osm/osm_building_tokyo.tsv' delimiter E'\t' csv;" -psql -U postgres -d postgres -c "\copy osm.tateyama_bld from './data/202405/osm/osm_building_tateyama.tsv' delimiter E'\t' csv;" -psql -U postgres -d postgres -c "\copy osm.hamamatsu_bld from './data/202405/osm/osm_building_hamamatsu.tsv' delimiter E'\t' csv;" -psql -U postgres -d postgres -c "\copy osm.kumamoto_bld from './data/202405/osm/osm_building_kumamoto.tsv' delimiter E'\t' csv;" -psql -U postgres -d postgres -c "\copy osm.higashi_hiroshima_bld from './data/202405/osm/osm_building_higashi_hiroshima.tsv' delimiter E'\t' csv;" -psql -U postgres -d postgres -c "\copy osm.morioka_bld from './data/202405/osm/osm_building_morioka.tsv' delimiter E'\t' csv;" - From aae06b35564d9162258b9d15301f124e99de6475 Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 14:45:39 +0900 Subject: [PATCH 05/10] add analyzing codes --- .gitignore | 1 + .vscode/settings.json | 2 +- Bash/OMF/Analyze/copy_category_main_to_tsv.sh | 38 +++++++++ Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh | 79 +++++++++++++++++++ Bash/OMF/Analyze/copy_place_to_tsv.sh | 45 +++++++++++ .../Analyze/copy_source_in_place_to_tsv.sh | 46 +++++++++++ 6 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 Bash/OMF/Analyze/copy_category_main_to_tsv.sh create mode 100644 Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh create mode 100644 Bash/OMF/Analyze/copy_place_to_tsv.sh create mode 100644 Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh diff --git a/.gitignore b/.gitignore index cc0394e..17d56ca 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ other.py # ignore data dir Bash/**/data/** +Bash/**/tsv/** Bash/**/overturemaps-py/** \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 421ed4c..37bad0c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "editor.formatOnSave": true, + "editor.formatOnSave": false, "editor.tabSize": 2, "editor.defaultFormatter": "esbenp.prettier-vscode", "python.testing.unittestArgs": ["-v", "-s", ".", "-p", "test_*.py"], diff --git a/Bash/OMF/Analyze/copy_category_main_to_tsv.sh b/Bash/OMF/Analyze/copy_category_main_to_tsv.sh new file mode 100644 index 0000000..16f1857 --- /dev/null +++ b/Bash/OMF/Analyze/copy_category_main_to_tsv.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "select + count(categories->>'main') as category_cnt + from omf.${city}_place + where categories->>'main' is not null" + psql -d postgres -c "\copy ( + select + distinct categories->>'main' as main_category, + count(*) as category_cnt + from omf.${city}_place + where categories->>'main' is not null + group by categories->>'main' + order by category_cnt desc + ) to './tsv/omf_place_category_main_$city.tsv' + with csv delimiter E'\t';" +done + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" diff --git a/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh new file mode 100644 index 0000000..bbb8a02 --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +output="./tsv/omf_place_cnt.tsv" +rm $output + +# export header +psql -d postgres -c "\copy ( + select + '$city' as city, + count(ogc_fid) as ogc_fid_cnt, + count(wkb_geometry) as wkb_geometry_cnt, + count(id) as id_cnt, + count(version) as version_cnt, + count(update_time) as update_time_cnt, + count(sources) as sources_cnt, + count(names) as names_cnt, + count(categories) as categories_cnt, + count(confidence) as confidence_cnt, + count(websites) as websites_cnt, + count(emails) as emails_cnt, + count(socials) as socials_cnt, + count(phones) as phones_cnt, + count(addresses) as addresses_cnt, + count(brand) as brand_cnt + from + omf.${city}_place + where id is null + ) to 'omf.tmp' + with csv header delimiter E'\t';" +cat omf.tmp >> "$output" + +# export data +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + '$city' as city, + count(ogc_fid) as ogc_fid_cnt, + count(wkb_geometry) as wkb_geometry_cnt, + count(id) as id_cnt, + count(version) as version_cnt, + count(update_time) as update_time_cnt, + count(sources) as sources_cnt, + count(names) as names_cnt, + count(categories) as categories_cnt, + count(confidence) as confidence_cnt, + count(websites) as websites_cnt, + count(emails) as emails_cnt, + count(socials) as socials_cnt, + count(phones) as phones_cnt, + count(addresses) as addresses_cnt, + count(brand) as brand_cnt + from + omf.${city}_place + ) to 'omf.tmp' + with csv delimiter E'\t';" + cat omf.tmp >> "$output" +done + +# rm tmp +rm omf.tmp + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_to_tsv.sh b/Bash/OMF/Analyze/copy_place_to_tsv.sh new file mode 100644 index 0000000..d3065d5 --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_to_tsv.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + ogc_fid, + ST_AsText(wkb_geometry) as wkb_geometry, + id, + version, + update_time, + sources::text, + names::text, + categories::text, + confidence, + websites, + emails, + socials, + phones, + addresses::text, + brand::text + from + omf.${city}_place + ) to './tsv/omf_place_$city.tsv' + with csv delimiter E'\t';" +done + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" diff --git a/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh b/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh new file mode 100644 index 0000000..0cea1c2 --- /dev/null +++ b/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +output="./tsv/omf_source_in_place.tsv" +rm $output + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + dataset, cnt + from ( + select 'total' as dataset, count(*) as cnt from omf.${city}_place + union + select distinct source->>'dataset' as dataset, count(*) as cnt + from omf.${city}_place, + jsonb_array_elements(sources) as source + where source->>'dataset' is not null + group by source->>'dataset' + ) a + order by dataset desc + ) to 'omf_source_in_place.tmp' + with csv delimiter E'\t';" + echo "city $city" >> "$output" + cat omf_source_in_place.tmp >> "$output" +done + +# rm tmp +rm omf_source_in_place.tmp + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" From e0cbfd777b8684218d001174ec4a59c9909ae361 Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 15:28:45 +0900 Subject: [PATCH 06/10] fix omf codes --- Bash/OMF/Analyze/copy_category_main_to_tsv.sh | 2 ++ Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh | 4 ++- Bash/OMF/Analyze/copy_place_to_tsv.sh | 2 ++ .../Analyze/copy_source_in_place_to_tsv.sh | 2 ++ Bash/OMF/ETL/omf_download.sh | 30 ++++++++++++++----- Bash/OMF/ETL/omf_import.sh | 10 +++++++ Bash/OMF/ETL/requirements.txt | 1 - Bash/OMF/requirements.txt | 2 ++ Bash/OMF/{ETL => }/setup.sh | 4 ++- Bash/readme.md | 28 +++++++++++------ 10 files changed, 66 insertions(+), 19 deletions(-) delete mode 100644 Bash/OMF/ETL/requirements.txt create mode 100644 Bash/OMF/requirements.txt rename Bash/OMF/{ETL => }/setup.sh (97%) diff --git a/Bash/OMF/Analyze/copy_category_main_to_tsv.sh b/Bash/OMF/Analyze/copy_category_main_to_tsv.sh index 16f1857..52991d4 100644 --- a/Bash/OMF/Analyze/copy_category_main_to_tsv.sh +++ b/Bash/OMF/Analyze/copy_category_main_to_tsv.sh @@ -36,3 +36,5 @@ time=$SECONDS ((sec=time%60, min=(time%3600)/60, hrs=time/3600)) timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh index bbb8a02..0fc1515 100644 --- a/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh +++ b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh @@ -76,4 +76,6 @@ rm omf.tmp time=$SECONDS ((sec=time%60, min=(time%3600)/60, hrs=time/3600)) timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" \ No newline at end of file +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_to_tsv.sh b/Bash/OMF/Analyze/copy_place_to_tsv.sh index d3065d5..6c7813b 100644 --- a/Bash/OMF/Analyze/copy_place_to_tsv.sh +++ b/Bash/OMF/Analyze/copy_place_to_tsv.sh @@ -43,3 +43,5 @@ time=$SECONDS ((sec=time%60, min=(time%3600)/60, hrs=time/3600)) timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh b/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh index 0cea1c2..87072ec 100644 --- a/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh +++ b/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh @@ -44,3 +44,5 @@ time=$SECONDS ((sec=time%60, min=(time%3600)/60, hrs=time/3600)) timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/omf_download.sh b/Bash/OMF/ETL/omf_download.sh index ed302cc..9f2ca74 100644 --- a/Bash/OMF/ETL/omf_download.sh +++ b/Bash/OMF/ETL/omf_download.sh @@ -1,5 +1,7 @@ #!/bin/bash +### Preprocessing ### + # Initialize the command in seconds. SECONDS=0 @@ -18,7 +20,7 @@ if [ -z "$2" ]; then exit 1 fi TYPE=$1 # e.g. [building, place] -RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1] +RELEASE_VERSION=$2 # e.g. [latest, 2024-07-22.0, 2024-06-13-beta.1] # create data directory DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" @@ -26,27 +28,41 @@ mkdir -p $DATA_DIR # change resource in "git@github.com:OvertureMaps/overturemaps-py.git" to change cd "${SCRIPT_DIR}/overturemaps-py" +git checkout . +git pull origin main before="overturemaps-us-west-2/release/.*/theme" after="overturemaps-us-west-2/release/$RELEASE_VERSION/theme" sed -i '' "s|$before|$after|g" "${SCRIPT_DIR}/overturemaps-py/overturemaps/core.py" +function omf_download(){ + if [ "$RELEASE_VERSION" = "latest" ]; then + overturemaps download $@ + else + poetry run overturemaps download $@ + fi +} + +### Main ### + # Tokyo -poetry run overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=$TYPE -o "$DATA_DIR/tokyo_$TYPE.geojson" +omf_download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=$TYPE -o "$DATA_DIR/tokyo_$TYPE.geojson" # Hamamatsu -poetry run overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=$TYPE -o "$DATA_DIR/hamamatsu_$TYPE.geojson" +omf_download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=$TYPE -o "$DATA_DIR/hamamatsu_$TYPE.geojson" # Tateyama -poetry run overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=$TYPE -o "$DATA_DIR/tateyama_$TYPE.geojson" +omf_download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=$TYPE -o "$DATA_DIR/tateyama_$TYPE.geojson" # Kumamoto -poetry run overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=$TYPE -o "$DATA_DIR/kumamoto_$TYPE.geojson" +omf_download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=$TYPE -o "$DATA_DIR/kumamoto_$TYPE.geojson" # Higashi_hiroshima -poetry run overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=$TYPE -o "$DATA_DIR/higashi_hiroshima_$TYPE.geojson" +omf_download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=$TYPE -o "$DATA_DIR/higashi_hiroshima_$TYPE.geojson" # Morioka -poetry run overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=$TYPE -o "$DATA_DIR/morioka_$TYPE.geojson" +omf_download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=$TYPE -o "$DATA_DIR/morioka_$TYPE.geojson" + +### Postprocessing ### # Display the measurement time. time=$SECONDS diff --git a/Bash/OMF/ETL/omf_import.sh b/Bash/OMF/ETL/omf_import.sh index 474c10a..67342a4 100644 --- a/Bash/OMF/ETL/omf_import.sh +++ b/Bash/OMF/ETL/omf_import.sh @@ -1,5 +1,7 @@ #!/bin/bash +### Preprocessing ### + # Initialize the command in seconds. SECONDS=0 @@ -23,6 +25,10 @@ RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1] # define data dir DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" +### Main ### + +echo "Change psql user(-U) or database(-d) if threre is error about postgres" + # create schema psql -U postgres -d postgres -c "create schema omf;" @@ -48,8 +54,12 @@ ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln o ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_$TYPE -nlt multipolygon $DATA_DIR/morioka_$TYPE.geojson +### Postprocessing ### + # Display the measurement time. time=$SECONDS ((sec=time%60, min=(time%3600)/60, hrs=time/3600)) timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/requirements.txt b/Bash/OMF/ETL/requirements.txt deleted file mode 100644 index 2bd745f..0000000 --- a/Bash/OMF/ETL/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -poetry \ No newline at end of file diff --git a/Bash/OMF/requirements.txt b/Bash/OMF/requirements.txt new file mode 100644 index 0000000..81701b4 --- /dev/null +++ b/Bash/OMF/requirements.txt @@ -0,0 +1,2 @@ +poetry +overturemaps \ No newline at end of file diff --git a/Bash/OMF/ETL/setup.sh b/Bash/OMF/setup.sh similarity index 97% rename from Bash/OMF/ETL/setup.sh rename to Bash/OMF/setup.sh index 8570bb2..1cdfbb4 100644 --- a/Bash/OMF/ETL/setup.sh +++ b/Bash/OMF/setup.sh @@ -16,4 +16,6 @@ if [ -d "$OMF_DIR" ]; then cd - else git clone git@github.com:OvertureMaps/overturemaps-py.git -fi \ No newline at end of file +fi + +exit 0 \ No newline at end of file diff --git a/Bash/readme.md b/Bash/readme.md index 311091b..7dfd46d 100644 --- a/Bash/readme.md +++ b/Bash/readme.md @@ -2,23 +2,33 @@ ## Overture Maps -Install a CLI tool to download Overture Maps data. +Install a CLI tool or overturemaps-py repo to download Overture Maps data. +```sh +bash OMF/setup.sh ``` -pip install overturemaps -``` - -Run the following code and it will download buildings in target areas from Overture Maps and import them into PostgreSQL -``` -bash buildings/download_import_ovt.sql +Run the following code and it will download building in target areas from Overture Maps and import them into PostgreSQL + +```sh +## address +bash OMF/ETL/omf_download.sh address latest +bash OMF/ETL/omf_import.sh address +## building +bash OMF/ETL/omf_download.sh building +bash OMF/ETL/omf_import.sh building +## place +bash OMF/ETL/omf_download.sh place +bash OMF/ETL/omf_import.sh place ``` +Check all version available + ## OpenStreetMap This script just import tsv files into PostgreSQL. You need to extract data before running this script. -``` -bash building/import_osm_tsv.shcd . +```sh +bash building/import_osm_tsv.sh ``` \ No newline at end of file From 975e74e7c78dfeaaa521e53332d1bd8ef2c786ff Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Fri, 26 Jul 2024 15:38:22 +0900 Subject: [PATCH 07/10] fix omf codes --- .vscode/settings.json | 2 +- Bash/OMF/ETL/omf_download.sh | 5 ++-- Bash/readme.md | 54 ++++++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 37bad0c..421ed4c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,5 @@ { - "editor.formatOnSave": false, + "editor.formatOnSave": true, "editor.tabSize": 2, "editor.defaultFormatter": "esbenp.prettier-vscode", "python.testing.unittestArgs": ["-v", "-s", ".", "-p", "test_*.py"], diff --git a/Bash/OMF/ETL/omf_download.sh b/Bash/OMF/ETL/omf_download.sh index 9f2ca74..6ad5faa 100644 --- a/Bash/OMF/ETL/omf_download.sh +++ b/Bash/OMF/ETL/omf_download.sh @@ -27,12 +27,13 @@ DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" mkdir -p $DATA_DIR # change resource in "git@github.com:OvertureMaps/overturemaps-py.git" to change -cd "${SCRIPT_DIR}/overturemaps-py" +cd "../overturemaps-py" git checkout . git pull origin main before="overturemaps-us-west-2/release/.*/theme" after="overturemaps-us-west-2/release/$RELEASE_VERSION/theme" -sed -i '' "s|$before|$after|g" "${SCRIPT_DIR}/overturemaps-py/overturemaps/core.py" +sed -i '' "s|$before|$after|g" "../overturemaps-py/overturemaps/core.py" +cd - function omf_download(){ if [ "$RELEASE_VERSION" = "latest" ]; then diff --git a/Bash/readme.md b/Bash/readme.md index 7dfd46d..6401375 100644 --- a/Bash/readme.md +++ b/Bash/readme.md @@ -2,27 +2,63 @@ ## Overture Maps -Install a CLI tool or overturemaps-py repo to download Overture Maps data. +- Install a CLI tool or overturemaps-py repo to download Overture Maps data. ```sh bash OMF/setup.sh ``` -Run the following code and it will download building in target areas from Overture Maps and import them into PostgreSQL +- Run the following code and it will download building in target areas from Overture Maps and import them into PostgreSQL ```sh +# get latest ## address bash OMF/ETL/omf_download.sh address latest -bash OMF/ETL/omf_import.sh address +bash OMF/ETL/omf_import.sh address latest ## building -bash OMF/ETL/omf_download.sh building -bash OMF/ETL/omf_import.sh building +bash OMF/ETL/omf_download.sh building latest +bash OMF/ETL/omf_import.sh building latest ## place -bash OMF/ETL/omf_download.sh place -bash OMF/ETL/omf_import.sh place +bash OMF/ETL/omf_download.sh place latest +bash OMF/ETL/omf_import.sh place latest + +# get target version +## address +bash OMF/ETL/omf_download.sh address 2024-07-22.0 +bash OMF/ETL/omf_import.sh address 2024-07-22.0 +... +``` + +- Check all versions available + +```sh +$ aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + PRE 2023-04-02-alpha/ + PRE 2023-07-26-alpha.0/ + PRE 2023-10-19-alpha.0/ + PRE 2023-11-14-alpha.0/ + PRE 2023-12-14-alpha.0/ + PRE 2024-01-17-alpha.0/ + PRE 2024-02-15-alpha.0/ + PRE 2024-03-12-alpha.0/ + PRE 2024-04-16-beta.0/ + PRE 2024-05-16-beta.0/ + PRE 2024-06-13-beta.0/ + PRE 2024-06-13-beta.1/ + PRE 2024-07-22.0/ ``` -Check all version available +- Check all themes available + +```sh +$ aws s3 ls s3://overturemaps-us-west-2/release/2024-07-22.0/ --region us-west-2 --no-sign-request + PRE theme=addresses/ + PRE theme=base/ + PRE theme=buildings/ + PRE theme=divisions/ + PRE theme=places/ + PRE theme=transportation/ +``` ## OpenStreetMap @@ -31,4 +67,4 @@ You need to extract data before running this script. ```sh bash building/import_osm_tsv.sh -``` \ No newline at end of file +``` From 408220b1099682c83bfc07e0d94a03fd8ad0efb7 Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Mon, 29 Jul 2024 10:01:58 +0900 Subject: [PATCH 08/10] rename --- ...category_main_to_tsv.sh => copy_place_category_main_to_tsv.sh} | 0 ...copy_source_in_place_to_tsv.sh => copy_place_source_to_tsv.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename Bash/OMF/Analyze/{copy_category_main_to_tsv.sh => copy_place_category_main_to_tsv.sh} (100%) rename Bash/OMF/Analyze/{copy_source_in_place_to_tsv.sh => copy_place_source_to_tsv.sh} (100%) diff --git a/Bash/OMF/Analyze/copy_category_main_to_tsv.sh b/Bash/OMF/Analyze/copy_place_category_main_to_tsv.sh similarity index 100% rename from Bash/OMF/Analyze/copy_category_main_to_tsv.sh rename to Bash/OMF/Analyze/copy_place_category_main_to_tsv.sh diff --git a/Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh b/Bash/OMF/Analyze/copy_place_source_to_tsv.sh similarity index 100% rename from Bash/OMF/Analyze/copy_source_in_place_to_tsv.sh rename to Bash/OMF/Analyze/copy_place_source_to_tsv.sh From 2a1f29a4cfe9a6760602f26fccf801d45ea54838 Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Mon, 29 Jul 2024 10:09:47 +0900 Subject: [PATCH 09/10] add script about osm --- Bash/OSM/Analyze/copy_building_to_tsv.sh | 9 ++++ Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh | 17 +++++++ Bash/OSM/Analyze/copy_poi_to_tsv.sh | 18 +++++++ Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh | 9 ++++ Bash/OSM/Analyze/copy_tags_to_tsv.sh | 4 ++ Bash/OSM/Analyze/count_height.sh | 19 ++++++++ Bash/OSM/Analyze/count_poi_amenity_shop.sh | 42 ++++++++++++++++ Bash/OSM/Analyze/coverage.sh | 22 +++++++++ .../Analyze/sql/copy_building_from_tsv.sql | 2 + .../OSM/Analyze/sql/create_table_building.sql | 42 ++++++++++++++++ .../Analyze/sql/create_table_building_tag.sql | 48 +++++++++++++++++++ Bash/OSM/Analyze/sql/create_table_poi.sql | 13 +++++ Bash/OSM/ETL/clip_select_insert_building.sh | 33 +++++++++++++ .../ETL/clip_select_insert_building_tag.sh | 37 ++++++++++++++ Bash/OSM/ETL/clip_select_insert_poi.sh | 39 +++++++++++++++ Bash/OSM/ETL/lua/building.lua | 32 +++++++++++++ Bash/OSM/ETL/lua/building_with_tag.lua | 39 +++++++++++++++ Bash/OSM/ETL/lua/poi.lua | 40 ++++++++++++++++ Bash/OSM/ETL/osm_buildings.sh | 33 +++++++++++++ Bash/OSM/ETL/osm_pois.sh | 34 +++++++++++++ Bash/OSM/ETL/sql/buildings2tsv.sql | 12 +++++ Bash/OSM/ETL/sql/pois2tsv.sql | 15 ++++++ Bash/OSM/road/network_assessment.sql | 14 ++++++ Bash/OSM/road/osm_road.sh | 41 ++++++++++++++++ Bash/OSM/road/road2tsv.sql | 29 +++++++++++ 25 files changed, 643 insertions(+) create mode 100644 Bash/OSM/Analyze/copy_building_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_poi_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_tags_to_tsv.sh create mode 100644 Bash/OSM/Analyze/count_height.sh create mode 100644 Bash/OSM/Analyze/count_poi_amenity_shop.sh create mode 100644 Bash/OSM/Analyze/coverage.sh create mode 100644 Bash/OSM/Analyze/sql/copy_building_from_tsv.sql create mode 100644 Bash/OSM/Analyze/sql/create_table_building.sql create mode 100644 Bash/OSM/Analyze/sql/create_table_building_tag.sql create mode 100644 Bash/OSM/Analyze/sql/create_table_poi.sql create mode 100644 Bash/OSM/ETL/clip_select_insert_building.sh create mode 100644 Bash/OSM/ETL/clip_select_insert_building_tag.sh create mode 100644 Bash/OSM/ETL/clip_select_insert_poi.sh create mode 100644 Bash/OSM/ETL/lua/building.lua create mode 100644 Bash/OSM/ETL/lua/building_with_tag.lua create mode 100644 Bash/OSM/ETL/lua/poi.lua create mode 100644 Bash/OSM/ETL/osm_buildings.sh create mode 100644 Bash/OSM/ETL/osm_pois.sh create mode 100644 Bash/OSM/ETL/sql/buildings2tsv.sql create mode 100644 Bash/OSM/ETL/sql/pois2tsv.sql create mode 100644 Bash/OSM/road/network_assessment.sql create mode 100644 Bash/OSM/road/osm_road.sh create mode 100644 Bash/OSM/road/road2tsv.sql diff --git a/Bash/OSM/Analyze/copy_building_to_tsv.sh b/Bash/OSM/Analyze/copy_building_to_tsv.sh new file mode 100644 index 0000000..064b033 --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_to_tsv.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + sql="\copy (select area_id, id, ST_AsText(geom)from building_$city) to '~/tsv/osm_building_$city.tsv' with csv delimiter E'\t';" + psql -d shimazaki -U postgres -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh new file mode 100644 index 0000000..36eae82 --- /dev/null +++ b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + class, subclass, count(*) as subclass_cnt + from poi_$city + group by class, subclass + order by class, subclass_cnt desc) + to '~/osm/tsv/poi_cnt_$city.tsv' + with csv delimiter E'\t'; + " + psql -d shimazaki -U postgres -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_poi_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_to_tsv.sh new file mode 100644 index 0000000..e17aa02 --- /dev/null +++ b/Bash/OSM/Analyze/copy_poi_to_tsv.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + osm_type , + osm_id , + name , + class , + subclass , + ST_AsText(geom) + from poi_$city + ) to '~/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" + psql -d shimazaki -U postgres -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh new file mode 100644 index 0000000..a70136a --- /dev/null +++ b/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + sql="\copy (select building, count(*) as cnt from building_tag_$city group by building order by cnt desc) to '~/buildings/tsv/building_tag_cnt_$city.tsv' with csv delimiter E'\t';" + psql -d shimazaki -U postgres -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_to_tsv.sh new file mode 100644 index 0000000..293cc6b --- /dev/null +++ b/Bash/OSM/Analyze/copy_tags_to_tsv.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +sql="\copy (select distinct building from building_tag order by building) to '~/buildings/tsv/building_distinct_tag.tsv' with csv delimiter E'\t';" +psql -d shimazaki -U postgres -c "$sql" diff --git a/Bash/OSM/Analyze/count_height.sh b/Bash/OSM/Analyze/count_height.sh new file mode 100644 index 0000000..e9c9846 --- /dev/null +++ b/Bash/OSM/Analyze/count_height.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql=" + select + count(height) as cnt_height + from building_tag_$city + " + psql -d shimazaki -U postgres -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/count_poi_amenity_shop.sh b/Bash/OSM/Analyze/count_poi_amenity_shop.sh new file mode 100644 index 0000000..e331544 --- /dev/null +++ b/Bash/OSM/Analyze/count_poi_amenity_shop.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql=" + select + name, + cnt + from ( + select + 'poi_cnt' as name, + count(*) as cnt + from poi_$city + + union + + select + 'poi_amenity_cnt' as name, + count(*) as cnt + from poi_$city + where class='amenity' + + union + + select + 'poi_shop_cnt' as name, + count(*) as cnt + from poi_$city + where class='shop' + ) a + order by name + " + psql -d shimazaki -U postgres -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/coverage.sh b/Bash/OSM/Analyze/coverage.sh new file mode 100644 index 0000000..15e94f2 --- /dev/null +++ b/Bash/OSM/Analyze/coverage.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from building_$city" + building_area=$(psql -t -d shimazaki -U postgres -c "$sql") + echo $building_area + + sql="select ST_Area(geom::geography) / 1000000 from shape where name='${city_labels[$cnt]}'" + whole_area=$(psql -t -d shimazaki -U postgres -c "$sql") + echo $whole_area + + echo "scale=5; $building_area / $whole_area * 100" | bc + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql b/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql new file mode 100644 index 0000000..b73f7b4 --- /dev/null +++ b/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql @@ -0,0 +1,2 @@ +\copy shimazaki from '../osm_buildings_japan.tsv' ( delimiter E'\t', format csv, header true ); + diff --git a/Bash/OSM/Analyze/sql/create_table_building.sql b/Bash/OSM/Analyze/sql/create_table_building.sql new file mode 100644 index 0000000..985f861 --- /dev/null +++ b/Bash/OSM/Analyze/sql/create_table_building.sql @@ -0,0 +1,42 @@ +drop table if exists building; +create table building ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_tokyo; +create table building_tokyo ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_hamamatsu; +create table building_hamamatsu ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_tateyama; +create table building_tateyama ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_morioka; +create table building_morioka ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_higashi_hiroshima; +create table building_higashi_hiroshima ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); +drop table if exists building_kumamoto; +create table building_kumamoto ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326) +); diff --git a/Bash/OSM/Analyze/sql/create_table_building_tag.sql b/Bash/OSM/Analyze/sql/create_table_building_tag.sql new file mode 100644 index 0000000..5285ddc --- /dev/null +++ b/Bash/OSM/Analyze/sql/create_table_building_tag.sql @@ -0,0 +1,48 @@ +drop table if exists building_tag_tokyo; +create table building_tag_tokyo ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); +drop table if exists building_tag_hamamatsu; +create table building_tag_hamamatsu ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); +drop table if exists building_tag_tateyama; +create table building_tag_tateyama ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); +drop table if exists building_tag_morioka; +create table building_tag_morioka ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); +drop table if exists building_tag_higashi_hiroshima; +create table building_tag_higashi_hiroshima ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); +drop table if exists building_tag_kumamoto; +create table building_tag_kumamoto ( + area_id bigint, + id bigint, + geom geometry(multipolygon,4326), + building text, + height text +); diff --git a/Bash/OSM/Analyze/sql/create_table_poi.sql b/Bash/OSM/Analyze/sql/create_table_poi.sql new file mode 100644 index 0000000..a48a2a0 --- /dev/null +++ b/Bash/OSM/Analyze/sql/create_table_poi.sql @@ -0,0 +1,13 @@ +drop table if exists poi_tokyo; +create table poi_tokyo (LIKE pois INCLUDING ALL); +drop table if exists poi_hamamatsu; +create table poi_hamamatsu (LIKE pois INCLUDING ALL); +drop table if exists poi_tateyama; +create table poi_tateyama (LIKE pois INCLUDING ALL); +drop table if exists poi_morioka; +create table poi_morioka (LIKE pois INCLUDING ALL); +drop table if exists poi_higashi_hiroshima; +create table poi_higashi_hiroshima (LIKE pois INCLUDING ALL); +drop table if exists poi_kumamoto; +create table poi_kumamoto (LIKE pois INCLUDING ALL); + diff --git a/Bash/OSM/ETL/clip_select_insert_building.sh b/Bash/OSM/ETL/clip_select_insert_building.sh new file mode 100644 index 0000000..61b60c7 --- /dev/null +++ b/Bash/OSM/ETL/clip_select_insert_building.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql="truncate building_$city" + psql -d shimazaki -U postgres -c "$sql" + + sql=" + with clipping_area as ( + select * from shape where name='${city_labels[$cnt]}' + ) + insert into building_$city ( + area_id, + id, + geom + ) + select + area_id, + id, + ST_AsText(ST_Intersection(building.geom, clipping_area.geom)) as geom + from building, clipping_area + where ST_Intersects(building.geom, clipping_area.geom) + " + psql -d shimazaki -U postgres -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/ETL/clip_select_insert_building_tag.sh b/Bash/OSM/ETL/clip_select_insert_building_tag.sh new file mode 100644 index 0000000..3fc3b0a --- /dev/null +++ b/Bash/OSM/ETL/clip_select_insert_building_tag.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql="truncate building_tag_$city" + psql -d shimazaki -U postgres -c "$sql" + + sql=" + with clipping_area as ( + select * from shape where name='${city_labels[$cnt]}' + ) + insert into building_tag_$city ( + area_id, + id, + geom, + building, + height + ) + select + area_id, + id, + ST_AsText(ST_Intersection(building_tag.geom, clipping_area.geom)) as geom, + building, + height + from building_tag, clipping_area + where ST_Intersects(building_tag.geom, clipping_area.geom) + " + psql -d shimazaki -U postgres -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/ETL/clip_select_insert_poi.sh b/Bash/OSM/ETL/clip_select_insert_poi.sh new file mode 100644 index 0000000..382a9b1 --- /dev/null +++ b/Bash/OSM/ETL/clip_select_insert_poi.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") + +cnt=0 +for city in "${cities[@]}"; do + echo $city + city_lable="$city_labels[$cnt]" + + sql="truncate poi_$city" + psql -d shimazaki -U postgres -c "$sql" + + sql=" + with clipping_area as ( + select * from shape where name='${city_labels[$cnt]}' + ) + insert into poi_$city ( + osm_type , + osm_id , + name , + class , + subclass , + geom + ) + select + osm_type , + osm_id , + pois.name, + class , + subclass , + ST_AsText(ST_Intersection(pois.geom, clipping_area.geom)) as geom + from pois, clipping_area + where ST_Intersects(pois.geom, clipping_area.geom) + " + psql -d shimazaki -U postgres -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/ETL/lua/building.lua b/Bash/OSM/ETL/lua/building.lua new file mode 100644 index 0000000..82fd115 --- /dev/null +++ b/Bash/OSM/ETL/lua/building.lua @@ -0,0 +1,32 @@ +local buildings = osm2pgsql.define_area_table('buildings', { + -- Define an autoincrementing id column, QGIS likes a unique id on the table + { column = 'id', sql_type = 'serial', create_only = true }, + { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, +}, { indexes = { + -- So we get an index on the id column + { column = 'id', method = 'btree', unique = true }, + -- If we define any indexes we don't get the default index on the geometry + -- column, so we add it here. + { column = 'geom', method = 'gist' } +}}) + +function osm2pgsql.process_way(object) + if object.is_closed and object.tags.building then + buildings:insert({ + geom = object:as_polygon():transform(4326) + }) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.type == 'multipolygon' and object.tags.building then + -- From the relation we get multipolygons... + local mp = object:as_multipolygon():transform(4326) + -- ...and split them into polygons which we insert into the table + for geom in mp:geometries() do + buildings:insert({ + geom = geom + }) + end + end +end diff --git a/Bash/OSM/ETL/lua/building_with_tag.lua b/Bash/OSM/ETL/lua/building_with_tag.lua new file mode 100644 index 0000000..518cb0a --- /dev/null +++ b/Bash/OSM/ETL/lua/building_with_tag.lua @@ -0,0 +1,39 @@ +local building_tag = osm2pgsql.define_area_table('building_tag', { + -- Define an autoincrementing id column, QGIS likes a unique id on the table + { column = 'id', sql_type = 'serial', create_only = true }, + { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, + -- Add columns for tags + { column = 'building', type = 'text' }, + { column = 'height', type = 'text' } +}, { indexes = { + -- So we get an index on the id column + { column = 'id', method = 'btree', unique = true }, + -- If we define any indexes we don't get the default index on the geometry + -- column, so we add it here. + { column = 'geom', method = 'gist' } +}}) + +function osm2pgsql.process_way(object) + if object.is_closed and object.tags.building then + building_tag:insert({ + geom = object:as_polygon():transform(4326), + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag + }) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.type == 'multipolygon' and object.tags.building then + -- From the relation we get multipolygons... + local mp = object:as_multipolygon():transform(4326) + -- ...and split them into polygons which we insert into the table + for geom in mp:geometries() do + building_tag:insert({ + geom = geom, + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag + }) + end + end +end diff --git a/Bash/OSM/ETL/lua/poi.lua b/Bash/OSM/ETL/lua/poi.lua new file mode 100644 index 0000000..43350c4 --- /dev/null +++ b/Bash/OSM/ETL/lua/poi.lua @@ -0,0 +1,40 @@ + +local pois = osm2pgsql.define_table({ + name = 'pois', + ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' }, + columns = { + { column = 'name' }, + { column = 'class', not_null = true }, + { column = 'subclass' }, + { column = 'geom', type = 'point', not_null = true, projection = 4326}, +}}) + +function process_poi(object, geom) + local a = { + name = object.tags.name, + geom = geom + } + + if object.tags.amenity then + a.class = 'amenity' + a.subclass = object.tags.amenity + elseif object.tags.shop then + a.class = 'shop' + a.subclass = object.tags.shop + else + return + end + + pois:insert(a) +end + +function osm2pgsql.process_node(object) + process_poi(object, object:as_point():transform(4326)) +end + +function osm2pgsql.process_way(object) + if object.is_closed and object.tags.building then + process_poi(object, object:as_polygon():centroid()) + end +end + diff --git a/Bash/OSM/ETL/osm_buildings.sh b/Bash/OSM/ETL/osm_buildings.sh new file mode 100644 index 0000000..74e3c7b --- /dev/null +++ b/Bash/OSM/ETL/osm_buildings.sh @@ -0,0 +1,33 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +psql -U postgres -d $3 -c "drop table if exists public.buildings;" + +# import pbf file to database +/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/buildings.lua -E 4326 $1 + +mkdir ./osm_buildings_${2}/ + +# export table to filqs +pg_dump -U postgres -t public.buildings $3 | gzip > ./osm_buildings_${2}/osm_buildings_${2}.sql.gz + +psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/buildings2tsv.sql +mv ./buildings.tsv ./osm_buildings_${2}/osm_buildings_${2}.tsv +rm ./osm_buildings_${2}/osm_buildings_${2}.tsv.gz +gzip ./osm_buildings_${2}/osm_buildings_${2}.tsv + +# export both tables to shp and compression them +mkdir ./osm_buildings_${2}/osm_buildings_${2}_shp/ +pgsql2shp -u postgres -P postgres -f ./osm_buildings_${2}/osm_buildings_${2}_shp/osm_buildings_${2}.shp $3 public.buildings +zip ./osm_buildings_${2}/osm_buildings_${2}_shp.zip -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ +rm -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ + +psql -U postgres -d $3 -c "drop table if exists public.buildings;" + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_pois.sh b/Bash/OSM/ETL/osm_pois.sh new file mode 100644 index 0000000..a47a4bd --- /dev/null +++ b/Bash/OSM/ETL/osm_pois.sh @@ -0,0 +1,34 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +echo $2 +psql -U postgres -d $3 -c "drop table if exists public.pois;" + +# import pbf file to database +/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/pois.lua -E 4326 $1 + +mkdir ./osm_pois_$2/ + +# export table to files +pg_dump -U postgres -t public.pois $3 | gzip > ./osm_pois_$2/osm_pois_$2.sql.gz + +psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/pois2tsv.sql +mv ./pois.tsv ./osm_pois_$2/osm_pois_${2}.tsv +rm ./osm_pois_${2}/osm_pois_${2}.tsv.gz +gzip ./osm_pois_${2}/osm_pois_${2}.tsv + +# export both tables to shp and compression them +mkdir ./osm_pois_${2}/osm_pois_${2}_shp/ +pgsql2shp -u postgres -P postgres -f ./osm_pois_${2}/osm_pois_${2}_shp/osm_pois_${2}.shp $3 public.pois +zip ./osm_pois_${2}/osm_pois_${2}_shp.zip -r ./osm_pois_${2}/osm_pois_${2}_shp/ +rm -r ./osm_pois_${2}/osm_pois_${2}_shp/ + +psql -U postgres -d $3 -c "drop table if exists public.pois;" + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/buildings2tsv.sql b/Bash/OSM/ETL/sql/buildings2tsv.sql new file mode 100644 index 0000000..87fff60 --- /dev/null +++ b/Bash/OSM/ETL/sql/buildings2tsv.sql @@ -0,0 +1,12 @@ +copy ( + select + area_id, + id, + ST_asText(geom) + from + public.buildings + where + area_id is not null and geom is not null + order by + area_id +) to STDOUT with csv header delimiter E'\t' \g 'buildings.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/pois2tsv.sql b/Bash/OSM/ETL/sql/pois2tsv.sql new file mode 100644 index 0000000..eca60be --- /dev/null +++ b/Bash/OSM/ETL/sql/pois2tsv.sql @@ -0,0 +1,15 @@ +copy ( + select + osm_id, + osm_type, + name, + class, + subclass, + ST_asText(geom) + from + public.pois + where + osm_id is not null and geom is not null + order by + osm_id +) to STDOUT with csv header delimiter E'\t' \g 'pois.tsv' \ No newline at end of file diff --git a/Bash/OSM/road/network_assessment.sql b/Bash/OSM/road/network_assessment.sql new file mode 100644 index 0000000..ebef3ac --- /dev/null +++ b/Bash/OSM/road/network_assessment.sql @@ -0,0 +1,14 @@ +alter table public.osm_road_available add column groupno integer; + +update public.osm_road_available +set groupno = val.groupno +from ( + select + oras.groupno as groupno, + orav.id as id + from + public.osm_road_assessment oras, + public.osm_road_available orav + where ST_Intersects(orav.geom_way, oras.geom) +) as val where public.osm_road_available.id = val.id ; + diff --git a/Bash/OSM/road/osm_road.sh b/Bash/OSM/road/osm_road.sh new file mode 100644 index 0000000..7433f1e --- /dev/null +++ b/Bash/OSM/road/osm_road.sh @@ -0,0 +1,41 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +# remove output files generated previpusly +rm -r ./osm_road + +# convert osm.pbf to sql.gz by using osm2po +sh ${0%/*}/osm2road/conv_osm_road.sh $1 + +psql -d postgres -c "drop table if exists public.osm_road_available;" +psql -d postgres -c "drop table if exists public.osm_road_assessment;" + +# import the files into tables +gzip -cd ./osm_road/osm_road_available.sql.gz | psql -d postgres +gzip -cd ./osm_road/osm_road_assessment.sql.gz | psql -d postgres + +# assess the network +psql -v ON_ERROR_STOP=1 -d postgres -f ${0%/*}/network_assessment.sql + +# export the network to files +psql -d postgres -f ${0%/*}/road2tsv.sql +mv ./osm_road/road.tsv ./osm_road/osm_road_$2.tsv +pg_dump -d postgres -t osm_road_available | gzip > ./osm_road/osm_road_$2.sql.gz + +mkdir ./osm_road/osm_road_${2}_shp/ +pgsql2shp -P postgreses -f ./osm_road/osm_road_${2}_shp/osm_road_${2}.shp miyake public.road +zip ./osm_road/osm_road_${2}_shp.zip -r ./osm_road/osm_road_${2}_shp/ +rm -r ./osm_road/osm_road_${2}_shp/ + +mv ./osm_road/ ./osm_road_${2}/ + +psql -d postgres -c "drop table if exists public.osm_road_available;" +psql -d postgres -c "drop table if exists public.osm_road_assessment;" + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" diff --git a/Bash/OSM/road/road2tsv.sql b/Bash/OSM/road/road2tsv.sql new file mode 100644 index 0000000..fa8237c --- /dev/null +++ b/Bash/OSM/road/road2tsv.sql @@ -0,0 +1,29 @@ +copy ( + select + groupno, + osm_id, + id, + osm_source_id, + osm_target_id, + clazz, + flags, + source, + target, + km, + kmh, + cost, + reverse_cost, + x1, + y1, + x2, + y2, + osm_name, + osm_meta, + ST_asText(geom_way) + from + public.osm_road_available + where + osm_id is not null and geom_way is not null + order by + osm_id +) to STDOUT with csv header delimiter E'\t' \g 'road.tsv' \ No newline at end of file From eb2754527b9c0c05dc86e8505963378db5eaaadc Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Tue, 30 Jul 2024 13:53:08 +0900 Subject: [PATCH 10/10] fix osm script --- Bash/OMF/ETL/omf_import.sh | 2 +- .../ETL/{ => sql}/create_tables_building.sql | 0 .../OMF/ETL/{ => sql}/create_tables_place.sql | 0 Bash/OMF/overturemaps-py | 1 + Bash/OSM/Analyze/config.sh | 10 ++ .../Analyze/copy_building_tags_cnt_to_tsv.sh | 24 +++ Bash/OSM/Analyze/copy_building_tags_to_tsv.sh | 17 ++ Bash/OSM/Analyze/copy_building_to_tsv.sh | 20 ++- Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh | 17 +- Bash/OSM/Analyze/copy_poi_to_tsv.sh | 13 +- Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh | 9 -- Bash/OSM/Analyze/copy_tags_to_tsv.sh | 4 - Bash/OSM/Analyze/count_building_height.sh | 21 +++ Bash/OSM/Analyze/count_height.sh | 19 --- Bash/OSM/Analyze/count_poi_amenity_shop.sh | 28 ++-- Bash/OSM/Analyze/coverage.sh | 17 +- Bash/OSM/Analyze/sql/building2tsv.sql | 16 ++ .../Analyze/sql/copy_building_from_tsv.sql | 2 - .../OSM/Analyze/sql/create_table_building.sql | 42 ----- .../Analyze/sql/create_table_building_tag.sql | 48 ------ Bash/OSM/Analyze/sql/create_table_poi.sql | 13 -- .../pois2tsv.sql => Analyze/sql/poi2tsv.sql} | 4 +- Bash/OSM/ETL/clip_select_insert_building.sh | 33 ---- .../ETL/clip_select_insert_building_tag.sh | 37 ----- Bash/OSM/ETL/clip_select_insert_poi.sh | 39 ----- Bash/OSM/ETL/lua/building.lua | 19 ++- Bash/OSM/ETL/lua/building_with_tag.lua | 39 ----- Bash/OSM/ETL/lua/poi.lua | 6 +- Bash/OSM/ETL/osm_buildings.sh | 33 ---- Bash/OSM/ETL/osm_download.sh | 28 ++++ Bash/OSM/ETL/osm_import.sh | 72 +++++++++ Bash/OSM/ETL/osm_pois.sh | 34 ---- Bash/OSM/ETL/shape/bbox_list.cpg | 1 + Bash/OSM/ETL/shape/bbox_list.dbf | Bin 0 -> 4043 bytes Bash/OSM/ETL/shape/bbox_list.prj | 1 + Bash/OSM/ETL/shape/bbox_list.qix | Bin 0 -> 88 bytes Bash/OSM/ETL/shape/bbox_list.shp | Bin 0 -> 1052 bytes Bash/OSM/ETL/shape/bbox_list.shx | Bin 0 -> 156 bytes Bash/OSM/ETL/sql/buildings2tsv.sql | 12 -- .../ETL/sql/clip_select_insert_building.sql | 143 +++++++++++++++++ Bash/OSM/ETL/sql/clip_select_insert_poi.sql | 149 ++++++++++++++++++ Bash/OSM/ETL/sql/create_tables_building.sql | 31 ++++ Bash/OSM/ETL/sql/create_tables_poi.sql | 31 ++++ Bash/OSM/road/network_assessment.sql | 14 -- Bash/OSM/road/osm_road.sh | 41 ----- Bash/OSM/road/road2tsv.sql | 29 ---- Bash/readme.md | 27 +++- 47 files changed, 648 insertions(+), 498 deletions(-) rename Bash/OMF/ETL/{ => sql}/create_tables_building.sql (100%) rename Bash/OMF/ETL/{ => sql}/create_tables_place.sql (100%) create mode 160000 Bash/OMF/overturemaps-py create mode 100644 Bash/OSM/Analyze/config.sh create mode 100644 Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_building_tags_to_tsv.sh delete mode 100644 Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh delete mode 100644 Bash/OSM/Analyze/copy_tags_to_tsv.sh create mode 100644 Bash/OSM/Analyze/count_building_height.sh delete mode 100644 Bash/OSM/Analyze/count_height.sh create mode 100644 Bash/OSM/Analyze/sql/building2tsv.sql delete mode 100644 Bash/OSM/Analyze/sql/copy_building_from_tsv.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_building.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_building_tag.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_poi.sql rename Bash/OSM/{ETL/sql/pois2tsv.sql => Analyze/sql/poi2tsv.sql} (73%) delete mode 100644 Bash/OSM/ETL/clip_select_insert_building.sh delete mode 100644 Bash/OSM/ETL/clip_select_insert_building_tag.sh delete mode 100644 Bash/OSM/ETL/clip_select_insert_poi.sh delete mode 100644 Bash/OSM/ETL/lua/building_with_tag.lua delete mode 100644 Bash/OSM/ETL/osm_buildings.sh create mode 100755 Bash/OSM/ETL/osm_download.sh create mode 100755 Bash/OSM/ETL/osm_import.sh delete mode 100644 Bash/OSM/ETL/osm_pois.sh create mode 100644 Bash/OSM/ETL/shape/bbox_list.cpg create mode 100644 Bash/OSM/ETL/shape/bbox_list.dbf create mode 100644 Bash/OSM/ETL/shape/bbox_list.prj create mode 100644 Bash/OSM/ETL/shape/bbox_list.qix create mode 100644 Bash/OSM/ETL/shape/bbox_list.shp create mode 100644 Bash/OSM/ETL/shape/bbox_list.shx delete mode 100644 Bash/OSM/ETL/sql/buildings2tsv.sql create mode 100755 Bash/OSM/ETL/sql/clip_select_insert_building.sql create mode 100755 Bash/OSM/ETL/sql/clip_select_insert_poi.sql create mode 100644 Bash/OSM/ETL/sql/create_tables_building.sql create mode 100644 Bash/OSM/ETL/sql/create_tables_poi.sql delete mode 100644 Bash/OSM/road/network_assessment.sql delete mode 100644 Bash/OSM/road/osm_road.sh delete mode 100644 Bash/OSM/road/road2tsv.sql diff --git a/Bash/OMF/ETL/omf_import.sh b/Bash/OMF/ETL/omf_import.sh index 67342a4..2f7601d 100644 --- a/Bash/OMF/ETL/omf_import.sh +++ b/Bash/OMF/ETL/omf_import.sh @@ -33,7 +33,7 @@ echo "Change psql user(-U) or database(-d) if threre is error about postgres" psql -U postgres -d postgres -c "create schema omf;" # create table -psql -U postgres -d postgres -f ./create_tables_$TYPE.sql +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" # Tokyo ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_$TYPE -nlt multipolygon $DATA_DIR/tokyo_$TYPE.geojson diff --git a/Bash/OMF/ETL/create_tables_building.sql b/Bash/OMF/ETL/sql/create_tables_building.sql similarity index 100% rename from Bash/OMF/ETL/create_tables_building.sql rename to Bash/OMF/ETL/sql/create_tables_building.sql diff --git a/Bash/OMF/ETL/create_tables_place.sql b/Bash/OMF/ETL/sql/create_tables_place.sql similarity index 100% rename from Bash/OMF/ETL/create_tables_place.sql rename to Bash/OMF/ETL/sql/create_tables_place.sql diff --git a/Bash/OMF/overturemaps-py b/Bash/OMF/overturemaps-py new file mode 160000 index 0000000..5148bf4 --- /dev/null +++ b/Bash/OMF/overturemaps-py @@ -0,0 +1 @@ +Subproject commit 5148bf40fa01d125095c29f70c4c91db17e4dfdd diff --git a/Bash/OSM/Analyze/config.sh b/Bash/OSM/Analyze/config.sh new file mode 100644 index 0000000..b1d3423 --- /dev/null +++ b/Bash/OSM/Analyze/config.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# define cities +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hmaamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") ## becase of shape file 'Hmaamatsu' + diff --git a/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh new file mode 100644 index 0000000..0477e14 --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + building, + count(*) as cnt + from osm.building_$city + group by + building + order by + cnt desc + ) to '$SCRIPT_DIR/tsv/osm_building_cnt_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh new file mode 100644 index 0000000..99707ee --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +sql="\copy ( + select + distinct building + from osm.building + order by building + ) to '$SCRIPT_DIR/tsv/osm_building_distinct_tag.tsv' + with csv delimiter E'\t';" +psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" diff --git a/Bash/OSM/Analyze/copy_building_to_tsv.sh b/Bash/OSM/Analyze/copy_building_to_tsv.sh index 064b033..fc08ef1 100644 --- a/Bash/OSM/Analyze/copy_building_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_building_to_tsv.sh @@ -1,9 +1,23 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city - sql="\copy (select area_id, id, ST_AsText(geom)from building_$city) to '~/tsv/osm_building_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" + sql="\copy ( + select + area_id, + id, + ST_AsText(geom), + building, + height + from osm.building_$city + ) to '$SCRIPT_DIR/tsv/osm_building_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh index 36eae82..86c6822 100644 --- a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh @@ -1,17 +1,22 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city sql="\copy ( select class, subclass, count(*) as subclass_cnt - from poi_$city + from osm.poi_$city group by class, subclass - order by class, subclass_cnt desc) - to '~/osm/tsv/poi_cnt_$city.tsv' - with csv delimiter E'\t'; + order by class, subclass_cnt desc + ) to '$SCRIPT_DIR/tsv/osm_poi_cnt_$city.tsv' + with csv delimiter E'\t'; " - psql -d shimazaki -U postgres -c "$sql" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_poi_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_to_tsv.sh index e17aa02..c1f7105 100644 --- a/Bash/OSM/Analyze/copy_poi_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_poi_to_tsv.sh @@ -1,6 +1,11 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city @@ -12,7 +17,7 @@ for city in "${cities[@]}"; do class , subclass , ST_AsText(geom) - from poi_$city - ) to '~/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" + from osm.poi_$city + ) to '$SCRIPT_DIR/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh deleted file mode 100644 index a70136a..0000000 --- a/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") - -for city in "${cities[@]}"; do - echo $city - sql="\copy (select building, count(*) as cnt from building_tag_$city group by building order by cnt desc) to '~/buildings/tsv/building_tag_cnt_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" -done diff --git a/Bash/OSM/Analyze/copy_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_to_tsv.sh deleted file mode 100644 index 293cc6b..0000000 --- a/Bash/OSM/Analyze/copy_tags_to_tsv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -sql="\copy (select distinct building from building_tag order by building) to '~/buildings/tsv/building_distinct_tag.tsv' with csv delimiter E'\t';" -psql -d shimazaki -U postgres -c "$sql" diff --git a/Bash/OSM/Analyze/count_building_height.sh b/Bash/OSM/Analyze/count_building_height.sh new file mode 100644 index 0000000..682dd4a --- /dev/null +++ b/Bash/OSM/Analyze/count_building_height.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +cnt=0 +for city in "${cities[@]}"; do + echo $city + sql=" + select + count(height) as cnt_height + from osm.building_$city + " + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/count_height.sh b/Bash/OSM/Analyze/count_height.sh deleted file mode 100644 index e9c9846..0000000 --- a/Bash/OSM/Analyze/count_height.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql=" - select - count(height) as cnt_height - from building_tag_$city - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/Analyze/count_poi_amenity_shop.sh b/Bash/OSM/Analyze/count_poi_amenity_shop.sh index e331544..52a4af0 100644 --- a/Bash/OSM/Analyze/count_poi_amenity_shop.sh +++ b/Bash/OSM/Analyze/count_poi_amenity_shop.sh @@ -1,13 +1,15 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh cnt=0 for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - + echo $city sql=" select name, @@ -16,27 +18,27 @@ for city in "${cities[@]}"; do select 'poi_cnt' as name, count(*) as cnt - from poi_$city + from osm.poi_$city union - select + select 'poi_amenity_cnt' as name, - count(*) as cnt - from poi_$city + count(*) as cnt + from osm.poi_$city where class='amenity' union select 'poi_shop_cnt' as name, - count(*) as cnt - from poi_$city - where class='shop' + count(*) as cnt + from osm.poi_$city + where class='shop' ) a order by name " - psql -d shimazaki -U postgres -c "$sql" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" ((cnt++)) done diff --git a/Bash/OSM/Analyze/coverage.sh b/Bash/OSM/Analyze/coverage.sh index 15e94f2..d958268 100644 --- a/Bash/OSM/Analyze/coverage.sh +++ b/Bash/OSM/Analyze/coverage.sh @@ -1,19 +1,22 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh cnt=0 for city in "${cities[@]}"; do echo $city - city_lable="$city_labels[$cnt]" - sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from building_$city" - building_area=$(psql -t -d shimazaki -U postgres -c "$sql") + sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from osm.building_$city" + building_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") echo $building_area - sql="select ST_Area(geom::geography) / 1000000 from shape where name='${city_labels[$cnt]}'" - whole_area=$(psql -t -d shimazaki -U postgres -c "$sql") + sql="select ST_Area(geom::geography) / 1000000 from osm.shape where name='${city_labels[$cnt]}'" + whole_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") echo $whole_area echo "scale=5; $building_area / $whole_area * 100" | bc diff --git a/Bash/OSM/Analyze/sql/building2tsv.sql b/Bash/OSM/Analyze/sql/building2tsv.sql new file mode 100644 index 0000000..5adcdf7 --- /dev/null +++ b/Bash/OSM/Analyze/sql/building2tsv.sql @@ -0,0 +1,16 @@ +copy ( + select + area_id, + id, + ST_asText (geom), + building, + height + from + osm.building + where + area_id is not null + and geom is not null + order by + area_id +) +to STDOUT with csv header delimiter E'\t' \g '../tsv/building.tsv' \ No newline at end of file diff --git a/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql b/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql deleted file mode 100644 index b73f7b4..0000000 --- a/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql +++ /dev/null @@ -1,2 +0,0 @@ -\copy shimazaki from '../osm_buildings_japan.tsv' ( delimiter E'\t', format csv, header true ); - diff --git a/Bash/OSM/Analyze/sql/create_table_building.sql b/Bash/OSM/Analyze/sql/create_table_building.sql deleted file mode 100644 index 985f861..0000000 --- a/Bash/OSM/Analyze/sql/create_table_building.sql +++ /dev/null @@ -1,42 +0,0 @@ -drop table if exists building; -create table building ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_tokyo; -create table building_tokyo ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_hamamatsu; -create table building_hamamatsu ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_tateyama; -create table building_tateyama ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_morioka; -create table building_morioka ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_higashi_hiroshima; -create table building_higashi_hiroshima ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_kumamoto; -create table building_kumamoto ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); diff --git a/Bash/OSM/Analyze/sql/create_table_building_tag.sql b/Bash/OSM/Analyze/sql/create_table_building_tag.sql deleted file mode 100644 index 5285ddc..0000000 --- a/Bash/OSM/Analyze/sql/create_table_building_tag.sql +++ /dev/null @@ -1,48 +0,0 @@ -drop table if exists building_tag_tokyo; -create table building_tag_tokyo ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_hamamatsu; -create table building_tag_hamamatsu ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_tateyama; -create table building_tag_tateyama ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_morioka; -create table building_tag_morioka ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_higashi_hiroshima; -create table building_tag_higashi_hiroshima ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_kumamoto; -create table building_tag_kumamoto ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); diff --git a/Bash/OSM/Analyze/sql/create_table_poi.sql b/Bash/OSM/Analyze/sql/create_table_poi.sql deleted file mode 100644 index a48a2a0..0000000 --- a/Bash/OSM/Analyze/sql/create_table_poi.sql +++ /dev/null @@ -1,13 +0,0 @@ -drop table if exists poi_tokyo; -create table poi_tokyo (LIKE pois INCLUDING ALL); -drop table if exists poi_hamamatsu; -create table poi_hamamatsu (LIKE pois INCLUDING ALL); -drop table if exists poi_tateyama; -create table poi_tateyama (LIKE pois INCLUDING ALL); -drop table if exists poi_morioka; -create table poi_morioka (LIKE pois INCLUDING ALL); -drop table if exists poi_higashi_hiroshima; -create table poi_higashi_hiroshima (LIKE pois INCLUDING ALL); -drop table if exists poi_kumamoto; -create table poi_kumamoto (LIKE pois INCLUDING ALL); - diff --git a/Bash/OSM/ETL/sql/pois2tsv.sql b/Bash/OSM/Analyze/sql/poi2tsv.sql similarity index 73% rename from Bash/OSM/ETL/sql/pois2tsv.sql rename to Bash/OSM/Analyze/sql/poi2tsv.sql index eca60be..f38b3e2 100644 --- a/Bash/OSM/ETL/sql/pois2tsv.sql +++ b/Bash/OSM/Analyze/sql/poi2tsv.sql @@ -7,9 +7,9 @@ copy ( subclass, ST_asText(geom) from - public.pois + osm.poi where osm_id is not null and geom is not null order by osm_id -) to STDOUT with csv header delimiter E'\t' \g 'pois.tsv' \ No newline at end of file +) to STDOUT with csv header delimiter E'\t' \g '../tsv/poi.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/clip_select_insert_building.sh b/Bash/OSM/ETL/clip_select_insert_building.sh deleted file mode 100644 index 61b60c7..0000000 --- a/Bash/OSM/ETL/clip_select_insert_building.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate building_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into building_$city ( - area_id, - id, - geom - ) - select - area_id, - id, - ST_AsText(ST_Intersection(building.geom, clipping_area.geom)) as geom - from building, clipping_area - where ST_Intersects(building.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/clip_select_insert_building_tag.sh b/Bash/OSM/ETL/clip_select_insert_building_tag.sh deleted file mode 100644 index 3fc3b0a..0000000 --- a/Bash/OSM/ETL/clip_select_insert_building_tag.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate building_tag_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into building_tag_$city ( - area_id, - id, - geom, - building, - height - ) - select - area_id, - id, - ST_AsText(ST_Intersection(building_tag.geom, clipping_area.geom)) as geom, - building, - height - from building_tag, clipping_area - where ST_Intersects(building_tag.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/clip_select_insert_poi.sh b/Bash/OSM/ETL/clip_select_insert_poi.sh deleted file mode 100644 index 382a9b1..0000000 --- a/Bash/OSM/ETL/clip_select_insert_poi.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate poi_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into poi_$city ( - osm_type , - osm_id , - name , - class , - subclass , - geom - ) - select - osm_type , - osm_id , - pois.name, - class , - subclass , - ST_AsText(ST_Intersection(pois.geom, clipping_area.geom)) as geom - from pois, clipping_area - where ST_Intersects(pois.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/lua/building.lua b/Bash/OSM/ETL/lua/building.lua index 82fd115..3b35e73 100644 --- a/Bash/OSM/ETL/lua/building.lua +++ b/Bash/OSM/ETL/lua/building.lua @@ -1,7 +1,10 @@ -local buildings = osm2pgsql.define_area_table('buildings', { +local building = osm2pgsql.define_area_table('building', { -- Define an autoincrementing id column, QGIS likes a unique id on the table { column = 'id', sql_type = 'serial', create_only = true }, - { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, + { column = 'geom', type = 'multipolygon', not_null = true, projection = 4326}, + -- Add columns for tags + { column = 'building', type = 'text' }, + { column = 'height', type = 'text' } }, { indexes = { -- So we get an index on the id column { column = 'id', method = 'btree', unique = true }, @@ -12,8 +15,10 @@ local buildings = osm2pgsql.define_area_table('buildings', { function osm2pgsql.process_way(object) if object.is_closed and object.tags.building then - buildings:insert({ - geom = object:as_polygon():transform(4326) + building:insert({ + geom = object:as_polygon():transform(4326), + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag }) end end @@ -24,8 +29,10 @@ function osm2pgsql.process_relation(object) local mp = object:as_multipolygon():transform(4326) -- ...and split them into polygons which we insert into the table for geom in mp:geometries() do - buildings:insert({ - geom = geom + building:insert({ + geom = geom, + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag }) end end diff --git a/Bash/OSM/ETL/lua/building_with_tag.lua b/Bash/OSM/ETL/lua/building_with_tag.lua deleted file mode 100644 index 518cb0a..0000000 --- a/Bash/OSM/ETL/lua/building_with_tag.lua +++ /dev/null @@ -1,39 +0,0 @@ -local building_tag = osm2pgsql.define_area_table('building_tag', { - -- Define an autoincrementing id column, QGIS likes a unique id on the table - { column = 'id', sql_type = 'serial', create_only = true }, - { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, - -- Add columns for tags - { column = 'building', type = 'text' }, - { column = 'height', type = 'text' } -}, { indexes = { - -- So we get an index on the id column - { column = 'id', method = 'btree', unique = true }, - -- If we define any indexes we don't get the default index on the geometry - -- column, so we add it here. - { column = 'geom', method = 'gist' } -}}) - -function osm2pgsql.process_way(object) - if object.is_closed and object.tags.building then - building_tag:insert({ - geom = object:as_polygon():transform(4326), - building = object.tags.building, -- Example: Assuming 'building' tag - height = object.tags.height -- Example: Assuming 'height' tag - }) - end -end - -function osm2pgsql.process_relation(object) - if object.tags.type == 'multipolygon' and object.tags.building then - -- From the relation we get multipolygons... - local mp = object:as_multipolygon():transform(4326) - -- ...and split them into polygons which we insert into the table - for geom in mp:geometries() do - building_tag:insert({ - geom = geom, - building = object.tags.building, -- Example: Assuming 'building' tag - height = object.tags.height -- Example: Assuming 'height' tag - }) - end - end -end diff --git a/Bash/OSM/ETL/lua/poi.lua b/Bash/OSM/ETL/lua/poi.lua index 43350c4..9c62123 100644 --- a/Bash/OSM/ETL/lua/poi.lua +++ b/Bash/OSM/ETL/lua/poi.lua @@ -1,6 +1,6 @@ -local pois = osm2pgsql.define_table({ - name = 'pois', +local poi = osm2pgsql.define_table({ + name = 'poi', ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' }, columns = { { column = 'name' }, @@ -25,7 +25,7 @@ function process_poi(object, geom) return end - pois:insert(a) + poi:insert(a) end function osm2pgsql.process_node(object) diff --git a/Bash/OSM/ETL/osm_buildings.sh b/Bash/OSM/ETL/osm_buildings.sh deleted file mode 100644 index 74e3c7b..0000000 --- a/Bash/OSM/ETL/osm_buildings.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -psql -U postgres -d $3 -c "drop table if exists public.buildings;" - -# import pbf file to database -/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/buildings.lua -E 4326 $1 - -mkdir ./osm_buildings_${2}/ - -# export table to filqs -pg_dump -U postgres -t public.buildings $3 | gzip > ./osm_buildings_${2}/osm_buildings_${2}.sql.gz - -psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/buildings2tsv.sql -mv ./buildings.tsv ./osm_buildings_${2}/osm_buildings_${2}.tsv -rm ./osm_buildings_${2}/osm_buildings_${2}.tsv.gz -gzip ./osm_buildings_${2}/osm_buildings_${2}.tsv - -# export both tables to shp and compression them -mkdir ./osm_buildings_${2}/osm_buildings_${2}_shp/ -pgsql2shp -u postgres -P postgres -f ./osm_buildings_${2}/osm_buildings_${2}_shp/osm_buildings_${2}.shp $3 public.buildings -zip ./osm_buildings_${2}/osm_buildings_${2}_shp.zip -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ -rm -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ - -psql -U postgres -d $3 -c "drop table if exists public.buildings;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_download.sh b/Bash/OSM/ETL/osm_download.sh new file mode 100755 index 0000000..f767655 --- /dev/null +++ b/Bash/OSM/ETL/osm_download.sh @@ -0,0 +1,28 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$CURRENT_DATE" +mkdir -p "$DATA_DIR" + +### Main ### + +# download +curl -o "$DATA_DIR/japan-latest.osm.pbf" https://download.geofabrik.de/asia/japan-latest.osm.pbf + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_import.sh b/Bash/OSM/ETL/osm_import.sh new file mode 100755 index 0000000..af0ab8f --- /dev/null +++ b/Bash/OSM/ETL/osm_import.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +### Preprocessing ### + +# Initialize the command in seconds. +SECONDS=0 + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, poi]" + exit 1 +fi +if [ -z "$2" ]; then + $2 = $CURRENT_DATE + exit 1 +fi +TYPE=$1 # e.g. [building, poi] +YYYYMMDD=$2 # e.g. [20240729] + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$YYYYMMDD" +if [ ! -d "$DATA_DIR" ] ; then + echo -e "DATA_DIR was empty: $DATA_DIR" + echo -e "Write Type in second argument.\n e.g. [YYYYMMDD]" +fi + +# pbf file +PBF_FILE="$DATA_DIR/japan-latest.osm.pbf" +if [ ! -f "$PBF_FILE" ] ; then + echo -e "There is no pbf file. Dowload at first!" + exit 1 +fi + +# shp file +SHP_FILE="$SCRIPT_DIR/shape/bbox_list.shp" + +### Main ### + +# create database, and drop $TYPE table made by osm2pgsql +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "create schema osm;" +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "drop table if exists osm.$TYPE;" + +# import shape to database +shp2pgsql -I -s 4326 $SHP_FILE osm.shape | psql -d $POSTGRES_DATABASE -U $POSTGRES_USER + +# import pbf file to database +osm2pgsql -d $POSTGRES_DATABASE --schema osm -O flex -S "$SCRIPT_DIR/lua/$TYPE.lua" -E 4326 "$PBF_FILE" + +# create table in each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" + +# insert data about each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/clip_select_insert_$TYPE.sql" + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_pois.sh b/Bash/OSM/ETL/osm_pois.sh deleted file mode 100644 index a47a4bd..0000000 --- a/Bash/OSM/ETL/osm_pois.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -echo $2 -psql -U postgres -d $3 -c "drop table if exists public.pois;" - -# import pbf file to database -/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/pois.lua -E 4326 $1 - -mkdir ./osm_pois_$2/ - -# export table to files -pg_dump -U postgres -t public.pois $3 | gzip > ./osm_pois_$2/osm_pois_$2.sql.gz - -psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/pois2tsv.sql -mv ./pois.tsv ./osm_pois_$2/osm_pois_${2}.tsv -rm ./osm_pois_${2}/osm_pois_${2}.tsv.gz -gzip ./osm_pois_${2}/osm_pois_${2}.tsv - -# export both tables to shp and compression them -mkdir ./osm_pois_${2}/osm_pois_${2}_shp/ -pgsql2shp -u postgres -P postgres -f ./osm_pois_${2}/osm_pois_${2}_shp/osm_pois_${2}.shp $3 public.pois -zip ./osm_pois_${2}/osm_pois_${2}_shp.zip -r ./osm_pois_${2}/osm_pois_${2}_shp/ -rm -r ./osm_pois_${2}/osm_pois_${2}_shp/ - -psql -U postgres -d $3 -c "drop table if exists public.pois;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.cpg b/Bash/OSM/ETL/shape/bbox_list.cpg new file mode 100644 index 0000000..3ad133c --- /dev/null +++ b/Bash/OSM/ETL/shape/bbox_list.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.dbf b/Bash/OSM/ETL/shape/bbox_list.dbf new file mode 100644 index 0000000000000000000000000000000000000000..e6918215d1225d128209d7a0210fb51d5ad1b4d4 GIT binary patch literal 4043 zcmdUyO^e$w5Qakw?Wva@_vB*&u4-vUA9LbU!{uZO?z@863V>vi?39eR=P z%I#gu)y4V`+0F6ku=!p+o$jk$wp=bHN>NyAG-}9zgmDU-fjqkeR)f`2s)k6T{R(M# zNtz^swP3YcWs_d9s60~3<3SRS2T7AA_un6OQ$260$8yYbdUk=rTf{1Z1!bUwRfLWl zc~@KzS4fNCbVIE7jz+6^O`4>m1v0H;VoiChmO+rjAV?Zh+E+jK+i_cNt3x#+(Y{?s zH~@0q2^pXuZHY8UHbqRdC=?MHuni$4oz)qn#?z5HGObm(NU^58r?%0QN18?yb_P># z%fsEN+>Y5^`gXCxfU(WXM_~+scDmj}&WS~7q-udxXbNf^mIUcoflO@}37sakOUQ#i z0qb|xMii4))C^28JjLs6>CJN3ABQjL+eHaOC>>zEyZD2PQoe3nz;lAyp_8(1Q@&|A zRrr$fhLi;9Sb1NuZKiJ*f3T}R zs;0FVerF9%m&0D000D>4wV1^ literal 0 HcmV?d00001 diff --git a/Bash/OSM/ETL/shape/bbox_list.shp b/Bash/OSM/ETL/shape/bbox_list.shp new file mode 100644 index 0000000000000000000000000000000000000000..cf7aca88407fdab88250fea41f259e4707d6af01 GIT binary patch literal 1052 zcmZQzQ0HR63Yhp_Ff%Z)0_DUOT3Wc4$2o*D8OJGg896Nae=_R(_9TafGiirwlH47z zYDX1h1acifW)-}MQmt8(=-_-&H9z9Ivjd3lvoz6xfqUJnst3*vAcY`xAb?FRNH0t+ zx?UI`*=%IHfJQO_G1wi;qg@35g(f(d=Nx(yAM4-%;-8C1a9Hhdt80Fm1ByGasRikU zsYTZd<0G4mY!}cCz>tNw14uJ$s7iDY{;V*4ewCvGh~H3~=#c#T`0H!UjwtQ`$)T$S z@nLGw^}_hbW~1q60jdFq2S{y#Nuq=Igrcssp%8Z%m?t{yPwCQl8;9u*bhRKpOfASP zWW6vxve{_*K~WBN2j51Q17_0_95QVmKFK$9bO7<$W+ph?75v$>-U`(n*wljb!qlSc zh4GQiMz#y+1~!;GKyH3ep6HCgIV}2rGV1&GB!`AGX@_f)+#RrM dM-epw@<8g!p!6;%{SQJjIzj0+D18b<0|2(<6q^75 literal 0 HcmV?d00001 diff --git a/Bash/OSM/ETL/sql/buildings2tsv.sql b/Bash/OSM/ETL/sql/buildings2tsv.sql deleted file mode 100644 index 87fff60..0000000 --- a/Bash/OSM/ETL/sql/buildings2tsv.sql +++ /dev/null @@ -1,12 +0,0 @@ -copy ( - select - area_id, - id, - ST_asText(geom) - from - public.buildings - where - area_id is not null and geom is not null - order by - area_id -) to STDOUT with csv header delimiter E'\t' \g 'buildings.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/clip_select_insert_building.sql b/Bash/OSM/ETL/sql/clip_select_insert_building.sql new file mode 100755 index 0000000..3bede6c --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_building.sql @@ -0,0 +1,143 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.building_tokyo (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.building_tateyama (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.building_hamamatsu (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.building_higashi_hiroshima (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.building_kumamoto (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.building_morioka (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/clip_select_insert_poi.sql b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql new file mode 100755 index 0000000..2c8e408 --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql @@ -0,0 +1,149 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.poi_tokyo (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.poi_tateyama (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.poi_hamamatsu (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.poi_higashi_hiroshima (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.poi_kumamoto (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.poi_morioka (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_building.sql b/Bash/OSM/ETL/sql/create_tables_building.sql new file mode 100644 index 0000000..946fdf3 --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_building.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.building_tokyo; + +drop table if exists osm.building_hamamatsu; + +drop table if exists osm.building_tateyama; + +drop table if exists osm.building_kumamoto; + +drop table if exists osm.building_higashi_hiroshima; + +drop table if exists osm.building_morioka; + +-- create table +create table + osm.building_tokyo (like osm.building including all); + +create table + osm.building_hamamatsu (like osm.building including all); + +create table + osm.building_tateyama (like osm.building including all); + +create table + osm.building_kumamoto (like osm.building including all); + +create table + osm.building_higashi_hiroshima (like osm.building including all); + +create table + osm.building_morioka (like osm.building including all); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_poi.sql b/Bash/OSM/ETL/sql/create_tables_poi.sql new file mode 100644 index 0000000..cea7c1b --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_poi.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.poi_tokyo; + +drop table if exists osm.poi_hamamatsu; + +drop table if exists osm.poi_tateyama; + +drop table if exists osm.poi_kumamoto; + +drop table if exists osm.poi_higashi_hiroshima; + +drop table if exists osm.poi_morioka; + +-- create table +create table + osm.poi_tokyo (like osm.poi including all); + +create table + osm.poi_hamamatsu (like osm.poi including all); + +create table + osm.poi_tateyama (like osm.poi including all); + +create table + osm.poi_kumamoto (like osm.poi including all); + +create table + osm.poi_higashi_hiroshima (like osm.poi including all); + +create table + osm.poi_morioka (like osm.poi including all); \ No newline at end of file diff --git a/Bash/OSM/road/network_assessment.sql b/Bash/OSM/road/network_assessment.sql deleted file mode 100644 index ebef3ac..0000000 --- a/Bash/OSM/road/network_assessment.sql +++ /dev/null @@ -1,14 +0,0 @@ -alter table public.osm_road_available add column groupno integer; - -update public.osm_road_available -set groupno = val.groupno -from ( - select - oras.groupno as groupno, - orav.id as id - from - public.osm_road_assessment oras, - public.osm_road_available orav - where ST_Intersects(orav.geom_way, oras.geom) -) as val where public.osm_road_available.id = val.id ; - diff --git a/Bash/OSM/road/osm_road.sh b/Bash/OSM/road/osm_road.sh deleted file mode 100644 index 7433f1e..0000000 --- a/Bash/OSM/road/osm_road.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -# remove output files generated previpusly -rm -r ./osm_road - -# convert osm.pbf to sql.gz by using osm2po -sh ${0%/*}/osm2road/conv_osm_road.sh $1 - -psql -d postgres -c "drop table if exists public.osm_road_available;" -psql -d postgres -c "drop table if exists public.osm_road_assessment;" - -# import the files into tables -gzip -cd ./osm_road/osm_road_available.sql.gz | psql -d postgres -gzip -cd ./osm_road/osm_road_assessment.sql.gz | psql -d postgres - -# assess the network -psql -v ON_ERROR_STOP=1 -d postgres -f ${0%/*}/network_assessment.sql - -# export the network to files -psql -d postgres -f ${0%/*}/road2tsv.sql -mv ./osm_road/road.tsv ./osm_road/osm_road_$2.tsv -pg_dump -d postgres -t osm_road_available | gzip > ./osm_road/osm_road_$2.sql.gz - -mkdir ./osm_road/osm_road_${2}_shp/ -pgsql2shp -P postgreses -f ./osm_road/osm_road_${2}_shp/osm_road_${2}.shp miyake public.road -zip ./osm_road/osm_road_${2}_shp.zip -r ./osm_road/osm_road_${2}_shp/ -rm -r ./osm_road/osm_road_${2}_shp/ - -mv ./osm_road/ ./osm_road_${2}/ - -psql -d postgres -c "drop table if exists public.osm_road_available;" -psql -d postgres -c "drop table if exists public.osm_road_assessment;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" diff --git a/Bash/OSM/road/road2tsv.sql b/Bash/OSM/road/road2tsv.sql deleted file mode 100644 index fa8237c..0000000 --- a/Bash/OSM/road/road2tsv.sql +++ /dev/null @@ -1,29 +0,0 @@ -copy ( - select - groupno, - osm_id, - id, - osm_source_id, - osm_target_id, - clazz, - flags, - source, - target, - km, - kmh, - cost, - reverse_cost, - x1, - y1, - x2, - y2, - osm_name, - osm_meta, - ST_asText(geom_way) - from - public.osm_road_available - where - osm_id is not null and geom_way is not null - order by - osm_id -) to STDOUT with csv header delimiter E'\t' \g 'road.tsv' \ No newline at end of file diff --git a/Bash/readme.md b/Bash/readme.md index 6401375..541cef3 100644 --- a/Bash/readme.md +++ b/Bash/readme.md @@ -1,6 +1,6 @@ # Buildings -## Overture Maps +## Overture Maps Foundation (OMF) - Install a CLI tool or overturemaps-py repo to download Overture Maps data. @@ -60,11 +60,28 @@ $ aws s3 ls s3://overturemaps-us-west-2/release/2024-07-22.0/ --region us-west-2 PRE theme=transportation/ ``` -## OpenStreetMap +## OpenStreetMap (OSM) -This script just import tsv files into PostgreSQL. -You need to extract data before running this script. +- Install a CLI tool ```sh -bash building/import_osm_tsv.sh +# osm2pgsql: tool for importing OSM pbf data into postgres +## [Other OS](https://osm2pgsql.org/doc/install.html) +## Mac +$ brew install osm2pgsql +``` + +- Download data from osm as a pbf format. + +```sh +bash OSM/ETL/osm_download.sh +``` + +- Then, import that data to postgres + +```sh +# building +bash OSM/ETL/osm_import.sh building YYYYMMDD +# poi +bash OSM/ETL/osm_import.sh poi YYYYMMDD ```