diff --git a/.gitignore b/.gitignore index 14a6b30..17d56ca 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,9 @@ __pycache__/ cache/ .streamlit/ tiles -other.py \ No newline at end of file +other.py + +# ignore data dir +Bash/**/data/** +Bash/**/tsv/** +Bash/**/overturemaps-py/** \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..421ed4c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,19 @@ +{ + "editor.formatOnSave": true, + "editor.tabSize": 2, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "python.testing.unittestArgs": ["-v", "-s", ".", "-p", "test_*.py"], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true, + "black-formatter.args": ["--line-length", "120"], + "flake8.args": ["--max-line-length", "120"], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "[js]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[sql]": { + "editor.defaultFormatter": "inferrinizzard.prettier-sql-vscode" + } +} diff --git a/Bash/OMF/Analyze/copy_place_category_main_to_tsv.sh b/Bash/OMF/Analyze/copy_place_category_main_to_tsv.sh new file mode 100644 index 0000000..52991d4 --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_category_main_to_tsv.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "select + count(categories->>'main') as category_cnt + from omf.${city}_place + where categories->>'main' is not null" + psql -d postgres -c "\copy ( + select + distinct categories->>'main' as main_category, + count(*) as category_cnt + from omf.${city}_place + where categories->>'main' is not null + group by categories->>'main' + order by category_cnt desc + ) to './tsv/omf_place_category_main_$city.tsv' + with csv delimiter E'\t';" +done + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh new file mode 100644 index 0000000..0fc1515 --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_cnt_to_tsv.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +output="./tsv/omf_place_cnt.tsv" +rm $output + +# export header +psql -d postgres -c "\copy ( + select + '$city' as city, + count(ogc_fid) as ogc_fid_cnt, + count(wkb_geometry) as wkb_geometry_cnt, + count(id) as id_cnt, + count(version) as version_cnt, + count(update_time) as update_time_cnt, + count(sources) as sources_cnt, + count(names) as names_cnt, + count(categories) as categories_cnt, + count(confidence) as confidence_cnt, + count(websites) as websites_cnt, + count(emails) as emails_cnt, + count(socials) as socials_cnt, + count(phones) as phones_cnt, + count(addresses) as addresses_cnt, + count(brand) as brand_cnt + from + omf.${city}_place + where id is null + ) to 'omf.tmp' + with csv header delimiter E'\t';" +cat omf.tmp >> "$output" + +# export data +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + '$city' as city, + count(ogc_fid) as ogc_fid_cnt, + count(wkb_geometry) as wkb_geometry_cnt, + count(id) as id_cnt, + count(version) as version_cnt, + count(update_time) as update_time_cnt, + count(sources) as sources_cnt, + count(names) as names_cnt, + count(categories) as categories_cnt, + count(confidence) as confidence_cnt, + count(websites) as websites_cnt, + count(emails) as emails_cnt, + count(socials) as socials_cnt, + count(phones) as phones_cnt, + count(addresses) as addresses_cnt, + count(brand) as brand_cnt + from + omf.${city}_place + ) to 'omf.tmp' + with csv delimiter E'\t';" + cat omf.tmp >> "$output" +done + +# rm tmp +rm omf.tmp + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_source_to_tsv.sh b/Bash/OMF/Analyze/copy_place_source_to_tsv.sh new file mode 100644 index 0000000..87072ec --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_source_to_tsv.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +output="./tsv/omf_source_in_place.tsv" +rm $output + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + dataset, cnt + from ( + select 'total' as dataset, count(*) as cnt from omf.${city}_place + union + select distinct source->>'dataset' as dataset, count(*) as cnt + from omf.${city}_place, + jsonb_array_elements(sources) as source + where source->>'dataset' is not null + group by source->>'dataset' + ) a + order by dataset desc + ) to 'omf_source_in_place.tmp' + with csv delimiter E'\t';" + echo "city $city" >> "$output" + cat omf_source_in_place.tmp >> "$output" +done + +# rm tmp +rm omf_source_in_place.tmp + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/Analyze/copy_place_to_tsv.sh b/Bash/OMF/Analyze/copy_place_to_tsv.sh new file mode 100644 index 0000000..6c7813b --- /dev/null +++ b/Bash/OMF/Analyze/copy_place_to_tsv.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# make tsv dir +mkdir -p "${SCRIPT_DIR}/tsv" + +# analyze +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") + +for city in "${cities[@]}"; do + echo $city + psql -d postgres -c "\copy ( + select + ogc_fid, + ST_AsText(wkb_geometry) as wkb_geometry, + id, + version, + update_time, + sources::text, + names::text, + categories::text, + confidence, + websites, + emails, + socials, + phones, + addresses::text, + brand::text + from + omf.${city}_place + ) to './tsv/omf_place_$city.tsv' + with csv delimiter E'\t';" +done + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/omf_download.sh b/Bash/OMF/ETL/omf_download.sh new file mode 100644 index 0000000..6ad5faa --- /dev/null +++ b/Bash/OMF/ETL/omf_download.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +### Preprocessing ### + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, place]" + exit 1 +fi +if [ -z "$2" ]; then + echo -e "Write Release Version in second argument.\nYou could check list of versions:" + aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + exit 1 +fi +TYPE=$1 # e.g. [building, place] +RELEASE_VERSION=$2 # e.g. [latest, 2024-07-22.0, 2024-06-13-beta.1] + +# create data directory +DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" +mkdir -p $DATA_DIR + +# change resource in "git@github.com:OvertureMaps/overturemaps-py.git" to change +cd "../overturemaps-py" +git checkout . +git pull origin main +before="overturemaps-us-west-2/release/.*/theme" +after="overturemaps-us-west-2/release/$RELEASE_VERSION/theme" +sed -i '' "s|$before|$after|g" "../overturemaps-py/overturemaps/core.py" +cd - + +function omf_download(){ + if [ "$RELEASE_VERSION" = "latest" ]; then + overturemaps download $@ + else + poetry run overturemaps download $@ + fi +} + +### Main ### + +# Tokyo +omf_download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=$TYPE -o "$DATA_DIR/tokyo_$TYPE.geojson" + +# Hamamatsu +omf_download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=$TYPE -o "$DATA_DIR/hamamatsu_$TYPE.geojson" + +# Tateyama +omf_download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=$TYPE -o "$DATA_DIR/tateyama_$TYPE.geojson" + +# Kumamoto +omf_download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=$TYPE -o "$DATA_DIR/kumamoto_$TYPE.geojson" + +# Higashi_hiroshima +omf_download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=$TYPE -o "$DATA_DIR/higashi_hiroshima_$TYPE.geojson" + +# Morioka +omf_download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=$TYPE -o "$DATA_DIR/morioka_$TYPE.geojson" + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/omf_import.sh b/Bash/OMF/ETL/omf_import.sh new file mode 100644 index 0000000..2f7601d --- /dev/null +++ b/Bash/OMF/ETL/omf_import.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +### Preprocessing ### + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, place]" + exit 1 +fi +if [ -z "$2" ]; then + echo -e "Write Release Version in second argument.\nYou could check list of versions:" + aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + exit 1 +fi +TYPE=$1 # e.g. [building, place] +RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1] + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/" + +### Main ### + +echo "Change psql user(-U) or database(-d) if threre is error about postgres" + +# create schema +psql -U postgres -d postgres -c "create schema omf;" + +# create table +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" + +# Tokyo +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_$TYPE -nlt multipolygon $DATA_DIR/tokyo_$TYPE.geojson + +# Hamamatsu +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.hamamatsu_$TYPE -nlt multipolygon $DATA_DIR/hamamatsu_$TYPE.geojson + +# Tateyama +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tateyama_$TYPE -nlt multipolygon $DATA_DIR/tateyama_$TYPE.geojson + +# Kumamoto +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.kumamoto_$TYPE -nlt multipolygon $DATA_DIR/kumamoto_$TYPE.geojson + +# Higashi_hiroshima +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.higashi_hiroshima_$TYPE -nlt multipolygon $DATA_DIR/higashi_hiroshima_$TYPE.geojson + +# Morioka +ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_$TYPE -nlt multipolygon $DATA_DIR/morioka_$TYPE.geojson + + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" + +exit 0 \ No newline at end of file diff --git a/Bash/OMF/ETL/sql/create_tables_building.sql b/Bash/OMF/ETL/sql/create_tables_building.sql new file mode 100644 index 0000000..9de1e6b --- /dev/null +++ b/Bash/OMF/ETL/sql/create_tables_building.sql @@ -0,0 +1,54 @@ +-- drop table +drop table if exists omf.tokyo_building; +drop table if exists omf.hamamatsu_building; +drop table if exists omf.tateyama_building; +drop table if exists omf.kumamoto_building; +drop table if exists omf.higashi_hiroshima_building; +drop table if exists omf.morioka_building; +drop table if exists omf.building; + +-- create table +create table omf.building ( + ogc_fid integer, + wkb_geometry geometry(MULTIPOLYGON, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources json, + subtype varchar, + names json, + class varchar, + level integer, + has_parts boolean, + height double precision, + min_height double precision, + num_floors integer, + facade_color varchar, + facade_material varchar, + min_floor integer, + roof_material varchar, + roof_shape varchar, + roof_direction double precision, + roof_orientation varchar, + roof_color varchar, + area_name char(50) +) +; + +create table + omf.tokyo_building (like omf.building including all); + +create table + omf.hamamatsu_building (like omf.building including all); + +create table + omf.tateyama_building (like omf.building including all); + +create table + omf.kumamoto_building (like omf.building including all); + +create table + omf.higashi_hiroshima_building (like omf.building including all); + +create table + omf.morioka_building (like omf.building including all); diff --git a/Bash/OMF/ETL/sql/create_tables_place.sql b/Bash/OMF/ETL/sql/create_tables_place.sql new file mode 100644 index 0000000..8a96e79 --- /dev/null +++ b/Bash/OMF/ETL/sql/create_tables_place.sql @@ -0,0 +1,46 @@ +-- drop table +drop table if exists omf.tokyo_place; +drop table if exists omf.hamamatsu_place; +drop table if exists omf.tateyama_place; +drop table if exists omf.kumamoto_place; +drop table if exists omf.higashi_hiroshima_place; +drop table if exists omf.morioka_place; +drop table if exists omf.place; + +-- create table +create table + omf.place ( + ogc_fid serial primary key, + wkb_geometry geometry (point, 4326), + id varchar, + version integer, + update_time timestamp with time zone, + sources jsonb, + names jsonb, + categories jsonb, + confidence double precision, + websites text, + emails text, + socials text, + phones text, + addresses jsonb, + brand jsonb + ); + +create table + omf.tokyo_place (like omf.place including all); + +create table + omf.hamamatsu_place (like omf.place including all); + +create table + omf.tateyama_place (like omf.place including all); + +create table + omf.kumamoto_place (like omf.place including all); + +create table + omf.higashi_hiroshima_place (like omf.place including all); + +create table + omf.morioka_place (like omf.place including all); \ No newline at end of file diff --git a/Bash/OMF/overturemaps-py b/Bash/OMF/overturemaps-py new file mode 160000 index 0000000..5148bf4 --- /dev/null +++ b/Bash/OMF/overturemaps-py @@ -0,0 +1 @@ +Subproject commit 5148bf40fa01d125095c29f70c4c91db17e4dfdd diff --git a/Bash/OMF/requirements.txt b/Bash/OMF/requirements.txt new file mode 100644 index 0000000..81701b4 --- /dev/null +++ b/Bash/OMF/requirements.txt @@ -0,0 +1,2 @@ +poetry +overturemaps \ No newline at end of file diff --git a/Bash/OMF/setup.sh b/Bash/OMF/setup.sh new file mode 100644 index 0000000..1cdfbb4 --- /dev/null +++ b/Bash/OMF/setup.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# pip install +pip install -r "$SCRIPT_DIR/requirements.txt" + +# get latest resource +OMF_DIR="$SCRIPT_DIR/overturemaps-py" +if [ -d "$OMF_DIR" ]; then + cd "$OMF_DIR" + git checkout . + git pull origin main + cd - +else + git clone git@github.com:OvertureMaps/overturemaps-py.git +fi + +exit 0 \ No newline at end of file diff --git a/Bash/OSM/Analyze/config.sh b/Bash/OSM/Analyze/config.sh new file mode 100644 index 0000000..b1d3423 --- /dev/null +++ b/Bash/OSM/Analyze/config.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# define cities +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hmaamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") ## becase of shape file 'Hmaamatsu' + diff --git a/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh new file mode 100644 index 0000000..0477e14 --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + building, + count(*) as cnt + from osm.building_$city + group by + building + order by + cnt desc + ) to '$SCRIPT_DIR/tsv/osm_building_cnt_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh new file mode 100644 index 0000000..99707ee --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +sql="\copy ( + select + distinct building + from osm.building + order by building + ) to '$SCRIPT_DIR/tsv/osm_building_distinct_tag.tsv' + with csv delimiter E'\t';" +psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" diff --git a/Bash/OSM/Analyze/copy_building_to_tsv.sh b/Bash/OSM/Analyze/copy_building_to_tsv.sh new file mode 100644 index 0000000..fc08ef1 --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_to_tsv.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + area_id, + id, + ST_AsText(geom), + building, + height + from osm.building_$city + ) to '$SCRIPT_DIR/tsv/osm_building_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh new file mode 100644 index 0000000..86c6822 --- /dev/null +++ b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + class, subclass, count(*) as subclass_cnt + from osm.poi_$city + group by class, subclass + order by class, subclass_cnt desc + ) to '$SCRIPT_DIR/tsv/osm_poi_cnt_$city.tsv' + with csv delimiter E'\t'; + " + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_poi_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_to_tsv.sh new file mode 100644 index 0000000..c1f7105 --- /dev/null +++ b/Bash/OSM/Analyze/copy_poi_to_tsv.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + osm_type , + osm_id , + name , + class , + subclass , + ST_AsText(geom) + from osm.poi_$city + ) to '$SCRIPT_DIR/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/count_building_height.sh b/Bash/OSM/Analyze/count_building_height.sh new file mode 100644 index 0000000..682dd4a --- /dev/null +++ b/Bash/OSM/Analyze/count_building_height.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +cnt=0 +for city in "${cities[@]}"; do + echo $city + sql=" + select + count(height) as cnt_height + from osm.building_$city + " + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/count_poi_amenity_shop.sh b/Bash/OSM/Analyze/count_poi_amenity_shop.sh new file mode 100644 index 0000000..52a4af0 --- /dev/null +++ b/Bash/OSM/Analyze/count_poi_amenity_shop.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +cnt=0 +for city in "${cities[@]}"; do + echo $city + sql=" + select + name, + cnt + from ( + select + 'poi_cnt' as name, + count(*) as cnt + from osm.poi_$city + + union + + select + 'poi_amenity_cnt' as name, + count(*) as cnt + from osm.poi_$city + where class='amenity' + + union + + select + 'poi_shop_cnt' as name, + count(*) as cnt + from osm.poi_$city + where class='shop' + ) a + order by name + " + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/coverage.sh b/Bash/OSM/Analyze/coverage.sh new file mode 100644 index 0000000..d958268 --- /dev/null +++ b/Bash/OSM/Analyze/coverage.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +cnt=0 +for city in "${cities[@]}"; do + echo $city + + sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from osm.building_$city" + building_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") + echo $building_area + + sql="select ST_Area(geom::geography) / 1000000 from osm.shape where name='${city_labels[$cnt]}'" + whole_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") + echo $whole_area + + echo "scale=5; $building_area / $whole_area * 100" | bc + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/sql/building2tsv.sql b/Bash/OSM/Analyze/sql/building2tsv.sql new file mode 100644 index 0000000..5adcdf7 --- /dev/null +++ b/Bash/OSM/Analyze/sql/building2tsv.sql @@ -0,0 +1,16 @@ +copy ( + select + area_id, + id, + ST_asText (geom), + building, + height + from + osm.building + where + area_id is not null + and geom is not null + order by + area_id +) +to STDOUT with csv header delimiter E'\t' \g '../tsv/building.tsv' \ No newline at end of file diff --git a/Bash/OSM/Analyze/sql/poi2tsv.sql b/Bash/OSM/Analyze/sql/poi2tsv.sql new file mode 100644 index 0000000..f38b3e2 --- /dev/null +++ b/Bash/OSM/Analyze/sql/poi2tsv.sql @@ -0,0 +1,15 @@ +copy ( + select + osm_id, + osm_type, + name, + class, + subclass, + ST_asText(geom) + from + osm.poi + where + osm_id is not null and geom is not null + order by + osm_id +) to STDOUT with csv header delimiter E'\t' \g '../tsv/poi.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/lua/building.lua b/Bash/OSM/ETL/lua/building.lua new file mode 100644 index 0000000..3b35e73 --- /dev/null +++ b/Bash/OSM/ETL/lua/building.lua @@ -0,0 +1,39 @@ +local building = osm2pgsql.define_area_table('building', { + -- Define an autoincrementing id column, QGIS likes a unique id on the table + { column = 'id', sql_type = 'serial', create_only = true }, + { column = 'geom', type = 'multipolygon', not_null = true, projection = 4326}, + -- Add columns for tags + { column = 'building', type = 'text' }, + { column = 'height', type = 'text' } +}, { indexes = { + -- So we get an index on the id column + { column = 'id', method = 'btree', unique = true }, + -- If we define any indexes we don't get the default index on the geometry + -- column, so we add it here. + { column = 'geom', method = 'gist' } +}}) + +function osm2pgsql.process_way(object) + if object.is_closed and object.tags.building then + building:insert({ + geom = object:as_polygon():transform(4326), + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag + }) + end +end + +function osm2pgsql.process_relation(object) + if object.tags.type == 'multipolygon' and object.tags.building then + -- From the relation we get multipolygons... + local mp = object:as_multipolygon():transform(4326) + -- ...and split them into polygons which we insert into the table + for geom in mp:geometries() do + building:insert({ + geom = geom, + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag + }) + end + end +end diff --git a/Bash/OSM/ETL/lua/poi.lua b/Bash/OSM/ETL/lua/poi.lua new file mode 100644 index 0000000..9c62123 --- /dev/null +++ b/Bash/OSM/ETL/lua/poi.lua @@ -0,0 +1,40 @@ + +local poi = osm2pgsql.define_table({ + name = 'poi', + ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' }, + columns = { + { column = 'name' }, + { column = 'class', not_null = true }, + { column = 'subclass' }, + { column = 'geom', type = 'point', not_null = true, projection = 4326}, +}}) + +function process_poi(object, geom) + local a = { + name = object.tags.name, + geom = geom + } + + if object.tags.amenity then + a.class = 'amenity' + a.subclass = object.tags.amenity + elseif object.tags.shop then + a.class = 'shop' + a.subclass = object.tags.shop + else + return + end + + poi:insert(a) +end + +function osm2pgsql.process_node(object) + process_poi(object, object:as_point():transform(4326)) +end + +function osm2pgsql.process_way(object) + if object.is_closed and object.tags.building then + process_poi(object, object:as_polygon():centroid()) + end +end + diff --git a/Bash/OSM/ETL/osm_download.sh b/Bash/OSM/ETL/osm_download.sh new file mode 100755 index 0000000..f767655 --- /dev/null +++ b/Bash/OSM/ETL/osm_download.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$CURRENT_DATE" +mkdir -p "$DATA_DIR" + +### Main ### + +# download +curl -o "$DATA_DIR/japan-latest.osm.pbf" https://download.geofabrik.de/asia/japan-latest.osm.pbf + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_import.sh b/Bash/OSM/ETL/osm_import.sh new file mode 100755 index 0000000..af0ab8f --- /dev/null +++ b/Bash/OSM/ETL/osm_import.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +### Preprocessing ### + +# Initialize the command in seconds. +SECONDS=0 + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, poi]" + exit 1 +fi +if [ -z "$2" ]; then + $2 = $CURRENT_DATE + exit 1 +fi +TYPE=$1 # e.g. [building, poi] +YYYYMMDD=$2 # e.g. [20240729] + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$YYYYMMDD" +if [ ! -d "$DATA_DIR" ] ; then + echo -e "DATA_DIR was empty: $DATA_DIR" + echo -e "Write Type in second argument.\n e.g. [YYYYMMDD]" +fi + +# pbf file +PBF_FILE="$DATA_DIR/japan-latest.osm.pbf" +if [ ! -f "$PBF_FILE" ] ; then + echo -e "There is no pbf file. Dowload at first!" + exit 1 +fi + +# shp file +SHP_FILE="$SCRIPT_DIR/shape/bbox_list.shp" + +### Main ### + +# create database, and drop $TYPE table made by osm2pgsql +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "create schema osm;" +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "drop table if exists osm.$TYPE;" + +# import shape to database +shp2pgsql -I -s 4326 $SHP_FILE osm.shape | psql -d $POSTGRES_DATABASE -U $POSTGRES_USER + +# import pbf file to database +osm2pgsql -d $POSTGRES_DATABASE --schema osm -O flex -S "$SCRIPT_DIR/lua/$TYPE.lua" -E 4326 "$PBF_FILE" + +# create table in each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" + +# insert data about each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/clip_select_insert_$TYPE.sql" + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.cpg b/Bash/OSM/ETL/shape/bbox_list.cpg new file mode 100644 index 0000000..3ad133c --- /dev/null +++ b/Bash/OSM/ETL/shape/bbox_list.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.dbf b/Bash/OSM/ETL/shape/bbox_list.dbf new file mode 100644 index 0000000..e691821 Binary files /dev/null and b/Bash/OSM/ETL/shape/bbox_list.dbf differ diff --git a/Bash/OSM/ETL/shape/bbox_list.prj b/Bash/OSM/ETL/shape/bbox_list.prj new file mode 100644 index 0000000..6477a3f --- /dev/null +++ b/Bash/OSM/ETL/shape/bbox_list.prj @@ -0,0 +1 @@ +GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.qix b/Bash/OSM/ETL/shape/bbox_list.qix new file mode 100644 index 0000000..046ac5c Binary files /dev/null and b/Bash/OSM/ETL/shape/bbox_list.qix differ diff --git a/Bash/OSM/ETL/shape/bbox_list.shp b/Bash/OSM/ETL/shape/bbox_list.shp new file mode 100644 index 0000000..cf7aca8 Binary files /dev/null and b/Bash/OSM/ETL/shape/bbox_list.shp differ diff --git a/Bash/OSM/ETL/shape/bbox_list.shx b/Bash/OSM/ETL/shape/bbox_list.shx new file mode 100644 index 0000000..7818c7f Binary files /dev/null and b/Bash/OSM/ETL/shape/bbox_list.shx differ diff --git a/Bash/OSM/ETL/sql/clip_select_insert_building.sql b/Bash/OSM/ETL/sql/clip_select_insert_building.sql new file mode 100755 index 0000000..3bede6c --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_building.sql @@ -0,0 +1,143 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.building_tokyo (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.building_tateyama (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.building_hamamatsu (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.building_higashi_hiroshima (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.building_kumamoto (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.building_morioka (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/clip_select_insert_poi.sql b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql new file mode 100755 index 0000000..2c8e408 --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql @@ -0,0 +1,149 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.poi_tokyo (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.poi_tateyama (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.poi_hamamatsu (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.poi_higashi_hiroshima (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.poi_kumamoto (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.poi_morioka (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_building.sql b/Bash/OSM/ETL/sql/create_tables_building.sql new file mode 100644 index 0000000..946fdf3 --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_building.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.building_tokyo; + +drop table if exists osm.building_hamamatsu; + +drop table if exists osm.building_tateyama; + +drop table if exists osm.building_kumamoto; + +drop table if exists osm.building_higashi_hiroshima; + +drop table if exists osm.building_morioka; + +-- create table +create table + osm.building_tokyo (like osm.building including all); + +create table + osm.building_hamamatsu (like osm.building including all); + +create table + osm.building_tateyama (like osm.building including all); + +create table + osm.building_kumamoto (like osm.building including all); + +create table + osm.building_higashi_hiroshima (like osm.building including all); + +create table + osm.building_morioka (like osm.building including all); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_poi.sql b/Bash/OSM/ETL/sql/create_tables_poi.sql new file mode 100644 index 0000000..cea7c1b --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_poi.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.poi_tokyo; + +drop table if exists osm.poi_hamamatsu; + +drop table if exists osm.poi_tateyama; + +drop table if exists osm.poi_kumamoto; + +drop table if exists osm.poi_higashi_hiroshima; + +drop table if exists osm.poi_morioka; + +-- create table +create table + osm.poi_tokyo (like osm.poi including all); + +create table + osm.poi_hamamatsu (like osm.poi including all); + +create table + osm.poi_tateyama (like osm.poi including all); + +create table + osm.poi_kumamoto (like osm.poi including all); + +create table + osm.poi_higashi_hiroshima (like osm.poi including all); + +create table + osm.poi_morioka (like osm.poi including all); \ No newline at end of file diff --git a/Bash/readme.md b/Bash/readme.md new file mode 100644 index 0000000..541cef3 --- /dev/null +++ b/Bash/readme.md @@ -0,0 +1,87 @@ +# Buildings + +## Overture Maps Foundation (OMF) + +- Install a CLI tool or overturemaps-py repo to download Overture Maps data. + +```sh +bash OMF/setup.sh +``` + +- Run the following code and it will download building in target areas from Overture Maps and import them into PostgreSQL + +```sh +# get latest +## address +bash OMF/ETL/omf_download.sh address latest +bash OMF/ETL/omf_import.sh address latest +## building +bash OMF/ETL/omf_download.sh building latest +bash OMF/ETL/omf_import.sh building latest +## place +bash OMF/ETL/omf_download.sh place latest +bash OMF/ETL/omf_import.sh place latest + +# get target version +## address +bash OMF/ETL/omf_download.sh address 2024-07-22.0 +bash OMF/ETL/omf_import.sh address 2024-07-22.0 +... +``` + +- Check all versions available + +```sh +$ aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request + PRE 2023-04-02-alpha/ + PRE 2023-07-26-alpha.0/ + PRE 2023-10-19-alpha.0/ + PRE 2023-11-14-alpha.0/ + PRE 2023-12-14-alpha.0/ + PRE 2024-01-17-alpha.0/ + PRE 2024-02-15-alpha.0/ + PRE 2024-03-12-alpha.0/ + PRE 2024-04-16-beta.0/ + PRE 2024-05-16-beta.0/ + PRE 2024-06-13-beta.0/ + PRE 2024-06-13-beta.1/ + PRE 2024-07-22.0/ +``` + +- Check all themes available + +```sh +$ aws s3 ls s3://overturemaps-us-west-2/release/2024-07-22.0/ --region us-west-2 --no-sign-request + PRE theme=addresses/ + PRE theme=base/ + PRE theme=buildings/ + PRE theme=divisions/ + PRE theme=places/ + PRE theme=transportation/ +``` + +## OpenStreetMap (OSM) + +- Install a CLI tool + +```sh +# osm2pgsql: tool for importing OSM pbf data into postgres +## [Other OS](https://osm2pgsql.org/doc/install.html) +## Mac +$ brew install osm2pgsql +``` + +- Download data from osm as a pbf format. + +```sh +bash OSM/ETL/osm_download.sh +``` + +- Then, import that data to postgres + +```sh +# building +bash OSM/ETL/osm_import.sh building YYYYMMDD +# poi +bash OSM/ETL/osm_import.sh poi YYYYMMDD +```