From eb2754527b9c0c05dc86e8505963378db5eaaadc Mon Sep 17 00:00:00 2001 From: ke_shimazaki Date: Tue, 30 Jul 2024 13:53:08 +0900 Subject: [PATCH] fix osm script --- Bash/OMF/ETL/omf_import.sh | 2 +- .../ETL/{ => sql}/create_tables_building.sql | 0 .../OMF/ETL/{ => sql}/create_tables_place.sql | 0 Bash/OMF/overturemaps-py | 1 + Bash/OSM/Analyze/config.sh | 10 ++ .../Analyze/copy_building_tags_cnt_to_tsv.sh | 24 +++ Bash/OSM/Analyze/copy_building_tags_to_tsv.sh | 17 ++ Bash/OSM/Analyze/copy_building_to_tsv.sh | 20 ++- Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh | 17 +- Bash/OSM/Analyze/copy_poi_to_tsv.sh | 13 +- Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh | 9 -- Bash/OSM/Analyze/copy_tags_to_tsv.sh | 4 - Bash/OSM/Analyze/count_building_height.sh | 21 +++ Bash/OSM/Analyze/count_height.sh | 19 --- Bash/OSM/Analyze/count_poi_amenity_shop.sh | 28 ++-- Bash/OSM/Analyze/coverage.sh | 17 +- Bash/OSM/Analyze/sql/building2tsv.sql | 16 ++ .../Analyze/sql/copy_building_from_tsv.sql | 2 - .../OSM/Analyze/sql/create_table_building.sql | 42 ----- .../Analyze/sql/create_table_building_tag.sql | 48 ------ Bash/OSM/Analyze/sql/create_table_poi.sql | 13 -- .../pois2tsv.sql => Analyze/sql/poi2tsv.sql} | 4 +- Bash/OSM/ETL/clip_select_insert_building.sh | 33 ---- .../ETL/clip_select_insert_building_tag.sh | 37 ----- Bash/OSM/ETL/clip_select_insert_poi.sh | 39 ----- Bash/OSM/ETL/lua/building.lua | 19 ++- Bash/OSM/ETL/lua/building_with_tag.lua | 39 ----- Bash/OSM/ETL/lua/poi.lua | 6 +- Bash/OSM/ETL/osm_buildings.sh | 33 ---- Bash/OSM/ETL/osm_download.sh | 28 ++++ Bash/OSM/ETL/osm_import.sh | 72 +++++++++ Bash/OSM/ETL/osm_pois.sh | 34 ---- Bash/OSM/ETL/shape/bbox_list.cpg | 1 + Bash/OSM/ETL/shape/bbox_list.dbf | Bin 0 -> 4043 bytes Bash/OSM/ETL/shape/bbox_list.prj | 1 + Bash/OSM/ETL/shape/bbox_list.qix | Bin 0 -> 88 bytes Bash/OSM/ETL/shape/bbox_list.shp | Bin 0 -> 1052 bytes Bash/OSM/ETL/shape/bbox_list.shx | Bin 0 -> 156 bytes Bash/OSM/ETL/sql/buildings2tsv.sql | 12 -- .../ETL/sql/clip_select_insert_building.sql | 143 +++++++++++++++++ Bash/OSM/ETL/sql/clip_select_insert_poi.sql | 149 ++++++++++++++++++ Bash/OSM/ETL/sql/create_tables_building.sql | 31 ++++ Bash/OSM/ETL/sql/create_tables_poi.sql | 31 ++++ Bash/OSM/road/network_assessment.sql | 14 -- Bash/OSM/road/osm_road.sh | 41 ----- Bash/OSM/road/road2tsv.sql | 29 ---- Bash/readme.md | 27 +++- 47 files changed, 648 insertions(+), 498 deletions(-) rename Bash/OMF/ETL/{ => sql}/create_tables_building.sql (100%) rename Bash/OMF/ETL/{ => sql}/create_tables_place.sql (100%) create mode 160000 Bash/OMF/overturemaps-py create mode 100644 Bash/OSM/Analyze/config.sh create mode 100644 Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh create mode 100644 Bash/OSM/Analyze/copy_building_tags_to_tsv.sh delete mode 100644 Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh delete mode 100644 Bash/OSM/Analyze/copy_tags_to_tsv.sh create mode 100644 Bash/OSM/Analyze/count_building_height.sh delete mode 100644 Bash/OSM/Analyze/count_height.sh create mode 100644 Bash/OSM/Analyze/sql/building2tsv.sql delete mode 100644 Bash/OSM/Analyze/sql/copy_building_from_tsv.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_building.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_building_tag.sql delete mode 100644 Bash/OSM/Analyze/sql/create_table_poi.sql rename Bash/OSM/{ETL/sql/pois2tsv.sql => Analyze/sql/poi2tsv.sql} (73%) delete mode 100644 Bash/OSM/ETL/clip_select_insert_building.sh delete mode 100644 Bash/OSM/ETL/clip_select_insert_building_tag.sh delete mode 100644 Bash/OSM/ETL/clip_select_insert_poi.sh delete mode 100644 Bash/OSM/ETL/lua/building_with_tag.lua delete mode 100644 Bash/OSM/ETL/osm_buildings.sh create mode 100755 Bash/OSM/ETL/osm_download.sh create mode 100755 Bash/OSM/ETL/osm_import.sh delete mode 100644 Bash/OSM/ETL/osm_pois.sh create mode 100644 Bash/OSM/ETL/shape/bbox_list.cpg create mode 100644 Bash/OSM/ETL/shape/bbox_list.dbf create mode 100644 Bash/OSM/ETL/shape/bbox_list.prj create mode 100644 Bash/OSM/ETL/shape/bbox_list.qix create mode 100644 Bash/OSM/ETL/shape/bbox_list.shp create mode 100644 Bash/OSM/ETL/shape/bbox_list.shx delete mode 100644 Bash/OSM/ETL/sql/buildings2tsv.sql create mode 100755 Bash/OSM/ETL/sql/clip_select_insert_building.sql create mode 100755 Bash/OSM/ETL/sql/clip_select_insert_poi.sql create mode 100644 Bash/OSM/ETL/sql/create_tables_building.sql create mode 100644 Bash/OSM/ETL/sql/create_tables_poi.sql delete mode 100644 Bash/OSM/road/network_assessment.sql delete mode 100644 Bash/OSM/road/osm_road.sh delete mode 100644 Bash/OSM/road/road2tsv.sql diff --git a/Bash/OMF/ETL/omf_import.sh b/Bash/OMF/ETL/omf_import.sh index 67342a4..2f7601d 100644 --- a/Bash/OMF/ETL/omf_import.sh +++ b/Bash/OMF/ETL/omf_import.sh @@ -33,7 +33,7 @@ echo "Change psql user(-U) or database(-d) if threre is error about postgres" psql -U postgres -d postgres -c "create schema omf;" # create table -psql -U postgres -d postgres -f ./create_tables_$TYPE.sql +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" # Tokyo ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_$TYPE -nlt multipolygon $DATA_DIR/tokyo_$TYPE.geojson diff --git a/Bash/OMF/ETL/create_tables_building.sql b/Bash/OMF/ETL/sql/create_tables_building.sql similarity index 100% rename from Bash/OMF/ETL/create_tables_building.sql rename to Bash/OMF/ETL/sql/create_tables_building.sql diff --git a/Bash/OMF/ETL/create_tables_place.sql b/Bash/OMF/ETL/sql/create_tables_place.sql similarity index 100% rename from Bash/OMF/ETL/create_tables_place.sql rename to Bash/OMF/ETL/sql/create_tables_place.sql diff --git a/Bash/OMF/overturemaps-py b/Bash/OMF/overturemaps-py new file mode 160000 index 0000000..5148bf4 --- /dev/null +++ b/Bash/OMF/overturemaps-py @@ -0,0 +1 @@ +Subproject commit 5148bf40fa01d125095c29f70c4c91db17e4dfdd diff --git a/Bash/OSM/Analyze/config.sh b/Bash/OSM/Analyze/config.sh new file mode 100644 index 0000000..b1d3423 --- /dev/null +++ b/Bash/OSM/Analyze/config.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# define cities +cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +city_labels=("Tokyo" "Tateyama" "Hmaamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") ## becase of shape file 'Hmaamatsu' + diff --git a/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh new file mode 100644 index 0000000..0477e14 --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_cnt_to_tsv.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +for city in "${cities[@]}"; do + echo $city + sql="\copy ( + select + building, + count(*) as cnt + from osm.building_$city + group by + building + order by + cnt desc + ) to '$SCRIPT_DIR/tsv/osm_building_cnt_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" +done diff --git a/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh new file mode 100644 index 0000000..99707ee --- /dev/null +++ b/Bash/OSM/Analyze/copy_building_tags_to_tsv.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +sql="\copy ( + select + distinct building + from osm.building + order by building + ) to '$SCRIPT_DIR/tsv/osm_building_distinct_tag.tsv' + with csv delimiter E'\t';" +psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" diff --git a/Bash/OSM/Analyze/copy_building_to_tsv.sh b/Bash/OSM/Analyze/copy_building_to_tsv.sh index 064b033..fc08ef1 100644 --- a/Bash/OSM/Analyze/copy_building_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_building_to_tsv.sh @@ -1,9 +1,23 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city - sql="\copy (select area_id, id, ST_AsText(geom)from building_$city) to '~/tsv/osm_building_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" + sql="\copy ( + select + area_id, + id, + ST_AsText(geom), + building, + height + from osm.building_$city + ) to '$SCRIPT_DIR/tsv/osm_building_$city.tsv' + with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh index 36eae82..86c6822 100644 --- a/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_poi_cnt_to_tsv.sh @@ -1,17 +1,22 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city sql="\copy ( select class, subclass, count(*) as subclass_cnt - from poi_$city + from osm.poi_$city group by class, subclass - order by class, subclass_cnt desc) - to '~/osm/tsv/poi_cnt_$city.tsv' - with csv delimiter E'\t'; + order by class, subclass_cnt desc + ) to '$SCRIPT_DIR/tsv/osm_poi_cnt_$city.tsv' + with csv delimiter E'\t'; " - psql -d shimazaki -U postgres -c "$sql" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_poi_to_tsv.sh b/Bash/OSM/Analyze/copy_poi_to_tsv.sh index e17aa02..c1f7105 100644 --- a/Bash/OSM/Analyze/copy_poi_to_tsv.sh +++ b/Bash/OSM/Analyze/copy_poi_to_tsv.sh @@ -1,6 +1,11 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh for city in "${cities[@]}"; do echo $city @@ -12,7 +17,7 @@ for city in "${cities[@]}"; do class , subclass , ST_AsText(geom) - from poi_$city - ) to '~/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" + from osm.poi_$city + ) to '$SCRIPT_DIR/tsv/osm_poi_$city.tsv' with csv delimiter E'\t';" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" done diff --git a/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh deleted file mode 100644 index a70136a..0000000 --- a/Bash/OSM/Analyze/copy_tags_cnt_to_tsv.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") - -for city in "${cities[@]}"; do - echo $city - sql="\copy (select building, count(*) as cnt from building_tag_$city group by building order by cnt desc) to '~/buildings/tsv/building_tag_cnt_$city.tsv' with csv delimiter E'\t';" - psql -d shimazaki -U postgres -c "$sql" -done diff --git a/Bash/OSM/Analyze/copy_tags_to_tsv.sh b/Bash/OSM/Analyze/copy_tags_to_tsv.sh deleted file mode 100644 index 293cc6b..0000000 --- a/Bash/OSM/Analyze/copy_tags_to_tsv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -sql="\copy (select distinct building from building_tag order by building) to '~/buildings/tsv/building_distinct_tag.tsv' with csv delimiter E'\t';" -psql -d shimazaki -U postgres -c "$sql" diff --git a/Bash/OSM/Analyze/count_building_height.sh b/Bash/OSM/Analyze/count_building_height.sh new file mode 100644 index 0000000..682dd4a --- /dev/null +++ b/Bash/OSM/Analyze/count_building_height.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh + +cnt=0 +for city in "${cities[@]}"; do + echo $city + sql=" + select + count(height) as cnt_height + from osm.building_$city + " + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" + + ((cnt++)) +done diff --git a/Bash/OSM/Analyze/count_height.sh b/Bash/OSM/Analyze/count_height.sh deleted file mode 100644 index e9c9846..0000000 --- a/Bash/OSM/Analyze/count_height.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql=" - select - count(height) as cnt_height - from building_tag_$city - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/Analyze/count_poi_amenity_shop.sh b/Bash/OSM/Analyze/count_poi_amenity_shop.sh index e331544..52a4af0 100644 --- a/Bash/OSM/Analyze/count_poi_amenity_shop.sh +++ b/Bash/OSM/Analyze/count_poi_amenity_shop.sh @@ -1,13 +1,15 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh cnt=0 for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - + echo $city sql=" select name, @@ -16,27 +18,27 @@ for city in "${cities[@]}"; do select 'poi_cnt' as name, count(*) as cnt - from poi_$city + from osm.poi_$city union - select + select 'poi_amenity_cnt' as name, - count(*) as cnt - from poi_$city + count(*) as cnt + from osm.poi_$city where class='amenity' union select 'poi_shop_cnt' as name, - count(*) as cnt - from poi_$city - where class='shop' + count(*) as cnt + from osm.poi_$city + where class='shop' ) a order by name " - psql -d shimazaki -U postgres -c "$sql" + psql -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql" ((cnt++)) done diff --git a/Bash/OSM/Analyze/coverage.sh b/Bash/OSM/Analyze/coverage.sh index 15e94f2..d958268 100644 --- a/Bash/OSM/Analyze/coverage.sh +++ b/Bash/OSM/Analyze/coverage.sh @@ -1,19 +1,22 @@ #!/bin/bash -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# set config +source config.sh cnt=0 for city in "${cities[@]}"; do echo $city - city_lable="$city_labels[$cnt]" - sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from building_$city" - building_area=$(psql -t -d shimazaki -U postgres -c "$sql") + sql="select sum(ST_Area(building_$city.geom::geography)) / 1000000 from osm.building_$city" + building_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") echo $building_area - sql="select ST_Area(geom::geography) / 1000000 from shape where name='${city_labels[$cnt]}'" - whole_area=$(psql -t -d shimazaki -U postgres -c "$sql") + sql="select ST_Area(geom::geography) / 1000000 from osm.shape where name='${city_labels[$cnt]}'" + whole_area=$(psql -t -d $POSTGRES_DATABASE -U $POSTGRES_USER -c "$sql") echo $whole_area echo "scale=5; $building_area / $whole_area * 100" | bc diff --git a/Bash/OSM/Analyze/sql/building2tsv.sql b/Bash/OSM/Analyze/sql/building2tsv.sql new file mode 100644 index 0000000..5adcdf7 --- /dev/null +++ b/Bash/OSM/Analyze/sql/building2tsv.sql @@ -0,0 +1,16 @@ +copy ( + select + area_id, + id, + ST_asText (geom), + building, + height + from + osm.building + where + area_id is not null + and geom is not null + order by + area_id +) +to STDOUT with csv header delimiter E'\t' \g '../tsv/building.tsv' \ No newline at end of file diff --git a/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql b/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql deleted file mode 100644 index b73f7b4..0000000 --- a/Bash/OSM/Analyze/sql/copy_building_from_tsv.sql +++ /dev/null @@ -1,2 +0,0 @@ -\copy shimazaki from '../osm_buildings_japan.tsv' ( delimiter E'\t', format csv, header true ); - diff --git a/Bash/OSM/Analyze/sql/create_table_building.sql b/Bash/OSM/Analyze/sql/create_table_building.sql deleted file mode 100644 index 985f861..0000000 --- a/Bash/OSM/Analyze/sql/create_table_building.sql +++ /dev/null @@ -1,42 +0,0 @@ -drop table if exists building; -create table building ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_tokyo; -create table building_tokyo ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_hamamatsu; -create table building_hamamatsu ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_tateyama; -create table building_tateyama ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_morioka; -create table building_morioka ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_higashi_hiroshima; -create table building_higashi_hiroshima ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); -drop table if exists building_kumamoto; -create table building_kumamoto ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326) -); diff --git a/Bash/OSM/Analyze/sql/create_table_building_tag.sql b/Bash/OSM/Analyze/sql/create_table_building_tag.sql deleted file mode 100644 index 5285ddc..0000000 --- a/Bash/OSM/Analyze/sql/create_table_building_tag.sql +++ /dev/null @@ -1,48 +0,0 @@ -drop table if exists building_tag_tokyo; -create table building_tag_tokyo ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_hamamatsu; -create table building_tag_hamamatsu ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_tateyama; -create table building_tag_tateyama ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_morioka; -create table building_tag_morioka ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_higashi_hiroshima; -create table building_tag_higashi_hiroshima ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); -drop table if exists building_tag_kumamoto; -create table building_tag_kumamoto ( - area_id bigint, - id bigint, - geom geometry(multipolygon,4326), - building text, - height text -); diff --git a/Bash/OSM/Analyze/sql/create_table_poi.sql b/Bash/OSM/Analyze/sql/create_table_poi.sql deleted file mode 100644 index a48a2a0..0000000 --- a/Bash/OSM/Analyze/sql/create_table_poi.sql +++ /dev/null @@ -1,13 +0,0 @@ -drop table if exists poi_tokyo; -create table poi_tokyo (LIKE pois INCLUDING ALL); -drop table if exists poi_hamamatsu; -create table poi_hamamatsu (LIKE pois INCLUDING ALL); -drop table if exists poi_tateyama; -create table poi_tateyama (LIKE pois INCLUDING ALL); -drop table if exists poi_morioka; -create table poi_morioka (LIKE pois INCLUDING ALL); -drop table if exists poi_higashi_hiroshima; -create table poi_higashi_hiroshima (LIKE pois INCLUDING ALL); -drop table if exists poi_kumamoto; -create table poi_kumamoto (LIKE pois INCLUDING ALL); - diff --git a/Bash/OSM/ETL/sql/pois2tsv.sql b/Bash/OSM/Analyze/sql/poi2tsv.sql similarity index 73% rename from Bash/OSM/ETL/sql/pois2tsv.sql rename to Bash/OSM/Analyze/sql/poi2tsv.sql index eca60be..f38b3e2 100644 --- a/Bash/OSM/ETL/sql/pois2tsv.sql +++ b/Bash/OSM/Analyze/sql/poi2tsv.sql @@ -7,9 +7,9 @@ copy ( subclass, ST_asText(geom) from - public.pois + osm.poi where osm_id is not null and geom is not null order by osm_id -) to STDOUT with csv header delimiter E'\t' \g 'pois.tsv' \ No newline at end of file +) to STDOUT with csv header delimiter E'\t' \g '../tsv/poi.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/clip_select_insert_building.sh b/Bash/OSM/ETL/clip_select_insert_building.sh deleted file mode 100644 index 61b60c7..0000000 --- a/Bash/OSM/ETL/clip_select_insert_building.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate building_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into building_$city ( - area_id, - id, - geom - ) - select - area_id, - id, - ST_AsText(ST_Intersection(building.geom, clipping_area.geom)) as geom - from building, clipping_area - where ST_Intersects(building.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/clip_select_insert_building_tag.sh b/Bash/OSM/ETL/clip_select_insert_building_tag.sh deleted file mode 100644 index 3fc3b0a..0000000 --- a/Bash/OSM/ETL/clip_select_insert_building_tag.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate building_tag_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into building_tag_$city ( - area_id, - id, - geom, - building, - height - ) - select - area_id, - id, - ST_AsText(ST_Intersection(building_tag.geom, clipping_area.geom)) as geom, - building, - height - from building_tag, clipping_area - where ST_Intersects(building_tag.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/clip_select_insert_poi.sh b/Bash/OSM/ETL/clip_select_insert_poi.sh deleted file mode 100644 index 382a9b1..0000000 --- a/Bash/OSM/ETL/clip_select_insert_poi.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -cities=("tokyo" "tateyama" "hamamatsu" "higashi_hiroshima" "kumamoto" "morioka") -city_labels=("Tokyo" "Tateyama" "Hamamatsu" "Higashi-hiroshima" "Kumamoto" "Morioka") - -cnt=0 -for city in "${cities[@]}"; do - echo $city - city_lable="$city_labels[$cnt]" - - sql="truncate poi_$city" - psql -d shimazaki -U postgres -c "$sql" - - sql=" - with clipping_area as ( - select * from shape where name='${city_labels[$cnt]}' - ) - insert into poi_$city ( - osm_type , - osm_id , - name , - class , - subclass , - geom - ) - select - osm_type , - osm_id , - pois.name, - class , - subclass , - ST_AsText(ST_Intersection(pois.geom, clipping_area.geom)) as geom - from pois, clipping_area - where ST_Intersects(pois.geom, clipping_area.geom) - " - psql -d shimazaki -U postgres -c "$sql" - - ((cnt++)) -done diff --git a/Bash/OSM/ETL/lua/building.lua b/Bash/OSM/ETL/lua/building.lua index 82fd115..3b35e73 100644 --- a/Bash/OSM/ETL/lua/building.lua +++ b/Bash/OSM/ETL/lua/building.lua @@ -1,7 +1,10 @@ -local buildings = osm2pgsql.define_area_table('buildings', { +local building = osm2pgsql.define_area_table('building', { -- Define an autoincrementing id column, QGIS likes a unique id on the table { column = 'id', sql_type = 'serial', create_only = true }, - { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, + { column = 'geom', type = 'multipolygon', not_null = true, projection = 4326}, + -- Add columns for tags + { column = 'building', type = 'text' }, + { column = 'height', type = 'text' } }, { indexes = { -- So we get an index on the id column { column = 'id', method = 'btree', unique = true }, @@ -12,8 +15,10 @@ local buildings = osm2pgsql.define_area_table('buildings', { function osm2pgsql.process_way(object) if object.is_closed and object.tags.building then - buildings:insert({ - geom = object:as_polygon():transform(4326) + building:insert({ + geom = object:as_polygon():transform(4326), + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag }) end end @@ -24,8 +29,10 @@ function osm2pgsql.process_relation(object) local mp = object:as_multipolygon():transform(4326) -- ...and split them into polygons which we insert into the table for geom in mp:geometries() do - buildings:insert({ - geom = geom + building:insert({ + geom = geom, + building = object.tags.building, -- Example: Assuming 'building' tag + height = object.tags.height -- Example: Assuming 'height' tag }) end end diff --git a/Bash/OSM/ETL/lua/building_with_tag.lua b/Bash/OSM/ETL/lua/building_with_tag.lua deleted file mode 100644 index 518cb0a..0000000 --- a/Bash/OSM/ETL/lua/building_with_tag.lua +++ /dev/null @@ -1,39 +0,0 @@ -local building_tag = osm2pgsql.define_area_table('building_tag', { - -- Define an autoincrementing id column, QGIS likes a unique id on the table - { column = 'id', sql_type = 'serial', create_only = true }, - { column = 'geom', type = 'polygon', not_null = true, projection = 4326}, - -- Add columns for tags - { column = 'building', type = 'text' }, - { column = 'height', type = 'text' } -}, { indexes = { - -- So we get an index on the id column - { column = 'id', method = 'btree', unique = true }, - -- If we define any indexes we don't get the default index on the geometry - -- column, so we add it here. - { column = 'geom', method = 'gist' } -}}) - -function osm2pgsql.process_way(object) - if object.is_closed and object.tags.building then - building_tag:insert({ - geom = object:as_polygon():transform(4326), - building = object.tags.building, -- Example: Assuming 'building' tag - height = object.tags.height -- Example: Assuming 'height' tag - }) - end -end - -function osm2pgsql.process_relation(object) - if object.tags.type == 'multipolygon' and object.tags.building then - -- From the relation we get multipolygons... - local mp = object:as_multipolygon():transform(4326) - -- ...and split them into polygons which we insert into the table - for geom in mp:geometries() do - building_tag:insert({ - geom = geom, - building = object.tags.building, -- Example: Assuming 'building' tag - height = object.tags.height -- Example: Assuming 'height' tag - }) - end - end -end diff --git a/Bash/OSM/ETL/lua/poi.lua b/Bash/OSM/ETL/lua/poi.lua index 43350c4..9c62123 100644 --- a/Bash/OSM/ETL/lua/poi.lua +++ b/Bash/OSM/ETL/lua/poi.lua @@ -1,6 +1,6 @@ -local pois = osm2pgsql.define_table({ - name = 'pois', +local poi = osm2pgsql.define_table({ + name = 'poi', ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' }, columns = { { column = 'name' }, @@ -25,7 +25,7 @@ function process_poi(object, geom) return end - pois:insert(a) + poi:insert(a) end function osm2pgsql.process_node(object) diff --git a/Bash/OSM/ETL/osm_buildings.sh b/Bash/OSM/ETL/osm_buildings.sh deleted file mode 100644 index 74e3c7b..0000000 --- a/Bash/OSM/ETL/osm_buildings.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -psql -U postgres -d $3 -c "drop table if exists public.buildings;" - -# import pbf file to database -/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/buildings.lua -E 4326 $1 - -mkdir ./osm_buildings_${2}/ - -# export table to filqs -pg_dump -U postgres -t public.buildings $3 | gzip > ./osm_buildings_${2}/osm_buildings_${2}.sql.gz - -psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/buildings2tsv.sql -mv ./buildings.tsv ./osm_buildings_${2}/osm_buildings_${2}.tsv -rm ./osm_buildings_${2}/osm_buildings_${2}.tsv.gz -gzip ./osm_buildings_${2}/osm_buildings_${2}.tsv - -# export both tables to shp and compression them -mkdir ./osm_buildings_${2}/osm_buildings_${2}_shp/ -pgsql2shp -u postgres -P postgres -f ./osm_buildings_${2}/osm_buildings_${2}_shp/osm_buildings_${2}.shp $3 public.buildings -zip ./osm_buildings_${2}/osm_buildings_${2}_shp.zip -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ -rm -r ./osm_buildings_${2}/osm_buildings_${2}_shp/ - -psql -U postgres -d $3 -c "drop table if exists public.buildings;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_download.sh b/Bash/OSM/ETL/osm_download.sh new file mode 100755 index 0000000..f767655 --- /dev/null +++ b/Bash/OSM/ETL/osm_download.sh @@ -0,0 +1,28 @@ +#!/bin/bash　-e + +# Initialize the command in seconds. +SECONDS=0 + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$CURRENT_DATE" +mkdir -p "$DATA_DIR" + +### Main ### + +# download +curl -o "$DATA_DIR/japan-latest.osm.pbf" https://download.geofabrik.de/asia/japan-latest.osm.pbf + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_import.sh b/Bash/OSM/ETL/osm_import.sh new file mode 100755 index 0000000..af0ab8f --- /dev/null +++ b/Bash/OSM/ETL/osm_import.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +### Preprocessing ### + +# Initialize the command in seconds. +SECONDS=0 + +# postgres +POSTGRES_USER=postgres +POSTGRES_DATABASE=postgres + +# get current date +CURRENT_DATE=$(date +"%Y%m%d") + +# move to working directory +SCRIPT_DIR=$(cd $(dirname $0); pwd) +cd "${SCRIPT_DIR}" + +# get command line arguments +if [ -z "$1" ]; then + echo -e "Write Type in first argument.\n e.g. [building, poi]" + exit 1 +fi +if [ -z "$2" ]; then + $2 = $CURRENT_DATE + exit 1 +fi +TYPE=$1 # e.g. [building, poi] +YYYYMMDD=$2 # e.g. [20240729] + +# define data dir +DATA_DIR="$SCRIPT_DIR/data/$YYYYMMDD" +if [ ! -d "$DATA_DIR" ] ; then + echo -e "DATA_DIR was empty: $DATA_DIR" + echo -e "Write Type in second argument.\n e.g. [YYYYMMDD]" +fi + +# pbf file +PBF_FILE="$DATA_DIR/japan-latest.osm.pbf" +if [ ! -f "$PBF_FILE" ] ; then + echo -e "There is no pbf file. Dowload at first!" + exit 1 +fi + +# shp file +SHP_FILE="$SCRIPT_DIR/shape/bbox_list.shp" + +### Main ### + +# create database, and drop $TYPE table made by osm2pgsql +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "create schema osm;" +psql -U $POSTGRES_USER -d $POSTGRES_DATABASE -c "drop table if exists osm.$TYPE;" + +# import shape to database +shp2pgsql -I -s 4326 $SHP_FILE osm.shape | psql -d $POSTGRES_DATABASE -U $POSTGRES_USER + +# import pbf file to database +osm2pgsql -d $POSTGRES_DATABASE --schema osm -O flex -S "$SCRIPT_DIR/lua/$TYPE.lua" -E 4326 "$PBF_FILE" + +# create table in each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/create_tables_$TYPE.sql" + +# insert data about each cities +psql -U postgres -d postgres -f "$SCRIPT_DIR/sql/clip_select_insert_$TYPE.sql" + +### Postprocessing ### + +# Display the measurement time. +time=$SECONDS +((sec=time%60, min=(time%3600)/60, hrs=time/3600)) +timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") +echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/osm_pois.sh b/Bash/OSM/ETL/osm_pois.sh deleted file mode 100644 index a47a4bd..0000000 --- a/Bash/OSM/ETL/osm_pois.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -echo $2 -psql -U postgres -d $3 -c "drop table if exists public.pois;" - -# import pbf file to database -/data/tool/osm2pgsql/1.9.2/osm2pgsql -d $3 -O flex -S ${0%/*}/pois.lua -E 4326 $1 - -mkdir ./osm_pois_$2/ - -# export table to files -pg_dump -U postgres -t public.pois $3 | gzip > ./osm_pois_$2/osm_pois_$2.sql.gz - -psql -v ON_ERROR_STOP=1 -U postgres -d $3 -f ${0%/*}/pois2tsv.sql -mv ./pois.tsv ./osm_pois_$2/osm_pois_${2}.tsv -rm ./osm_pois_${2}/osm_pois_${2}.tsv.gz -gzip ./osm_pois_${2}/osm_pois_${2}.tsv - -# export both tables to shp and compression them -mkdir ./osm_pois_${2}/osm_pois_${2}_shp/ -pgsql2shp -u postgres -P postgres -f ./osm_pois_${2}/osm_pois_${2}_shp/osm_pois_${2}.shp $3 public.pois -zip ./osm_pois_${2}/osm_pois_${2}_shp.zip -r ./osm_pois_${2}/osm_pois_${2}_shp/ -rm -r ./osm_pois_${2}/osm_pois_${2}_shp/ - -psql -U postgres -d $3 -c "drop table if exists public.pois;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.cpg b/Bash/OSM/ETL/shape/bbox_list.cpg new file mode 100644 index 0000000..3ad133c --- /dev/null +++ b/Bash/OSM/ETL/shape/bbox_list.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/Bash/OSM/ETL/shape/bbox_list.dbf b/Bash/OSM/ETL/shape/bbox_list.dbf new file mode 100644 index 0000000000000000000000000000000000000000..e6918215d1225d128209d7a0210fb51d5ad1b4d4 GIT binary patch literal 4043 zcmdUyO^e$w5Qakw?Wva@_vB*&u4-vUA9LbU!{uZO?z@863V>vi?39eR=P z%I#gu)y4V`+0F6ku=!p+o$jk$wp=bHN>NyAG-}9zgmDU-fjqkeR)f`2s)k6T{R(M# zNtz^swP3YcWs_d9s60~3<3SRS2T7AA_un6OQ$260$8yYbdUk=rTf{1Z1!bUwRfLWl zc~@KzS4fNCbVIE7jz+6^O`4>m1v0H;VoiChmO+rjAV?Zh+E+jK+i_cNt3x#+(Y{?s zH~@0q2^pXuZHY8UHbqRdC=?MHuni$4oz)qn#?z5HGObm(NU^58r?%0QN18?yb_P># z%fsEN+>Y5^`gXCxfU(WXM_~+scDmj}&WS~7q-udxXbNf^mIUcoflO@}37sakOUQ#i z0qb|xMii4))C^28JjLs6>CJN3ABQjL+eHaOC>>zEyZD2PQoe3nz;lAyp_8(1Q@&|A zRrr$fhLi;9Sb1NuZKiJ*f3T}R zs;0FVerF9%m&0D000D>4wV1^ literal 0 HcmV?d00001 diff --git a/Bash/OSM/ETL/shape/bbox_list.shp b/Bash/OSM/ETL/shape/bbox_list.shp new file mode 100644 index 0000000000000000000000000000000000000000..cf7aca88407fdab88250fea41f259e4707d6af01 GIT binary patch literal 1052 zcmZQzQ0HR63Yhp_Ff%Z)0_DUOT3Wc4$2o*D8OJGg896Nae=_R(_9TafGiirwlH47z zYDX1h1acifW)-}MQmt8(=-_-&H9z9Ivjd3lvoz6xfqUJnst3*vAcY`xAb?FRNH0t+ zx?UI`*=%IHfJQO_G1wi;qg@35g(f(d=Nx(yAM4-%;-8C1a9Hhdt80Fm1ByGasRikU zsYTZd<0G4mY!}cCz>tNw14uJ$s7iDY{;V*4ewCvGh~H3~=#c#T`0H!UjwtQ`$)T$S z@nLGw^}_hbW~1q60jdFq2S{y#Nuq=Igrcssp%8Z%m?t{yPwCQl8;9u*bhRKpOfASP zWW6vxve{_*K~WBN2j51Q17_0_95QVmKFK$9bO7<$W+ph?75v$>-U`(n*wljb!qlSc zh4GQiMz#y+1~!;GKyH3ep6HCgIV}2rGV1&GB!`AGX@_f)+#RrM dM-epw@<8g!p!6;%{SQJjIzj0+D18b<0|2(<6q^75 literal 0 HcmV?d00001 diff --git a/Bash/OSM/ETL/sql/buildings2tsv.sql b/Bash/OSM/ETL/sql/buildings2tsv.sql deleted file mode 100644 index 87fff60..0000000 --- a/Bash/OSM/ETL/sql/buildings2tsv.sql +++ /dev/null @@ -1,12 +0,0 @@ -copy ( - select - area_id, - id, - ST_asText(geom) - from - public.buildings - where - area_id is not null and geom is not null - order by - area_id -) to STDOUT with csv header delimiter E'\t' \g 'buildings.tsv' \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/clip_select_insert_building.sql b/Bash/OSM/ETL/sql/clip_select_insert_building.sql new file mode 100755 index 0000000..3bede6c --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_building.sql @@ -0,0 +1,143 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.building_tokyo (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.building_tateyama (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.building_hamamatsu (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.building_higashi_hiroshima (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.building_kumamoto (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.building_morioka (area_id, id, geom, building, height) +select + area_id, + id, + ST_Intersection (building.geom, clipping_area.geom) as geom, + building, + height +from + osm.building, + clipping_area +where + ST_Intersects (building.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/clip_select_insert_poi.sql b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql new file mode 100755 index 0000000..2c8e408 --- /dev/null +++ b/Bash/OSM/ETL/sql/clip_select_insert_poi.sql @@ -0,0 +1,149 @@ +-- tokyo +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tokyo' + ) +insert into + osm.poi_tokyo (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- tateyama +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Tateyama' + ) +insert into + osm.poi_tateyama (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- hamamatsu +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Hmaamatsu' -- becase of shape file + ) +insert into + osm.poi_hamamatsu (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- higashi_hiroshima +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Higashi-hiroshima' + ) +insert into + osm.poi_higashi_hiroshima (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- kumamoto +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Kumamoto' + ) +insert into + osm.poi_kumamoto (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); + +-- morioka +with + clipping_area as ( + select + * + from + osm.shape + where + name = 'Morioka' + ) +insert into + osm.poi_morioka (osm_type, osm_id, name, class, subclass, geom) +select + osm_type, + osm_id, + poi.name, + class, + subclass, + ST_AsText (ST_Intersection (poi.geom, clipping_area.geom)) as geom +from + osm.poi, + clipping_area +where + ST_Intersects (poi.geom, clipping_area.geom); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_building.sql b/Bash/OSM/ETL/sql/create_tables_building.sql new file mode 100644 index 0000000..946fdf3 --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_building.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.building_tokyo; + +drop table if exists osm.building_hamamatsu; + +drop table if exists osm.building_tateyama; + +drop table if exists osm.building_kumamoto; + +drop table if exists osm.building_higashi_hiroshima; + +drop table if exists osm.building_morioka; + +-- create table +create table + osm.building_tokyo (like osm.building including all); + +create table + osm.building_hamamatsu (like osm.building including all); + +create table + osm.building_tateyama (like osm.building including all); + +create table + osm.building_kumamoto (like osm.building including all); + +create table + osm.building_higashi_hiroshima (like osm.building including all); + +create table + osm.building_morioka (like osm.building including all); \ No newline at end of file diff --git a/Bash/OSM/ETL/sql/create_tables_poi.sql b/Bash/OSM/ETL/sql/create_tables_poi.sql new file mode 100644 index 0000000..cea7c1b --- /dev/null +++ b/Bash/OSM/ETL/sql/create_tables_poi.sql @@ -0,0 +1,31 @@ +-- drop table +drop table if exists osm.poi_tokyo; + +drop table if exists osm.poi_hamamatsu; + +drop table if exists osm.poi_tateyama; + +drop table if exists osm.poi_kumamoto; + +drop table if exists osm.poi_higashi_hiroshima; + +drop table if exists osm.poi_morioka; + +-- create table +create table + osm.poi_tokyo (like osm.poi including all); + +create table + osm.poi_hamamatsu (like osm.poi including all); + +create table + osm.poi_tateyama (like osm.poi including all); + +create table + osm.poi_kumamoto (like osm.poi including all); + +create table + osm.poi_higashi_hiroshima (like osm.poi including all); + +create table + osm.poi_morioka (like osm.poi including all); \ No newline at end of file diff --git a/Bash/OSM/road/network_assessment.sql b/Bash/OSM/road/network_assessment.sql deleted file mode 100644 index ebef3ac..0000000 --- a/Bash/OSM/road/network_assessment.sql +++ /dev/null @@ -1,14 +0,0 @@ -alter table public.osm_road_available add column groupno integer; - -update public.osm_road_available -set groupno = val.groupno -from ( - select - oras.groupno as groupno, - orav.id as id - from - public.osm_road_assessment oras, - public.osm_road_available orav - where ST_Intersects(orav.geom_way, oras.geom) -) as val where public.osm_road_available.id = val.id ; - diff --git a/Bash/OSM/road/osm_road.sh b/Bash/OSM/road/osm_road.sh deleted file mode 100644 index 7433f1e..0000000 --- a/Bash/OSM/road/osm_road.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash　-e - -# Initialize the command in seconds. -SECONDS=0 - -# remove output files generated previpusly -rm -r ./osm_road - -# convert osm.pbf to sql.gz by using osm2po -sh ${0%/*}/osm2road/conv_osm_road.sh $1 - -psql -d postgres -c "drop table if exists public.osm_road_available;" -psql -d postgres -c "drop table if exists public.osm_road_assessment;" - -# import the files into tables -gzip -cd ./osm_road/osm_road_available.sql.gz | psql -d postgres -gzip -cd ./osm_road/osm_road_assessment.sql.gz | psql -d postgres - -# assess the network -psql -v ON_ERROR_STOP=1 -d postgres -f ${0%/*}/network_assessment.sql - -# export the network to files -psql -d postgres -f ${0%/*}/road2tsv.sql -mv ./osm_road/road.tsv ./osm_road/osm_road_$2.tsv -pg_dump -d postgres -t osm_road_available | gzip > ./osm_road/osm_road_$2.sql.gz - -mkdir ./osm_road/osm_road_${2}_shp/ -pgsql2shp -P postgreses -f ./osm_road/osm_road_${2}_shp/osm_road_${2}.shp miyake public.road -zip ./osm_road/osm_road_${2}_shp.zip -r ./osm_road/osm_road_${2}_shp/ -rm -r ./osm_road/osm_road_${2}_shp/ - -mv ./osm_road/ ./osm_road_${2}/ - -psql -d postgres -c "drop table if exists public.osm_road_available;" -psql -d postgres -c "drop table if exists public.osm_road_assessment;" - -# Display the measurement time. -time=$SECONDS -((sec=time%60, min=(time%3600)/60, hrs=time/3600)) -timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec") -echo "Processing time is $timestamp" diff --git a/Bash/OSM/road/road2tsv.sql b/Bash/OSM/road/road2tsv.sql deleted file mode 100644 index fa8237c..0000000 --- a/Bash/OSM/road/road2tsv.sql +++ /dev/null @@ -1,29 +0,0 @@ -copy ( - select - groupno, - osm_id, - id, - osm_source_id, - osm_target_id, - clazz, - flags, - source, - target, - km, - kmh, - cost, - reverse_cost, - x1, - y1, - x2, - y2, - osm_name, - osm_meta, - ST_asText(geom_way) - from - public.osm_road_available - where - osm_id is not null and geom_way is not null - order by - osm_id -) to STDOUT with csv header delimiter E'\t' \g 'road.tsv' \ No newline at end of file diff --git a/Bash/readme.md b/Bash/readme.md index 6401375..541cef3 100644 --- a/Bash/readme.md +++ b/Bash/readme.md @@ -1,6 +1,6 @@ # Buildings -## Overture Maps +## Overture Maps Foundation (OMF) - Install a CLI tool or overturemaps-py repo to download Overture Maps data. @@ -60,11 +60,28 @@ $ aws s3 ls s3://overturemaps-us-west-2/release/2024-07-22.0/ --region us-west-2 PRE theme=transportation/ ``` -## OpenStreetMap +## OpenStreetMap (OSM) -This script just import tsv files into PostgreSQL. -You need to extract data before running this script. +- Install a CLI tool ```sh -bash building/import_osm_tsv.sh +# osm2pgsql: tool for importing OSM pbf data into postgres +## [Other OS](https://osm2pgsql.org/doc/install.html) +## Mac +$ brew install osm2pgsql +``` + +- Download data from osm as a pbf format. + +```sh +bash OSM/ETL/osm_download.sh +``` + +- Then, import that data to postgres + +```sh +# building +bash OSM/ETL/osm_import.sh building YYYYMMDD +# poi +bash OSM/ETL/osm_import.sh poi YYYYMMDD ```