Skip to content

Commit 57bd04b

Browse files
ke-lmke-lm
committed
fix dowload and import script
1 parent 6fd389c commit 57bd04b

File tree

10 files changed

+237
-247
lines changed

10 files changed

+237
-247
lines changed

‎.gitignore

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,8 @@ __pycache__/
33
cache/
44
.streamlit/
55
tiles
6-
other.py
6+
other.py
7+
8+
# ignore data dir
9+
Bash/**/data/**
10+
Bash/**/overturemaps-py/**

‎Bash/OMF/ETL/create_tables_building.sql

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
-- drop table
2+
drop table if exists omf.tokyo_building;
3+
drop table if exists omf.hamamatsu_building;
4+
drop table if exists omf.tateyama_building;
5+
drop table if exists omf.kumamoto_building;
6+
drop table if exists omf.higashi_hiroshima_building;
7+
drop table if exists omf.morioka_building;
8+
drop table if exists omf.building;
9+
10+
-- create table
11+
create table omf.building (
12+
ogc_fid integer,
13+
wkb_geometry geometry(MULTIPOLYGON, 4326),
14+
id varchar,
15+
version integer,
16+
update_time timestamp with time zone,
17+
sources json,
18+
subtype varchar,
19+
names json,
20+
class varchar,
21+
level integer,
22+
has_parts boolean,
23+
height double precision,
24+
min_height double precision,
25+
num_floors integer,
26+
facade_color varchar,
27+
facade_material varchar,
28+
min_floor integer,
29+
roof_material varchar,
30+
roof_shape varchar,
31+
roof_direction double precision,
32+
roof_orientation varchar,
33+
roof_color varchar,
34+
area_name char(50)
35+
)
36+
;
37+
38+
create table
39+
omf.tokyo_building (like omf.building including all);
40+
41+
create table
42+
omf.hamamatsu_building (like omf.building including all);
43+
44+
create table
45+
omf.tateyama_building (like omf.building including all);
46+
47+
create table
48+
omf.kumamoto_building (like omf.building including all);
49+
50+
create table
51+
omf.higashi_hiroshima_building (like omf.building including all);
52+
53+
create table
54+
omf.morioka_building (like omf.building including all);

‎Bash/OMF/ETL/create_tables_place.sql

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
-- drop table
2+
drop table if exists omf.tokyo_place;
3+
drop table if exists omf.hamamatsu_place;
4+
drop table if exists omf.tateyama_place;
5+
drop table if exists omf.kumamoto_place;
6+
drop table if exists omf.higashi_hiroshima_place;
7+
drop table if exists omf.morioka_place;
8+
drop table if exists omf.place;
9+
10+
-- create table
11+
create table
12+
omf.place (
13+
ogc_fid serial primary key,
14+
wkb_geometry geometry (point, 4326),
15+
id varchar,
16+
version integer,
17+
update_time timestamp with time zone,
18+
sources jsonb,
19+
names jsonb,
20+
categories jsonb,
21+
confidence double precision,
22+
websites text,
23+
emails text,
24+
socials text,
25+
phones text,
26+
addresses jsonb,
27+
brand jsonb
28+
);
29+
30+
create table
31+
omf.tokyo_place (like omf.place including all);
32+
33+
create table
34+
omf.hamamatsu_place (like omf.place including all);
35+
36+
create table
37+
omf.tateyama_place (like omf.place including all);
38+
39+
create table
40+
omf.kumamoto_place (like omf.place including all);
41+
42+
create table
43+
omf.higashi_hiroshima_place (like omf.place including all);
44+
45+
create table
46+
omf.morioka_place (like omf.place including all);

‎Bash/OMF/ETL/omf_download.sh

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#!/bin/bash
2+
3+
# Initialize the command in seconds.
4+
SECONDS=0
5+
6+
# move to working directory
7+
SCRIPT_DIR=$(cd $(dirname $0); pwd)
8+
cd "${SCRIPT_DIR}"
9+
10+
# get command line arguments
11+
if [ -z "$1" ]; then
12+
echo -e "Write Type in first argument.\n e.g. [building, place]"
13+
exit 1
14+
fi
15+
if [ -z "$2" ]; then
16+
echo -e "Write Release Version in second argument.\nYou could check list of versions:"
17+
aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request
18+
exit 1
19+
fi
20+
TYPE=$1 # e.g. [building, place]
21+
RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1]
22+
23+
# create data directory
24+
DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/"
25+
mkdir -p $DATA_DIR
26+
27+
# change resource in "git@github.com:OvertureMaps/overturemaps-py.git" to change
28+
cd "${SCRIPT_DIR}/overturemaps-py"
29+
before="overturemaps-us-west-2/release/.*/theme"
30+
after="overturemaps-us-west-2/release/$RELEASE_VERSION/theme"
31+
sed -i '' "s|$before|$after|g" "${SCRIPT_DIR}/overturemaps-py/overturemaps/core.py"
32+
33+
# Tokyo
34+
poetry run overturemaps download --bbox=139.74609375,35.67514744,139.83398438,35.74651226 -f geojson --type=$TYPE -o "$DATA_DIR/tokyo_$TYPE.geojson"
35+
36+
# Hamamatsu
37+
poetry run overturemaps download --bbox=137.63671875,34.66935855,137.72460938,34.7416125 -f geojson --type=$TYPE -o "$DATA_DIR/hamamatsu_$TYPE.geojson"
38+
39+
# Tateyama
40+
poetry run overturemaps download --bbox=139.83398438,34.95799531,139.921875,35.02999637 -f geojson --type=$TYPE -o "$DATA_DIR/tateyama_$TYPE.geojson"
41+
42+
# Kumamoto
43+
poetry run overturemaps download --bbox=130.68726409,32.72948989,130.77515472,32.80174385 -f geojson --type=$TYPE -o "$DATA_DIR/kumamoto_$TYPE.geojson"
44+
45+
# Higashi_hiroshima
46+
poetry run overturemaps download --bbox=132.69418348,34.38622724,132.7820741,34.45848119 -f geojson --type=$TYPE -o "$DATA_DIR/higashi_hiroshima_$TYPE.geojson"
47+
48+
# Morioka
49+
poetry run overturemaps download --bbox=141.07765453,39.6823863,141.16554516,39.75375112 -f geojson --type=$TYPE -o "$DATA_DIR/morioka_$TYPE.geojson"
50+
51+
# Display the measurement time.
52+
time=$SECONDS
53+
((sec=time%60, min=(time%3600)/60, hrs=time/3600))
54+
timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec")
55+
echo "Processing time is $timestamp"
56+
57+
exit 0

‎Bash/OMF/ETL/omf_import.sh

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Initialize the command in seconds.
4+
SECONDS=0
5+
6+
# move to working directory
7+
SCRIPT_DIR=$(cd $(dirname $0); pwd)
8+
cd "${SCRIPT_DIR}"
9+
10+
# get command line arguments
11+
if [ -z "$1" ]; then
12+
echo -e "Write Type in first argument.\n e.g. [building, place]"
13+
exit 1
14+
fi
15+
if [ -z "$2" ]; then
16+
echo -e "Write Release Version in second argument.\nYou could check list of versions:"
17+
aws s3 ls s3://overturemaps-us-west-2/release/ --region us-west-2 --no-sign-request
18+
exit 1
19+
fi
20+
TYPE=$1 # e.g. [building, place]
21+
RELEASE_VERSION=$2 # e.g. [2024-07-22.0, 2024-06-13-beta.1]
22+
23+
# define data dir
24+
DATA_DIR="$SCRIPT_DIR/data/$RELEASE_VERSION/overturemaps/"
25+
26+
# create schema
27+
psql -U postgres -d postgres -c "create schema omf;"
28+
29+
# create table
30+
psql -U postgres -d postgres -f ./create_tables_$TYPE.sql
31+
32+
# Tokyo
33+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tokyo_$TYPE -nlt multipolygon $DATA_DIR/tokyo_$TYPE.geojson
34+
35+
# Hamamatsu
36+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.hamamatsu_$TYPE -nlt multipolygon $DATA_DIR/hamamatsu_$TYPE.geojson
37+
38+
# Tateyama
39+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.tateyama_$TYPE -nlt multipolygon $DATA_DIR/tateyama_$TYPE.geojson
40+
41+
# Kumamoto
42+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.kumamoto_$TYPE -nlt multipolygon $DATA_DIR/kumamoto_$TYPE.geojson
43+
44+
# Higashi_hiroshima
45+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.higashi_hiroshima_$TYPE -nlt multipolygon $DATA_DIR/higashi_hiroshima_$TYPE.geojson
46+
47+
# Morioka
48+
ogr2ogr -f "PostgreSQL" PG:"host=localhost user=postgres dbname=postgres" -nln omf.morioka_$TYPE -nlt multipolygon $DATA_DIR/morioka_$TYPE.geojson
49+
50+
51+
# Display the measurement time.
52+
time=$SECONDS
53+
((sec=time%60, min=(time%3600)/60, hrs=time/3600))
54+
timestamp=$(printf "%d:%02d:%02d" "$hrs" "$min" "$sec")
55+
echo "Processing time is $timestamp"

‎Bash/OMF/ETL/requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
poetry

‎Bash/OMF/ETL/setup.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
# move to working directory
4+
SCRIPT_DIR=$(cd $(dirname $0); pwd)
5+
cd "${SCRIPT_DIR}"
6+
7+
# pip install
8+
pip install -r "$SCRIPT_DIR/requirements.txt"
9+
10+
# get latest resource
11+
OMF_DIR="$SCRIPT_DIR/overturemaps-py"
12+
if [ -d "$OMF_DIR" ]; then
13+
cd "$OMF_DIR"
14+
git checkout .
15+
git pull origin main
16+
cd -
17+
else
18+
git clone git@github.com:OvertureMaps/overturemaps-py.git
19+
fi

0 commit comments

Comments
 (0)