From 93b8b73bd2bdf316d3d807f6efb274a62598bb88 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:02:14 -0400 Subject: [PATCH 01/14] action: change tile-join/pmtiles flow Let's try to do this in one pass, without the intermediate step of publishing the pmtiles to R2. --- .github/workflows/tile-join.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 6a20115..dd979c1 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,31 +7,29 @@ jobs: build: runs-on: ubuntu-latest env: - AREA: north-america + # AREA: north-america + AREA: nova-scotia steps: - - name: Maximize build space - uses: AdityaGarg8/remove-unwanted-software@v2 - with: - remove-android: 'true' - remove-dotnet: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' +# - name: Maximize build space +# uses: AdityaGarg8/remove-unwanted-software@v2 +# with: +# remove-android: 'true' +# remove-dotnet: 'true' +# remove-haskell: 'true' +# remove-codeql: 'true' +# remove-docker-images: 'true' # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - - name: Fetch binaries - run: sudo mkdir /mnt/layers && sudo chown runner:docker /mnt/layers && ./bin/fetch-binaries - - name: Fetch tiles env: AWS_ENDPOINT_URL: https://ce5627cfc65c8b54d40bb91c0ba7298b.r2.cloudflarestorage.com AWS_EC2_METADATA_DISABLED: true AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: cd /mnt/layers && aws s3 sync s3://hikes-public/layers/"$AREA"/ . --no-progress + run: sudo mkdir /mnt/layers && sudo chown runner:docker /mnt/layers && cd /mnt/layers && aws s3 sync s3://hikes-public/layers/"$AREA"/ . --no-progress - name: tile-join run: /tmp/tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles From 6dda5b4f81a4558ad2a0d5bcadbd54ea44bfe678 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:09:22 -0400 Subject: [PATCH 02/14] use dockerized tile-join --- .github/workflows/tile-join.yml | 2 +- bin/tile-join-docker | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 bin/tile-join-docker diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index dd979c1..26f1fdc 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -32,7 +32,7 @@ jobs: run: sudo mkdir /mnt/layers && sudo chown runner:docker /mnt/layers && cd /mnt/layers && aws s3 sync s3://hikes-public/layers/"$AREA"/ . --no-progress - name: tile-join - run: /tmp/tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + run: ./bin/tile-join-docker --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles - name: Upload tiles env: diff --git a/bin/tile-join-docker b/bin/tile-join-docker new file mode 100755 index 0000000..575dacc --- /dev/null +++ b/bin/tile-join-docker @@ -0,0 +1,9 @@ +#!/bin/sh + +exec docker run \ + -it \ + --rm \ + --entrypoint /usr/bin/tile-join \ + -w $(readlink -f .) \ + -v $(git rev-parse --show-toplevel):$(git rev-parse --show-toplevel) \ + versatiles/tippecanoe From 3ded42d105abb5196cad703dbfbad7f26afb4b18 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:10:25 -0400 Subject: [PATCH 03/14] remove -it flag --- bin/tile-join-docker | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/tile-join-docker b/bin/tile-join-docker index 575dacc..95208ce 100755 --- a/bin/tile-join-docker +++ b/bin/tile-join-docker @@ -1,7 +1,6 @@ #!/bin/sh exec docker run \ - -it \ --rm \ --entrypoint /usr/bin/tile-join \ -w $(readlink -f .) \ From 88f0fc8822a33abfbbe33995a56ca546cb5bdf58 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:11:02 -0400 Subject: [PATCH 04/14] pass args --- bin/tile-join-docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/tile-join-docker b/bin/tile-join-docker index 95208ce..025c735 100755 --- a/bin/tile-join-docker +++ b/bin/tile-join-docker @@ -5,4 +5,4 @@ exec docker run \ --entrypoint /usr/bin/tile-join \ -w $(readlink -f .) \ -v $(git rev-parse --show-toplevel):$(git rev-parse --show-toplevel) \ - versatiles/tippecanoe + versatiles/tippecanoe "$@" From 683201b59b8517a90cc2cca3fa065f8e59c3d974 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:12:07 -0400 Subject: [PATCH 05/14] bind mount /mnt/layers --- bin/tile-join-docker | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/tile-join-docker b/bin/tile-join-docker index 025c735..d0e246c 100755 --- a/bin/tile-join-docker +++ b/bin/tile-join-docker @@ -5,4 +5,5 @@ exec docker run \ --entrypoint /usr/bin/tile-join \ -w $(readlink -f .) \ -v $(git rev-parse --show-toplevel):$(git rev-parse --show-toplevel) \ + -v /mnt/layers:/mnt/layers \ versatiles/tippecanoe "$@" From 1e32765a6cc1812036629beaf741e213e116ec94 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:21:37 -0400 Subject: [PATCH 06/14] convert to pmtiles also rename the docker bin scripts to have a common `docker-` prefix --- .github/workflows/tile-join.yml | 7 +++++-- .github/workflows/tilemaker-exports.yml | 2 +- .github/workflows/tilemaker-layers.yml | 2 +- bin/docker-pmtiles | 8 ++++++++ bin/{tile-join-docker => docker-tile-join} | 0 bin/{tilemaker-docker => docker-tilemaker} | 0 6 files changed, 15 insertions(+), 4 deletions(-) create mode 100755 bin/docker-pmtiles rename bin/{tile-join-docker => docker-tile-join} (100%) rename bin/{tilemaker-docker => docker-tilemaker} (100%) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 26f1fdc..af4d544 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -32,7 +32,10 @@ jobs: run: sudo mkdir /mnt/layers && sudo chown runner:docker /mnt/layers && cd /mnt/layers && aws s3 sync s3://hikes-public/layers/"$AREA"/ . --no-progress - name: tile-join - run: ./bin/tile-join-docker --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + + - name: convert to pmtiles + run: rm -f /mnt/layers/*.mbtiles && ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles - name: Upload tiles env: @@ -40,4 +43,4 @@ jobs: AWS_EC2_METADATA_DISABLED: true AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: aws s3 cp --no-progress tiles.mbtiles s3://hikes-public/"$AREA".mbtiles + run: aws s3 cp --no-progress /mnt/layers/tiles.pmtiles s3://hikes-public/"$AREA".pmtiles diff --git a/.github/workflows/tilemaker-exports.yml b/.github/workflows/tilemaker-exports.yml index ca32045..3ea8dd0 100644 --- a/.github/workflows/tilemaker-exports.yml +++ b/.github/workflows/tilemaker-exports.yml @@ -21,7 +21,7 @@ jobs: - name: Export env: - TILEMAKER: ${{ github.workspace }}/bin/tilemaker-docker + TILEMAKER: ${{ github.workspace }}/bin/docker-tilemaker run: ./export/export-all $AREA.pbf - name: Upload GeoJSON diff --git a/.github/workflows/tilemaker-layers.yml b/.github/workflows/tilemaker-layers.yml index 019a360..cd827a1 100644 --- a/.github/workflows/tilemaker-layers.yml +++ b/.github/workflows/tilemaker-layers.yml @@ -60,7 +60,7 @@ jobs: - name: Build layer env: - TILEMAKER: ${{ github.workspace }}/bin/tilemaker-docker + TILEMAKER: ${{ github.workspace }}/bin/docker-tilemaker LAYER: ${{ matrix.layer }} run: bun mapt build "$AREA".pbf $LAYER --mbtiles -- --store /mnt/store --fast $BBOX diff --git a/bin/docker-pmtiles b/bin/docker-pmtiles new file mode 100755 index 0000000..9d051c3 --- /dev/null +++ b/bin/docker-pmtiles @@ -0,0 +1,8 @@ +#!/bin/sh + +exec docker run \ + --rm \ + -w $(readlink -f .) \ + -v $(git rev-parse --show-toplevel):$(git rev-parse --show-toplevel) \ + -v /mnt/layers:/mnt/layers \ + protomaps/go-pmtiles:main "$@" diff --git a/bin/tile-join-docker b/bin/docker-tile-join similarity index 100% rename from bin/tile-join-docker rename to bin/docker-tile-join diff --git a/bin/tilemaker-docker b/bin/docker-tilemaker similarity index 100% rename from bin/tilemaker-docker rename to bin/docker-tilemaker From 0473cc647c6be78dec278b8b9bf55386c26d4026 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 19:31:02 -0400 Subject: [PATCH 07/14] try north america --- .github/workflows/tile-join.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index af4d544..2f88cba 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,18 +7,18 @@ jobs: build: runs-on: ubuntu-latest env: - # AREA: north-america - AREA: nova-scotia + AREA: north-america + #AREA: nova-scotia steps: -# - name: Maximize build space -# uses: AdityaGarg8/remove-unwanted-software@v2 -# with: -# remove-android: 'true' -# remove-dotnet: 'true' -# remove-haskell: 'true' -# remove-codeql: 'true' -# remove-docker-images: 'true' + - name: Maximize build space + uses: AdityaGarg8/remove-unwanted-software@v2 + with: + remove-android: 'true' + remove-dotnet: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 From a20ea530490eb632cc3f348a69b551218f9695a3 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 21:57:46 -0400 Subject: [PATCH 08/14] debug --- .github/workflows/tile-join.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 2f88cba..478a4cf 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest env: - AREA: north-america - #AREA: nova-scotia + #AREA: north-america + AREA: nova-scotia steps: - name: Maximize build space @@ -35,7 +35,7 @@ jobs: run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles - name: convert to pmtiles - run: rm -f /mnt/layers/*.mbtiles && ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles + run: df -h && rm -f /mnt/layers/*.mbtiles && ls -l /mnt/layers && ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles - name: Upload tiles env: From e75f9c4479b9f02b0d8c0b9058fdc5613ac3714f Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 22:07:44 -0400 Subject: [PATCH 09/14] debug --- .github/workflows/tile-join.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 478a4cf..8460917 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -34,8 +34,11 @@ jobs: - name: tile-join run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + - name: Free up space + run: df -h && rm -f /mny/layers/*.mbtiles && ls -l /mnt/layers && df -h + - name: convert to pmtiles - run: df -h && rm -f /mnt/layers/*.mbtiles && ls -l /mnt/layers && ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles + run: ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles - name: Upload tiles env: From d1d8029fc6774c4f20cfd4183fd45e9bfa7ce6f9 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 22:12:24 -0400 Subject: [PATCH 10/14] debug --- .github/workflows/tile-join.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 8460917..74a8648 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -35,7 +35,7 @@ jobs: run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles - name: Free up space - run: df -h && rm -f /mny/layers/*.mbtiles && ls -l /mnt/layers && df -h + run: df -h && rm -f /mnt/layers/*.mbtiles && ls -l /mnt/layers && df -h - name: convert to pmtiles run: ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles From b3f8d82d5958323660ecc94ac73ed4c93ebef03d Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Fri, 4 Oct 2024 22:33:45 -0400 Subject: [PATCH 11/14] try north america --- .github/workflows/tile-join.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 74a8648..e87b8b9 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest env: - #AREA: north-america - AREA: nova-scotia + AREA: north-america + #AREA: nova-scotia steps: - name: Maximize build space From c0afe68aab203919c54a6c69f717cf1bf04fd002 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Sat, 5 Oct 2024 01:12:15 -0400 Subject: [PATCH 12/14] set --tmpdir --- .github/workflows/tile-join.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index e87b8b9..9123115 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -38,7 +38,7 @@ jobs: run: df -h && rm -f /mnt/layers/*.mbtiles && ls -l /mnt/layers && df -h - name: convert to pmtiles - run: ./bin/docker-pmtiles convert tiles.mbtiles /mnt/layers/tiles.pmtiles + run: ./bin/docker-pmtiles convert --tmpdir /mnt/layers tiles.mbtiles /mnt/layers/tiles.pmtiles - name: Upload tiles env: From ab97abcaf4ab9f5350e7a303ac1d230114736e04 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Sat, 5 Oct 2024 16:11:29 -0400 Subject: [PATCH 13/14] use docker-tile-smush --- .github/workflows/tile-join.yml | 9 +++++---- bin/docker-tile-smush | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100755 bin/docker-tile-smush diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 9123115..7069faa 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest env: - AREA: north-america - #AREA: nova-scotia + #AREA: north-america + AREA: nova-scotia steps: - name: Maximize build space @@ -32,13 +32,14 @@ jobs: run: sudo mkdir /mnt/layers && sudo chown runner:docker /mnt/layers && cd /mnt/layers && aws s3 sync s3://hikes-public/layers/"$AREA"/ . --no-progress - name: tile-join - run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + #run: ./bin/docker-tile-join --no-tile-size-limit -o tiles.mbtiles /mnt/layers/*.mbtiles + run: ./bin/docker-tile-smush /mnt/layers/*.mbtiles - name: Free up space run: df -h && rm -f /mnt/layers/*.mbtiles && ls -l /mnt/layers && df -h - name: convert to pmtiles - run: ./bin/docker-pmtiles convert --tmpdir /mnt/layers tiles.mbtiles /mnt/layers/tiles.pmtiles + run: ./bin/docker-pmtiles convert --tmpdir /mnt/layers merged.mbtiles /mnt/layers/tiles.pmtiles - name: Upload tiles env: diff --git a/bin/docker-tile-smush b/bin/docker-tile-smush new file mode 100755 index 0000000..cedf3cc --- /dev/null +++ b/bin/docker-tile-smush @@ -0,0 +1,11 @@ +#!/bin/sh + +# An alternative to tile-join. It's special cased for our scenario, +# and is faster. It will explode if the input mbtiles have overlapping +# layers, though. +exec docker run \ + --rm \ + -w $(readlink -f .) \ + -v $(git rev-parse --show-toplevel):$(git rev-parse --show-toplevel) \ + -v /mnt/layers:/mnt/layers \ + ghcr.io/hikeratlas/tile-smush:master "$@" From df7475d784d7f9d22caaa0b2089d453f35486e45 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Sat, 5 Oct 2024 16:15:40 -0400 Subject: [PATCH 14/14] back to na --- .github/workflows/tile-join.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tile-join.yml b/.github/workflows/tile-join.yml index 7069faa..abe5d2c 100644 --- a/.github/workflows/tile-join.yml +++ b/.github/workflows/tile-join.yml @@ -7,8 +7,8 @@ jobs: build: runs-on: ubuntu-latest env: - #AREA: north-america - AREA: nova-scotia + AREA: north-america + #AREA: nova-scotia steps: - name: Maximize build space