Feat/sync script

open_buildings/overture/add_columns.py

@@ -4,15 +4,17 @@
 # parquet to geoparquet.
 
 
+import glob
 import os
-import duckdb
-import time
-import tempfile
+import shutil
 import subprocess
-import glob
-from duckdb.typing import *
+import tempfile
+import time
+
+import duckdb
 import mercantile
-import shutil
+from duckdb.typing import *
+
 
 def lat_lon_to_quadkey(lat: DOUBLE, lon: DOUBLE, level: INTEGER) -> VARCHAR:
     # Convert latitude and longitude to tile using mercantile
@@ -44,6 +46,7 @@ def add_quadkey(con):
     );
     """)
 
+
 def add_country_iso(con, country_parquet_path):
     # Load country parquet file into duckdb
     con.execute(f"CREATE TABLE countries AS SELECT * FROM read_parquet('{country_parquet_path}')")
@@ -88,8 +91,8 @@ def process_parquet_file(input_parquet_path, output_folder, country_parquet_path
 
     con.execute('LOAD spatial;')
 
-    # Load parquet file into duckdb
-    con.execute(f"CREATE TABLE buildings AS SELECT * FROM read_parquet('{input_parquet_path}')")
+    # NOTE: exclude names column because it's all NULL and causes InternalException: INTERNAL Error: Attempted to dereference unique_ptr that is NULL!
+    con.execute(f"CREATE OR REPLACE TABLE buildings AS SELECT * EXCLUDE(names) FROM read_parquet('{input_parquet_path}')")
 
     if add_quadkey_option:
         add_quadkey(con)
@@ -126,7 +129,14 @@ def process_parquet_files(input_path, output_folder, country_parquet_path, overw
         process_parquet_file(input_path, output_folder, country_parquet_path, overwrite, add_quadkey_option, add_country_iso_option, verbose)
 
 # Call the function - uncomment if you want to call this directly from python and put values in here.
+import pathlib
+
+release_version = "overture_02-15"  # Example version, adjust as necessary
+data_dir = pathlib.Path.home() / "data" / "src" / f"{release_version}" / "theme=buildings" / "type=building"
+out_dir = pathlib.Path.home() / "data" / "prc" / f"{release_version}" / "theme=buildings" / "type=building"
+
+input_path = data_dir / "part-00041-a34b09ea-399f-4872-b0b1-084a81bbb42f-c000.zstd.parquet"
 #input_path = '/Volumes/fastdata/overture/s3-data/buildings/'
 #output_folder = '/Volumes/fastdata/overture/refined-parquet/'
 #country_parquet_path = '/Volumes/fastdata/overture/countries.parquet'
-#process_parquet_files(input_path, output_folder, country_parquet_path, overwrite=False, add_quadkey_option=True, add_country_iso_option=True)
+process_parquet_files(input_path, out_dir, "", overwrite=False, add_quadkey_option=True, add_country_iso_option=False)

scripts/bash/sync-overture-to-local.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+DEFAULT_RELEASE="2024-02-15-alpha.0"
+DEFAULT_DESTINATION="$HOME/data/src/overture/$DEFAULT_RELEASE"
+
+while getopts ":d:r:" opt; do
+    case ${opt} in
+    d) # Process option for the destination
+        DESTINATION=$OPTARG
+        ;;
+    r) # Process option for the release
+        RELEASE=$OPTARG
+        ;;
+    \?)
+        echo "Usage: cmd [-d destination] [-r release]"
+        ;;
+    esac
+done
+
+# Set the destination directory based on the provided destination or release argument
+DESTINATION="${DESTINATION:-$DEFAULT_DESTINATION}"
+RELEASE="${RELEASE:-$DEFAULT_RELEASE}"
+
+mkdir -p "${DESTINATION}"
+cd "${DESTINATION}"
+
+aws s3 sync --no-sign-request "s3://overturemaps-us-west-2/release/${RELEASE}/" .
+
+# Verification step to ensure all files are transferred correctly
+# This is a simple re-sync operation; any missing or incomplete files will be re-downloaded
+echo "Verifying file transfer..."
+aws s3 sync --no-sign-request "s3://overturemaps-us-west-2/release/${RELEASE}/" . --dryrun