V0.13 (#202)

* update get_table_definition to work with api updates, update airports table schema in test * back to integers for data_precision * simplify looking for table schema, return empty description dict when schema not present in api * note changes, bump version * schema is now json, fix merge error * fix empty details check * add pyproject.toml, remove setup.py and move module folder into /src * testing workflow - remove refs to setup.py, fix paths * use psycopg2 bin * add release workflow * trigger workflow * format with ruff * write test file to tmpdir * do not upload to testpypi * bc2parquet proof of concept * remove 2parquet command, enhance dump command to default to always standardize spatial types and strip redundant/unsupported columns * date
smnorris · Dec 11, 2024 · 972c652 · 972c652
1 parent 107d165
commit 972c652
Show file tree

Hide file tree

Showing 23 changed files with 421 additions and 350 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,93 @@
+name: Publish to PyPI and create release on GitHub
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/bcdata  
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+      Sign the Python 🐍 distribution 📦 with Sigstore
+      and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/gh-action-sigstore-python@v3.0.0
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        '${{ github.ref_name }}'
+        --repo '${{ github.repository }}'
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      # Upload to GitHub Release using the `gh` CLI.
+      # `dist/` contains the built packages, and the
+      # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        '${{ github.ref_name }}' dist/**
+        --repo '${{ github.repository }}'
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -6,20 +6,18 @@ on:
     paths:
       - '.github/workflows/tests.yaml'
       - 'requirements*.txt'
-      - 'setup.py'
       - 'MANIFEST.in'
       - 'pyproject.toml'
-      - 'bcdata/**'
+      - 'src/bcdata/**'
       - 'tests/**'
   pull_request:
     branches: [ main ]
     paths:
       - '.github/workflows/tests.yaml'
       - 'requirements*.txt'
-      - 'setup.py'
       - 'MANIFEST.in'
       - 'pyproject.toml'
-      - 'bcdata/**'
+      - 'src/bcdata/**'
       - 'tests/**'
 
 jobs:
@@ -29,7 +27,7 @@ jobs:
     container: ghcr.io/osgeo/gdal:ubuntu-small-${{ matrix.gdal-version }}
     services:
       postgres:
-        image: postgis/postgis:16-3.4
+        image: postgis/postgis:17-3.5
         env:
           POSTGRES_PASSWORD: postgres
         # Set health checks to wait until postgres has started
@@ -51,7 +49,11 @@ jobs:
           - python-version: '3.11'
             gdal-version: '3.7.3'
           - python-version: '3.12'
-            gdal-version: '3.9.1'
+            gdal-version: '3.9.3'
+          - python-version: '3.12'
+            gdal-version: '3.10.0'
+          - python-version: '3.13'
+            gdal-version: '3.10.0'
 
     steps:
       - uses: actions/checkout@v4
@@ -74,16 +76,14 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python${{ matrix.python-version }} -m venv testenv
-          . testenv/bin/activate
+          python${{ matrix.python-version }} -m venv .venv
+          . .venv/bin/activate
           python -m pip install --upgrade pip
-          python -m pip install -r requirements-dev.txt
-          python setup.py clean
-          python -m pip install --no-deps --force-reinstall -e .[test]
+          python -m pip install .[test]
 
       - name: Run tests
         run: |
-          . testenv/bin/activate
+          . .venv/bin/activate
           python -m pytest -v -rxXs
         env:
           DATABASE_URL: postgresql://postgres:postgres@postgres:5432/postgres
diff --git a/.gitignore b/.gitignore
@@ -58,7 +58,3 @@ target/
 
 # ignore generated file that gets manually copied into README
 cli.md
-
-# ignore linting and pre-commit config
-.pre-commit-config.yaml
-.flake8
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,11 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+    - id: ruff
+      name: lint with ruff
+    - id: ruff
+      name: sort imports with ruff
+      args: [--select, I, --fix]
+    - id: ruff-format
+      name: format with ruff
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,6 +1,12 @@
 Changes
 =======
 
+0.13.0 (2024-12-11)
+------------------
+- support Data Catalogue API changes (#188)
+- bump dependencies
+- default to slightly cleaning/standardizing features returned from bcdata.get_data() / bcdata dump
+
 0.12.3 (2024-11-12)
 ------------------
 - ensure --geometry_type option is used correctly (#198)

diff --git a/README.md b/README.md
@@ -384,13 +384,12 @@ Load data to postgres and run a spatial query:
 
 Create virtualenv and install `bcdata` in development mode:
 
-    $ mkdir bcdata_env
-    $ virtualenv bcdata_env
-    $ source bcdata_env/bin/activate
-    (bcdata_env)$ git clone git@github.com:smnorris/bcdata.git
-    (bcdata_env)$ cd bcdata
-    (bcdata_env)$ pip install -e .[test]
-    (bcdata_env)$ py.test
+    $ git clone git@github.com:smnorris/bcdata.git
+    $ cd bcdata
+    $ python -m venv .venv
+    $ source .venv/bin/activate
+    (.venv)$ pip install -e .[test]
+    (.venv)$ py.test
 
 
 ## Other implementations

diff --git a/bcdata/__init__.py b/bcdata/__init__.py
diff --git a/data/primary_keys.json b/data/primary_keys.json
@@ -1,54 +1,53 @@
 {
-    "whse_admin_boundaries.clab_indian_reserves": "clab_id",
-    "whse_admin_boundaries.clab_national_parks": "national_park_id",
-    "whse_admin_boundaries.fadm_designated_areas": "feature_id",
-    "whse_admin_boundaries.fadm_special_protection_area": "feature_id",
-    "whse_admin_boundaries.fadm_tfl_all_sp": "tfl_all_sysid",
-    "whse_basemapping.bcgs_20k_grid": "map_tile",
-    "whse_basemapping.dbm_mof_50k_grid": "map_tile",
-    "whse_basemapping.gba_local_reg_greenspaces_sp": "local_reg_greenspace_id",
-    "whse_basemapping.gba_local_reg_greenspaces_sp": "local_reg_greenspace_id",
-    "whse_basemapping.gba_railway_structure_lines_sp": "railway_structure_line_id",
-    "whse_basemapping.gba_railway_tracks_sp": "railway_track_id",
-    "whse_basemapping.gba_transmission_lines_sp": "transmission_line_id",
-    "whse_basemapping.gns_geographical_names_sp": "geographical_names_id",
-    "whse_basemapping.nts_250k_grid": "map_tile",
-    "whse_basemapping.trim_cultural_lines": "objectid",
-    "whse_basemapping.trim_cultural_points": "objectid",
-    "whse_basemapping.trim_ebm_airfields": "objectid",
-    "whse_basemapping.trim_ebm_ocean": "objectid",
-    "whse_basemapping.utmg_utm_zones_sp": "utm_zone",
-    "whse_cadastre.pmbc_parcel_fabric_poly_svw": "parcel_fabric_poly_id",
-    "whse_environmental_monitoring.envcan_hydrometric_stn_sp": "hydrometric_station_id",
-    "whse_fish.fiss_stream_sample_sites_sp": "stream_sample_site_id",
-    "whse_fish.pscis_assessment_svw": "stream_crossing_id",
-    "whse_forest_tenure.ften_managed_licence_poly_svw": "objectid",
-    "whse_forest_tenure.ften_range_poly_svw": "objectid",
-    "whse_forest_tenure.ften_recreation_poly_svw": "rmf_skey",
-    "whse_forest_vegetation.ogsr_priority_def_area_cur_sp": "ogsr_pdac_sysid",
-    "whse_forest_vegetation.rec_visual_landscape_inventory": "vli_polygon_no",
-    "whse_forest_vegetation.veg_comp_lyr_r1_poly": "feature_id",
-    "whse_forest_vegetation.veg_consolidated_cut_blocks_sp": "veg_consolidated_cut_block_id",
-    "whse_human_cultural_economic.hist_historic_environments_sp": "historic_environment_id",
-    "whse_imagery_and_base_maps.mot_road_structure_sp": "hwy_structure_class_id",
-    "whse_land_use_planning.rmp_landscape_rsrv_design_sp": "rmp_lrd_sysid",
-    "whse_land_use_planning.rmp_ogma_legal_current_svw": "legal_ogma_internal_id",
-    "whse_land_use_planning.rmp_ogma_non_legal_current_svw": "non_legal_ogma_internal_id",
-    "whse_land_use_planning.rmp_plan_legal_poly_svw": "legal_feat_id",
-    "whse_land_use_planning.rmp_plan_non_legal_poly_svw": "non_legal_feat_id",
-    "whse_land_use_planning.rmp_strgc_land_rsrce_plan_svw": "strgc_land_rsrce_plan_id",
-    "whse_legal_admin_boundaries.abms_municipalities_sp": "lgl_admin_area_id",
-    "whse_legal_admin_boundaries.wcl_conservation_areas_ngo_sp": "conservation_areas_ngo_id",
-    "whse_legal_admin_boundaries.wcl_conservation_lands_sp": "conservation_land_id",
-    "whse_mineral_tenure.mta_acquired_tenure_svw": "tenure_number_id",
-    "whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp": "og_petrlm_dev_rd_pre06_pub_id",
-    "whse_mineral_tenure.og_road_segment_permit_sp": "og_road_segment_permit_id",
-    "whse_tantalis.ta_conservancy_areas_svw": "admin_area_sid",
-    "whse_tantalis.ta_crown_tenures_svw": "objectid",
-    "whse_tantalis.ta_park_ecores_pa_svw": "admin_area_sid",
-    "whse_tantalis.ta_wildlife_mgmt_areas_svw": "admin_area_sid",
-    "whse_water_management.wls_community_ws_pub_svw": "wls_cw_sysid",
-    "whse_wildlife_management.wcp_fish_sensitive_ws_poly": "fish_sensitive_ws_poly_id",
-    "whse_wildlife_management.wcp_ungulate_winter_range_sp": "ungulate_winter_range_id",
-    "whse_wildlife_management.wcp_wildlife_habitat_area_poly": "habitat_area_id"
-}
+  "whse_admin_boundaries.clab_indian_reserves": "clab_id",
+  "whse_admin_boundaries.clab_national_parks": "national_park_id",
+  "whse_admin_boundaries.fadm_designated_areas": "feature_id",
+  "whse_admin_boundaries.fadm_special_protection_area": "feature_id",
+  "whse_admin_boundaries.fadm_tfl_all_sp": "tfl_all_sysid",
+  "whse_basemapping.bcgs_20k_grid": "map_tile",
+  "whse_basemapping.dbm_mof_50k_grid": "map_tile",
+  "whse_basemapping.gba_local_reg_greenspaces_sp": "local_reg_greenspace_id",
+  "whse_basemapping.gba_railway_structure_lines_sp": "railway_structure_line_id",
+  "whse_basemapping.gba_railway_tracks_sp": "railway_track_id",
+  "whse_basemapping.gba_transmission_lines_sp": "transmission_line_id",
+  "whse_basemapping.gns_geographical_names_sp": "geographical_names_id",
+  "whse_basemapping.nts_250k_grid": "map_tile",
+  "whse_basemapping.trim_cultural_lines": "objectid",
+  "whse_basemapping.trim_cultural_points": "objectid",
+  "whse_basemapping.trim_ebm_airfields": "objectid",
+  "whse_basemapping.trim_ebm_ocean": "objectid",
+  "whse_basemapping.utmg_utm_zones_sp": "utm_zone",
+  "whse_cadastre.pmbc_parcel_fabric_poly_svw": "parcel_fabric_poly_id",
+  "whse_environmental_monitoring.envcan_hydrometric_stn_sp": "hydrometric_station_id",
+  "whse_fish.fiss_stream_sample_sites_sp": "stream_sample_site_id",
+  "whse_fish.pscis_assessment_svw": "stream_crossing_id",
+  "whse_forest_tenure.ften_managed_licence_poly_svw": "objectid",
+  "whse_forest_tenure.ften_range_poly_svw": "objectid",
+  "whse_forest_tenure.ften_recreation_poly_svw": "rmf_skey",
+  "whse_forest_vegetation.ogsr_priority_def_area_cur_sp": "ogsr_pdac_sysid",
+  "whse_forest_vegetation.rec_visual_landscape_inventory": "vli_polygon_no",
+  "whse_forest_vegetation.veg_comp_lyr_r1_poly": "feature_id",
+  "whse_forest_vegetation.veg_consolidated_cut_blocks_sp": "veg_consolidated_cut_block_id",
+  "whse_human_cultural_economic.hist_historic_environments_sp": "historic_environment_id",
+  "whse_imagery_and_base_maps.mot_road_structure_sp": "hwy_structure_class_id",
+  "whse_land_use_planning.rmp_landscape_rsrv_design_sp": "rmp_lrd_sysid",
+  "whse_land_use_planning.rmp_ogma_legal_current_svw": "legal_ogma_internal_id",
+  "whse_land_use_planning.rmp_ogma_non_legal_current_svw": "non_legal_ogma_internal_id",
+  "whse_land_use_planning.rmp_plan_legal_poly_svw": "legal_feat_id",
+  "whse_land_use_planning.rmp_plan_non_legal_poly_svw": "non_legal_feat_id",
+  "whse_land_use_planning.rmp_strgc_land_rsrce_plan_svw": "strgc_land_rsrce_plan_id",
+  "whse_legal_admin_boundaries.abms_municipalities_sp": "lgl_admin_area_id",
+  "whse_legal_admin_boundaries.wcl_conservation_areas_ngo_sp": "conservation_areas_ngo_id",
+  "whse_legal_admin_boundaries.wcl_conservation_lands_sp": "conservation_land_id",
+  "whse_mineral_tenure.mta_acquired_tenure_svw": "tenure_number_id",
+  "whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp": "og_petrlm_dev_rd_pre06_pub_id",
+  "whse_mineral_tenure.og_road_segment_permit_sp": "og_road_segment_permit_id",
+  "whse_tantalis.ta_conservancy_areas_svw": "admin_area_sid",
+  "whse_tantalis.ta_crown_tenures_svw": "objectid",
+  "whse_tantalis.ta_park_ecores_pa_svw": "admin_area_sid",
+  "whse_tantalis.ta_wildlife_mgmt_areas_svw": "admin_area_sid",
+  "whse_water_management.wls_community_ws_pub_svw": "wls_cw_sysid",
+  "whse_wildlife_management.wcp_fish_sensitive_ws_poly": "fish_sensitive_ws_poly_id",
+  "whse_wildlife_management.wcp_ungulate_winter_range_sp": "ungulate_winter_range_id",
+  "whse_wildlife_management.wcp_wildlife_habitat_area_poly": "habitat_area_id"
+}
diff --git a/data/validate.py b/data/validate.py
@@ -4,7 +4,6 @@
 
 import bcdata
 
-
 LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s: %(message)s"
 
 with open("primary_keys.json", "r") as file:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,7 +4,6 @@

		import bcdata


		LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s: %(message)s"

		with open("primary_keys.json", "r") as file:
Expand Down