Merge branch 'main' into task/1379

.github/actions/setup_env/action.yml

@@ -64,7 +64,9 @@ runs:
       with:
         environment-file: environment.yml
         environment-name: env
-        channels: conda-forge
+        condarc: |
+              channels:
+                - conda-forge
         init-shell: bash
         cache-environment: true
         cache-environment-key: ${{ runner.os }}${{ runner.arch }}-${{ env.WEEK }}-${{ hashFiles('environment.yml') }}

.github/workflows/build-and-test.yml

@@ -21,7 +21,7 @@ env:
   # nomkl: make sure numpy w/out mkl
   # setuptools_scm: needed for versioning to work
   CONDA_DEFAULT_DEPENDENCIES: python-build nomkl setuptools_scm
-  LATEST_SUPPORTED_PYTHON_VERSION: "3.10"
+  LATEST_SUPPORTED_PYTHON_VERSION: "3.11"
 
 jobs:
   check-syntax-errors:
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.8, 3.9, "3.10"]
+        python-version: [3.8, 3.9, "3.10", "3.11"]
         os: [windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v3
@@ -160,7 +160,7 @@ jobs:
     needs: check-syntax-errors
     strategy:
       matrix:
-        python-version: [3.8, 3.9, "3.10"]
+        python-version: [3.8, 3.9, "3.10", "3.11"]
         os: [windows-latest, macos-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/build-docker.yml

@@ -0,0 +1,73 @@
+name: Build containers
+
+on:
+    workflow_dispatch:
+    push:
+        paths:
+            - 'docker/*'
+            - '.github/workflows/build-docker.yml'
+            - 'src/**/*'
+            - 'requirements*.txt'
+            - 'pyproject.toml'
+            - 'setup.py'
+            - 'scripts/convert-requirements-to-conda-yml.py'
+
+concurrency:
+    # make sure only one run of this workflow for a given PR or a given branch
+    # can happen at one time. previous queued or started runs will be cancelled.
+    # github.workflow is the workflow name
+    # github.ref is the ref that triggered the workflow run
+    # on push, this is refs/heads/<branch name>
+    # on pull request, this is refs/pull/<pull request number>/merge
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+defaults:
+    run:
+        shell: bash -l {0}
+
+env:
+    # Customize this name if needed.
+    # The repo name is a very reasonable default!
+    CONTAINER_NAME: invest
+
+jobs:
+    build:
+        name: Build containers
+        runs-on: ubuntu-latest
+        permissions:
+            packages: write
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: login to GitHub Container Registry
+              uses: docker/login-action@v2
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Create environment file
+              run: |
+                  python scripts/convert-requirements-to-conda-yml.py requirements.txt > docker/environment.yml
+
+            - name: Build docker
+              run: |
+                  # Replace / (invalid tag character) with .
+                  SANITIZED_REF="$(echo ${{github.ref_name}} | sed 's|/|.|g')"
+                  cd docker && docker build \
+                        --build-arg="INVEST_REPO=${{ github.repository }}" \
+                        --build-arg="INVEST_VERSION=${{ github.sha }}" \
+                        -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest \
+                        -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${{ github.sha }} \
+                        -t ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:${SANITIZED_REF} \
+                        .
+
+            - name: Test that GDAL and InVEST import
+              run: |
+                  docker run --rm ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest python -c "from osgeo import gdal"
+                  docker run --rm ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}:latest python -m natcap.invest --version
+
+            - name: Push docker
+              if: github.event_name != 'pull_request'
+              run: docker image push --all-tags ghcr.io/${{ github.repository_owner }}/${{ env.CONTAINER_NAME }}

.readthedocs_environment.yml

@@ -10,8 +10,8 @@ channels:
 # https://github.com/conda/conda/issues/5003
 - nodefaults
 dependencies:
-- python=3.9
-- gdal>=3.4.2
+- python=3.11
+- gdal>=3.4.2,<3.6.0
 - pip
 - pip:
   - -r requirements.txt

HISTORY.rst

@@ -41,6 +41,42 @@ Unreleased Changes
     * In advance of the numpy 2.0 release, function calls to ``numpy.product``
       have been replaced with ``numpy.prod``.
       https://github.com/natcap/invest/issues/1410
+    * Add support for python 3.11 (`#1103 <https://github.com/natcap/invest/issues/1103>`_)
+    * Adding a docker container that is built on each commit where a change to
+      model code, requirements, or the docker configuration has been made.
+      https://github.com/natcap/invest/issues/1115
+    * Vector geometry types will now be validated for all models
+      (`#1374 <https://github.com/natcap/invest/issues/1374>`_)
+    * Datastack archives will now be correctly extracted
+      (`#1308 <https://github.com/natcap/invest/issues/1308>`_)
+* NDR
+    * Fixing an issue where minor geometric issues in the watersheds input
+      (such as a ring self-intersection) would raise an error in the model.
+      https://github.com/natcap/invest/issues/1412
+    * Fixed a task dependency issue where NDR would crash because of a race
+      condition when run with ``n_workers > 0``.
+      https://github.com/natcap/invest/issues/1426
+    * Fixed an issue in NDR's effective retention where, on rasters with more
+      than 2^31 pixels, the model would crash with an error relating to a
+      negative (overflowed) index. https://github.com/natcap/invest/issues/1431
+* SDR
+    * RKLS, USLE, avoided erosion, and avoided export rasters will now have
+      nodata in streams (`#1415 <https://github.com/natcap/invest/issues/1415>`_)
+    * Fixed an issue in SDR's sediment deposition where, on rasters with more
+      than 2^31 pixels, the model would crash with an error relating to a
+      negative (overflowed) index. https://github.com/natcap/invest/issues/1431
+* Seasonal Water Yield
+    * Fixed an issue in Seasonal Water Yield's baseflow routing and local
+      recharge functions where, on rasters with more than 2^31 pixels, the
+      model would crash with an error relating to a negative (overflowed)
+      index. https://github.com/natcap/invest/issues/1431
+* Urban Cooling
+    * Fixed a bug where model would error out if any feature in the buildings
+      vector was missing a geometry; now they will be skipped
+      (`#1401 <https://github.com/natcap/invest/issues/1401>`_)
+* Wind Energy
+    * Fixed a bug where model would error when the grid points path was empty
+      (`#1417 <https://github.com/natcap/invest/issues/1417>`_)
 
 
 3.14.0 (2023-09-08)

Makefile

@@ -6,7 +6,7 @@ GIT_SAMPLE_DATA_REPO_REV    := 2e7cd618c661ec3f3b2a3bddfd2ce7d4704abc05
 
 GIT_TEST_DATA_REPO          := https://bitbucket.org/natcap/invest-test-data.git
 GIT_TEST_DATA_REPO_PATH     := $(DATA_DIR)/invest-test-data
-GIT_TEST_DATA_REPO_REV      := f15ac3e0a17c446e5a4bec4ccd50ce8cec359595
+GIT_TEST_DATA_REPO_REV      := da013683e80ea094fbb2309197e2488c02794da8
 
 GIT_UG_REPO                 := https://github.com/natcap/invest.users-guide
 GIT_UG_REPO_PATH            := doc/users-guide

docker/.gitignore

@@ -0,0 +1 @@
+environment.yml

docker/Dockerfile

@@ -0,0 +1,26 @@
+# syntax=docker/dockerfile:1
+
+# Build the InVEST wheel in a separate container stage
+FROM debian:12.2 as build
+ARG INVEST_VERSION="main"
+ARG INVEST_REPO="natcap/invest"
+RUN apt update && apt install -y python3 python3-dev python3-pip python3-build build-essential git python3.11-venv
+RUN cd / && \
+        git clone https://github.com/${INVEST_REPO}.git && \
+        cd $(basename ${INVEST_REPO}) && \
+        git checkout ${INVEST_VERSION} && \
+        python3 -m build
+
+# Create the container for distribution that has runtime dependencies.
+FROM mambaorg/micromamba:1.5.0-bookworm-slim
+COPY --from=build /invest/dist/*.whl /tmp/
+
+# The environment.yml file will be built during github actions.
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml
+RUN micromamba install -y -n base -c conda-forge -f /tmp/environment.yml && \
+        micromamba clean --all --yes && \
+        /opt/conda/bin/python -m pip install /tmp/*.whl && \
+        /opt/conda/bin/python -m pip cache purge && \
+        micromamba remove -y -n base cxx-compiler git
+
+ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"]

pyproject.toml

@@ -2,7 +2,7 @@
 name = "natcap.invest"
 description = "InVEST Ecosystem Service models"
 readme = "README_PYTHON.rst"
-requires-python = ">=3.8,<3.11"
+requires-python = ">=3.8,<3.12"
 license = {file = "LICENSE.txt"}
 maintainers = [
     {name = "Natural Capital Project Software Team"}
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Programming Language :: Cython",
     "License :: OSI Approved :: BSD License",
     "Topic :: Scientific/Engineering :: GIS"
@@ -43,7 +44,7 @@ invest = "natcap.invest.cli:main"
 # that we can provide a much easier build experience so long as GDAL is
 # available at runtime.
 requires = [
-    'setuptools>=45', 'wheel', 'setuptools_scm>=6.4.0', 'cython', 'babel',
+    'setuptools>=61', 'wheel', 'setuptools_scm>=8.0', 'cython', 'babel',
     'oldest-supported-numpy'
 ]
 build-backend = "setuptools.build_meta"

requirements.txt

@@ -10,18 +10,19 @@
 # scripts/convert-requirements-to-conda-yml.py as though it can only be found
 # on pip.
 
-GDAL>=3.4.2,<3.7.0
+GDAL>=3.4.2,<3.6.0
 Pyro4==4.77  # pip-only
 pandas>=1.2.1
 numpy>=1.11.0,!=1.16.0
 Rtree>=0.8.2,!=0.9.1
-shapely>=1.7.1,<1.8.2  # https://github.com/shapely/shapely/issues/1385
+shapely>=2.0.0
 scipy>=1.9.0
-pygeoprocessing==2.4.0  # pip-only
-taskgraph[niced_processes]>=0.11.0  # pip-only
+pygeoprocessing==2.4.0
+taskgraph>=0.11.0
 psutil>=5.6.6
 chardet>=3.0.4
 pint
 Babel
 Flask
 flask_cors
+requests

scripts/convert-requirements-to-conda-yml.py

@@ -1,8 +1,9 @@
 # encoding=UTF-8
 """convert-requirements-to-conda-yml.py"""
 
-import sys
 import argparse
+import platform
+import sys
 
 YML_TEMPLATE = """channels:
 - conda-forge
@@ -46,8 +47,7 @@ def build_environment_from_requirements(cli_args):
     requirements_files = args.req
 
     pip_requirements = set()
-    # conda likes it when you list pip if you're using pip.
-    conda_requirements = set(['pip'])
+    conda_requirements = set()
     for requirement_file in requirements_files:
         with open(requirement_file) as file:
             for line in file:
@@ -58,7 +58,37 @@ def build_environment_from_requirements(cli_args):
                     continue
 
                 if line.endswith('# pip-only'):
+                    # Conda prefers that we explicitly include pip as a
+                    # requirement if we're using pip.
+                    conda_requirements.add('pip')
+
                     pip_requirements.add(line)
+
+                    # If an scm needs to be installed for pip to clone to a
+                    # revision, add it to the conda package list.
+                    #
+                    # Bazaar (bzr, which pip supports) is not listed;
+                    # deprecated as of 2016 and not available on conda-forge.
+                    install_compiler = False
+                    for prefix, scm_conda_pkg in [("git+", "git"),
+                                                  ("hg+", "mercurial"),
+                                                  ("svn+", "subversion")]:
+                        if line.startswith(prefix):
+                            conda_requirements.add(scm_conda_pkg)
+                            install_compiler = True
+                            break  # The line can only match 1 prefix
+
+                    # It's less common (like for pygeoprocessing) to have linux
+                    # wheels.  Install a compiler if we're on linux, to be
+                    # safe.
+                    if platform.system() == 'Linux' or install_compiler:
+                        # Always make the compiler available
+                        # cxx-compiler works on all OSes.
+                        # NOTE: do not use this as a dependency in a
+                        # conda-forge package recipe!
+                        # https://anaconda.org/conda-forge/cxx-compiler
+                        conda_requirements.add('cxx-compiler')
+
                 else:
                     conda_requirements.add(line)