From 52b7b283cb1b602efe1d90d460fcbc4c464341b9 Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Wed, 16 Jul 2025 13:07:18 -0500 Subject: [PATCH 1/5] Apply Sentera modifications to TiTiler 0.22.4 --- .dockerignore | 1 + build-lambda.sh | 26 +++++++++++++++++++ deployment/aws/lambda/Dockerfile | 5 ++-- pyproject.toml | 12 +++++---- src/titiler/application/pyproject.toml | 8 +++--- .../application/titiler/application/main.py | 17 ++++++++++++ src/titiler/core/pyproject.toml | 2 +- src/titiler/core/titiler/core/factory.py | 13 ++++++++++ src/titiler/extensions/pyproject.toml | 4 ++- src/titiler/mosaic/pyproject.toml | 4 ++- src/titiler/xarray/pyproject.toml | 6 +++-- 11 files changed, 83 insertions(+), 15 deletions(-) create mode 100755 build-lambda.sh diff --git a/.dockerignore b/.dockerignore index bf2b7d7f9..dc9263e98 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ deployment/* docs/* deployment/aws/** +!deployment/aws/lambda/handler.py diff --git a/build-lambda.sh b/build-lambda.sh new file mode 100755 index 000000000..df710ac3b --- /dev/null +++ b/build-lambda.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script compiles the service into a Python Lambda build artifact + +set -ex + +PROJ_DIR=$(git rev-parse --show-toplevel) +BUILD_DIR=${PROJ_DIR}/build +DOCKER_TAG=titiler_build +LAMBDA_PATH=${PROJ_DIR}/package.zip + +cd ${PROJ_DIR} + +# Clean out old build relics +rm -rf ${BUILD_DIR} +mkdir -p $(dirname ${BUILD_DIR}) +rm -f ${LAMBDA_PATH} +mkdir -p $(dirname ${LAMBDA_PATH}) + +# Build Docker image and copy build assets out: +docker build --no-cache -f ${PROJ_DIR}/deployment/aws/lambda/Dockerfile -t ${DOCKER_TAG} . +docker run --rm -it -v ${BUILD_DIR}:/asset-output --entrypoint /bin/bash ${DOCKER_TAG} -c 'cp -R /asset/. /asset-output/.' + +# Package up into .zip: +cd ${BUILD_DIR} +zip -r ${LAMBDA_PATH} . diff --git a/deployment/aws/lambda/Dockerfile b/deployment/aws/lambda/Dockerfile index ad9b8be95..fcc113bd9 100644 --- a/deployment/aws/lambda/Dockerfile +++ b/deployment/aws/lambda/Dockerfile @@ -8,7 +8,8 @@ WORKDIR /tmp RUN dnf install -y gcc-c++ && dnf clean all RUN python -m pip install pip -U -RUN python -m pip install "titiler.application==0.22.4" "mangum>=0.10.0" -t /asset --no-binary pydantic +COPY . /titiler +RUN python -m pip install "/titiler/src/titiler/application" "mangum>=0.10.0" -t /asset --no-binary pydantic # Reduce package size and remove useless files RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; @@ -17,7 +18,7 @@ RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc -COPY lambda/handler.py /asset/handler.py +COPY deployment/aws/lambda/handler.py /asset/handler.py # Ref: https://github.com/developmentseed/titiler/discussions/1108#discussioncomment-13045681 RUN cp /usr/lib64/libexpat.so.1 /asset/ diff --git a/pyproject.toml b/pyproject.toml index 3df36bad8..516ea8fb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,11 +31,13 @@ classifiers = [ ] version="0.22.4" dependencies = [ - "titiler.core==0.22.4", - "titiler.xarray==0.22.4", - "titiler.extensions==0.22.4", - "titiler.mosaic==0.22.4", - "titiler.application==0.22.4", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", + "titiler.xarray @ file:///titiler/src/titiler/xarray", + "titiler.extensions @ file:///titiler/src/titiler/extensions", + "titiler.mosaic @ file:///titiler/src/titiler/mosaic", + "titiler.application @ file:///titiler/src/titiler/application", ] [project.urls] diff --git a/src/titiler/application/pyproject.toml b/src/titiler/application/pyproject.toml index f2778dea6..cf57f9031 100644 --- a/src/titiler/application/pyproject.toml +++ b/src/titiler/application/pyproject.toml @@ -31,9 +31,11 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.22.4", - "titiler.extensions[cogeo,stac]==0.22.4", - "titiler.mosaic==0.22.4", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", + "titiler.extensions[cogeo,stac] @ file:///titiler/src/titiler/extensions", + "titiler.mosaic @ file:///titiler/src/titiler/mosaic", "starlette-cramjam>=0.4,<0.5", "pydantic-settings~=2.0", ] diff --git a/src/titiler/application/titiler/application/main.py b/src/titiler/application/titiler/application/main.py index 88cc152e6..bc00733e0 100644 --- a/src/titiler/application/titiler/application/main.py +++ b/src/titiler/application/titiler/application/main.py @@ -2,6 +2,7 @@ import json import logging +import os from logging import config as log_config from typing import Annotated, Literal, Optional @@ -43,10 +44,26 @@ from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.mosaic.factory import MosaicTilerFactory +LEVELS = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} +# Remove any AWS-injected logger handlers to fix Lambda logging to CloudWatch +# https://stackoverflow.com/a/45624044 +base = logging.getLogger() +if base.handlers: + for handler in base.handlers: + base.removeHandler(handler) +logging.basicConfig(level=LEVELS.get(os.environ.get("LOGLEVEL", "info"))) logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True logging.getLogger("rio-tiler").setLevel(logging.ERROR) +logger = logging.getLogger(__name__) +logger.info("TiTiler") api_settings = ApiSettings() diff --git a/src/titiler/core/pyproject.toml b/src/titiler/core/pyproject.toml index 0d1fc08f3..60c696432 100644 --- a/src/titiler/core/pyproject.toml +++ b/src/titiler/core/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "numpy", "pydantic~=2.0", "rasterio", - "rio-tiler>=7.7,<8.0", + "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/heads/7.8.1-sentera-changes-dev.tar.gz", "morecantile", "simplejson", "typing_extensions>=4.6.1", diff --git a/src/titiler/core/titiler/core/factory.py b/src/titiler/core/titiler/core/factory.py index 1480e75b4..3c3c809b0 100644 --- a/src/titiler/core/titiler/core/factory.py +++ b/src/titiler/core/titiler/core/factory.py @@ -1,6 +1,8 @@ """TiTiler Router factories.""" import abc +import json +from base64 import b64decode from typing import ( Any, Callable, @@ -226,6 +228,11 @@ def add_route_dependencies( route.dependencies.extend(dependencies) # type: ignore +def get_feature(aoi: str) -> Feature: + """Base64 encoded GeoJSON Feature.""" + return json.loads(b64decode(aoi)) + + @define(kw_only=True) class TilerFactory(BaseFactory): """Tiler Factory. @@ -821,6 +828,10 @@ def tile( description="Row (Y) index of the tile on the selected TileMatrix. It cannot exceed the MatrixWidth-1 for the selected TileMatrix.", ), ], + aoi: Annotated[ + Union[str, None], + "Area of interest to crop the tile.", + ] = None, tileMatrixSetId: Annotated[ Literal[tuple(self.supported_tms.list())], Path( @@ -850,6 +861,7 @@ def tile( env=Depends(self.environment_dependency), ): """Create map tile from a dataset.""" + feature = get_feature(aoi) if aoi else None tms = self.supported_tms.get(tileMatrixSetId) with rasterio.Env(**env): with self.reader( @@ -860,6 +872,7 @@ def tile( y, z, tilesize=scale * 256, + aoi=feature, **tile_params.as_dict(), **layer_params.as_dict(), **dataset_params.as_dict(), diff --git a/src/titiler/extensions/pyproject.toml b/src/titiler/extensions/pyproject.toml index 0c4adb746..cacd0fc23 100644 --- a/src/titiler/extensions/pyproject.toml +++ b/src/titiler/extensions/pyproject.toml @@ -31,7 +31,9 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.22.4" + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", ] [project.optional-dependencies] diff --git a/src/titiler/mosaic/pyproject.toml b/src/titiler/mosaic/pyproject.toml index 117dfa214..e04e410eb 100644 --- a/src/titiler/mosaic/pyproject.toml +++ b/src/titiler/mosaic/pyproject.toml @@ -31,7 +31,9 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.22.4", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", "cogeo-mosaic>=8.0,<9.0", ] diff --git a/src/titiler/xarray/pyproject.toml b/src/titiler/xarray/pyproject.toml index 45ca735a4..668d8da93 100644 --- a/src/titiler/xarray/pyproject.toml +++ b/src/titiler/xarray/pyproject.toml @@ -30,8 +30,10 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.22.4", - "rio-tiler>=7.6.1,<8.0", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", + "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/heads/7.8.1-sentera-changes-dev.tar.gz", "xarray", "rioxarray", ] From 949a37bf10047ed0c05326a4e1de8f14f38c81b4 Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Fri, 18 Jul 2025 14:54:27 -0500 Subject: [PATCH 2/5] Fix aoi argument definition --- src/titiler/core/titiler/core/factory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/titiler/core/titiler/core/factory.py b/src/titiler/core/titiler/core/factory.py index 3c3c809b0..479e3e204 100644 --- a/src/titiler/core/titiler/core/factory.py +++ b/src/titiler/core/titiler/core/factory.py @@ -828,10 +828,6 @@ def tile( description="Row (Y) index of the tile on the selected TileMatrix. It cannot exceed the MatrixWidth-1 for the selected TileMatrix.", ), ], - aoi: Annotated[ - Union[str, None], - "Area of interest to crop the tile.", - ] = None, tileMatrixSetId: Annotated[ Literal[tuple(self.supported_tms.list())], Path( @@ -850,6 +846,10 @@ def tile( description="Default will be automatically defined if the output image needs a mask (png) or not (jpeg)." ), ] = None, + aoi: Annotated[ + Union[str, None], + Query(description="Area of interest to crop the tile."), + ] = None, src_path=Depends(self.path_dependency), reader_params=Depends(self.reader_dependency), tile_params=Depends(self.tile_dependency), From 4338153852c58e5e04d28316b12fe84f35dcf32a Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Tue, 5 Aug 2025 17:31:32 -0500 Subject: [PATCH 3/5] Update Dockerfile for from-source builds of PROJ, GEOS, GDAL --- deployment/aws/lambda/Dockerfile | 97 +++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 14 deletions(-) diff --git a/deployment/aws/lambda/Dockerfile b/deployment/aws/lambda/Dockerfile index fcc113bd9..4da1b857c 100644 --- a/deployment/aws/lambda/Dockerfile +++ b/deployment/aws/lambda/Dockerfile @@ -1,29 +1,98 @@ -ARG PYTHON_VERSION=3.12 +FROM --platform=linux/amd64 public.ecr.aws/lambda/python:3.12 -FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} +# Update pip: +RUN pip install pip -U +# Install system dependencies +RUN dnf install -y \ + tar \ + gzip \ + bzip2 \ + sqlite-devel \ + libtiff-devel \ + git \ + cmake \ + gcc-c++ \ + curl-devel +RUN dnf clean all + +# Build PROJ from source +WORKDIR /tmp +RUN curl -L https://download.osgeo.org/proj/proj-9.6.2.tar.gz | tar -xz +RUN mkdir proj-9.6.2/build +WORKDIR /tmp/proj-9.6.2/build +RUN cmake .. -DCMAKE_INSTALL_PREFIX=/usr +RUN cmake --build . --target install -j10 + +# Build GEOS from source +WORKDIR /tmp +RUN curl -L https://download.osgeo.org/geos/geos-3.13.1.tar.bz2 | tar -xj +RUN mkdir geos-3.13.1/build +WORKDIR /tmp/geos-3.13.1/build +RUN cmake \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DBUILD_DOCUMENTATION=OFF \ + -DBUILD_TESTING=OFF \ + .. +RUN make -j10 +RUN make install + +# Build the latest GDAL from source: WORKDIR /tmp +RUN git clone https://github.com/OSGeo/gdal.git +RUN mkdir -p gdal/build +WORKDIR /tmp/gdal/build +RUN cmake \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DGDAL_USE_GEOS=ON \ + -DGDAL_USE_CURL=ON \ + -DGDAL_USE_LIBKML=OFF \ + -DGDAL_USE_GRIB=OFF \ + -DBUILD_APPS=OFF \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + .. +RUN cmake --build . --target install -j10 -# Install system dependencies to compile (numexpr) -RUN dnf install -y gcc-c++ && dnf clean all +# Build a rasterio wheel from source, which will make use of our GDAL +# build. The wheel collects up the necessary system libraries for +# eventual bundling into a Lambda artifact. +RUN pip install build auditwheel patchelf +RUN pip wheel rasterio --no-binary rasterio -w /tmp/rasterio-build +RUN auditwheel repair /tmp/rasterio-build/rasterio-*.whl -w /tmp/rasterio-wheel +RUN pip install /tmp/rasterio-wheel/rasterio-*.whl -t /asset -RUN python -m pip install pip -U +# Build TiTiler from source: +WORKDIR /tmp COPY . /titiler -RUN python -m pip install "/titiler/src/titiler/application" "mangum>=0.10.0" -t /asset --no-binary pydantic +RUN pip install \ + # From source to pull in our tile-clipping feature: + "/titiler/src/titiler/application" \ + # mangum to host the app on AWS Lambda: + "mangum>=0.10.0" typing_extensions \ + # Build pydantic from source: + --no-binary pydantic \ + # Bundle the output into /asset: + -t /asset # Reduce package size and remove useless files -RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; -RUN cd /asset && find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf -RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f -RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf -RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc +WORKDIR /asset +RUN find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; +RUN find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +# RUN find . -type f -a -name '*.py' -print0 | xargs -0 rm -f +RUN find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf +RUN rm -rdf numpy/doc/ boto3* botocore* bin geos_license Misc -COPY deployment/aws/lambda/handler.py /asset/handler.py +# Copy bits into /asset +WORKDIR /asset +RUN mkdir -p /asset/proj +RUN cp -r /usr/share/proj/* /asset/proj/ +RUN mkdir -p /asset/gdal +RUN cp -r /usr/share/gdal/* /asset/gdal/ +COPY deployment/aws/lambda/handler.py handler.py # Ref: https://github.com/developmentseed/titiler/discussions/1108#discussioncomment-13045681 -RUN cp /usr/lib64/libexpat.so.1 /asset/ +RUN cp /usr/lib64/libexpat.so.1 . -WORKDIR /asset RUN python -c "from handler import handler; print('All Good')" CMD ["echo", "hello world"] From a805c10af5ebf0342684b19ca3f2d70eeb78699d Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Thu, 7 Aug 2025 16:03:45 -0500 Subject: [PATCH 4/5] Dockerfile cleanup --- deployment/aws/lambda/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment/aws/lambda/Dockerfile b/deployment/aws/lambda/Dockerfile index 4da1b857c..d64011ba6 100644 --- a/deployment/aws/lambda/Dockerfile +++ b/deployment/aws/lambda/Dockerfile @@ -78,11 +78,14 @@ RUN pip install \ WORKDIR /asset RUN find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; RUN find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf -# RUN find . -type f -a -name '*.py' -print0 | xargs -0 rm -f RUN find . -type d -a -name 'tests' -print0 | xargs -0 rm -rf RUN rm -rdf numpy/doc/ boto3* botocore* bin geos_license Misc # Copy bits into /asset +# NOTE: I *also* found that a couple Lambda environment variables need +# to be set for all this to work: +# - GDAL_DATA : "/var/task/gdal" +# - PROJ_DATA : "/var/task/proj" WORKDIR /asset RUN mkdir -p /asset/proj RUN cp -r /usr/share/proj/* /asset/proj/ From 0580be56c361691a700c29d3d0ef622c90832bcd Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Fri, 15 Aug 2025 11:00:26 -0500 Subject: [PATCH 5/5] Point to 7.8.1-0 release of SenteraLLC/rio-tiler --- src/titiler/core/pyproject.toml | 2 +- src/titiler/xarray/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/titiler/core/pyproject.toml b/src/titiler/core/pyproject.toml index 60c696432..a7757e6fc 100644 --- a/src/titiler/core/pyproject.toml +++ b/src/titiler/core/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "numpy", "pydantic~=2.0", "rasterio", - "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/heads/7.8.1-sentera-changes-dev.tar.gz", + "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/tags/7.8.1-0.tar.gz", "morecantile", "simplejson", "typing_extensions>=4.6.1", diff --git a/src/titiler/xarray/pyproject.toml b/src/titiler/xarray/pyproject.toml index 668d8da93..d5e8c8aa8 100644 --- a/src/titiler/xarray/pyproject.toml +++ b/src/titiler/xarray/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ # Configured for volume-mount path of Lambda Dockerfile. For local # development, update accordingly for your local filesystem path: "titiler.core @ file:///titiler/src/titiler/core", - "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/heads/7.8.1-sentera-changes-dev.tar.gz", + "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/tags/7.8.1-0.tar.gz", "xarray", "rioxarray", ]