From 870c8d6ad3075d915a5e3ae86437c9d48472ede5 Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Thu, 28 Sep 2023 16:03:41 -0500 Subject: [PATCH 1/4] Re-apply AOI clipping and non-root route features to latest release --- .dockerignore | 1 + build-lambda.sh | 26 +++++++++ deployment/aws/lambda/Dockerfile | 10 +++- pyproject.toml | 10 ++-- src/titiler/application/pyproject.toml | 8 ++- .../application/titiler/application/main.py | 55 ++++++++++++++----- .../titiler/application/settings.py | 1 + src/titiler/core/pyproject.toml | 2 +- src/titiler/core/titiler/core/factory.py | 13 +++++ src/titiler/extensions/pyproject.toml | 4 +- src/titiler/mosaic/pyproject.toml | 4 +- 11 files changed, 108 insertions(+), 26 deletions(-) create mode 100755 build-lambda.sh diff --git a/.dockerignore b/.dockerignore index bf2b7d7f9..dc9263e98 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,4 @@ deployment/* docs/* deployment/aws/** +!deployment/aws/lambda/handler.py diff --git a/build-lambda.sh b/build-lambda.sh new file mode 100755 index 000000000..df710ac3b --- /dev/null +++ b/build-lambda.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# This script compiles the service into a Python Lambda build artifact + +set -ex + +PROJ_DIR=$(git rev-parse --show-toplevel) +BUILD_DIR=${PROJ_DIR}/build +DOCKER_TAG=titiler_build +LAMBDA_PATH=${PROJ_DIR}/package.zip + +cd ${PROJ_DIR} + +# Clean out old build relics +rm -rf ${BUILD_DIR} +mkdir -p $(dirname ${BUILD_DIR}) +rm -f ${LAMBDA_PATH} +mkdir -p $(dirname ${LAMBDA_PATH}) + +# Build Docker image and copy build assets out: +docker build --no-cache -f ${PROJ_DIR}/deployment/aws/lambda/Dockerfile -t ${DOCKER_TAG} . +docker run --rm -it -v ${BUILD_DIR}:/asset-output --entrypoint /bin/bash ${DOCKER_TAG} -c 'cp -R /asset/. /asset-output/.' + +# Package up into .zip: +cd ${BUILD_DIR} +zip -r ${LAMBDA_PATH} . diff --git a/deployment/aws/lambda/Dockerfile b/deployment/aws/lambda/Dockerfile index f459fd58e..159b58c60 100644 --- a/deployment/aws/lambda/Dockerfile +++ b/deployment/aws/lambda/Dockerfile @@ -1,11 +1,15 @@ -ARG PYTHON_VERSION=3.11 +ARG PYTHON_VERSION=3.10 FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} WORKDIR /tmp RUN pip install pip -U -RUN pip install "titiler.application==0.14.1" "mangum>=0.10.0" -t /asset --no-binary pydantic +# Install TiTiler from local source. This is necessary because +# pyproject.toml refers to TiTiler dependencies (core, extensions, +# mosaic, and application) that also use filesystem paths. +COPY . /titiler +RUN pip install "/titiler/src/titiler/application" "mangum>=0.10.0" -t /asset --no-binary pydantic # Reduce package size and remove useless files RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; @@ -14,6 +18,6 @@ RUN cd /asset && find . -type f -a -name '*.py' -print0 | xargs -0 rm -f RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf RUN rm -rdf /asset/numpy/doc/ /asset/boto3* /asset/botocore* /asset/bin /asset/geos_license /asset/Misc -COPY lambda/handler.py /asset/handler.py +COPY deployment/aws/lambda/handler.py /asset/handler.py CMD ["echo", "hello world"] diff --git a/pyproject.toml b/pyproject.toml index dae1df667..5601f53da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,10 +29,12 @@ classifiers = [ ] version="0.14.1" dependencies = [ - "titiler.core==0.14.1", - "titiler.extensions==0.14.1", - "titiler.mosaic==0.14.1", - "titiler.application==0.14.1", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", + "titiler.extensions @ file:///titiler/src/titiler/extensions", + "titiler.mosaic @ file:///titiler/src/titiler/mosaic", + "titiler.application @ file:///titiler/src/titiler/application", ] [project.optional-dependencies] diff --git a/src/titiler/application/pyproject.toml b/src/titiler/application/pyproject.toml index b04a16860..7844e1e06 100644 --- a/src/titiler/application/pyproject.toml +++ b/src/titiler/application/pyproject.toml @@ -29,9 +29,11 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.14.1", - "titiler.extensions[cogeo,stac]==0.14.1", - "titiler.mosaic==0.14.1", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", + "titiler.extensions[cogeo,stac] @ file:///titiler/src/titiler/extensions", + "titiler.mosaic @ file:///titiler/src/titiler/mosaic", "starlette-cramjam>=0.3,<0.4", "pydantic-settings~=2.0", ] diff --git a/src/titiler/application/titiler/application/main.py b/src/titiler/application/titiler/application/main.py index b7e1af4a0..fe371458f 100644 --- a/src/titiler/application/titiler/application/main.py +++ b/src/titiler/application/titiler/application/main.py @@ -1,9 +1,10 @@ """titiler app.""" import logging +import os import jinja2 -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from rio_tiler.io import STACReader from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request @@ -35,17 +36,36 @@ from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.mosaic.factory import MosaicTilerFactory +LEVELS = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} +# Remove any AWS-injected logger handlers to fix Lambda logging to CloudWatch +# https://stackoverflow.com/a/45624044 +base = logging.getLogger() +if base.handlers: + for handler in base.handlers: + base.removeHandler(handler) +logging.basicConfig(level=LEVELS.get(os.environ.get("LOGLEVEL", "info"))) logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True logging.getLogger("rio-tiler").setLevel(logging.ERROR) +logger = logging.getLogger(__name__) +logger.info("TiTiler") + templates = Jinja2Templates( directory="", loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")]), ) # type:ignore - api_settings = ApiSettings() +global_prefix = api_settings.path_prefix +logger.debug(f"Root path: {api_settings.root_path}") +logger.debug(f"Path prefix: {global_prefix}") app = FastAPI( title=api_settings.name, @@ -67,9 +87,10 @@ ############################################################################### # Simple Dataset endpoints (e.g Cloud Optimized GeoTIFF) +cog_prefix = global_prefix + "/cog" if not api_settings.disable_cog: cog = TilerFactory( - router_prefix="/cog", + router_prefix=cog_prefix, extensions=[ cogValidateExtension(), cogViewerExtension(), @@ -77,39 +98,41 @@ ], ) - app.include_router(cog.router, prefix="/cog", tags=["Cloud Optimized GeoTIFF"]) + app.include_router(cog.router, prefix=cog_prefix, tags=["Cloud Optimized GeoTIFF"]) ############################################################################### # STAC endpoints +stac_prefix = global_prefix + "/stac" if not api_settings.disable_stac: stac = MultiBaseTilerFactory( reader=STACReader, - router_prefix="/stac", + router_prefix=stac_prefix, extensions=[ stacViewerExtension(), ], ) app.include_router( - stac.router, prefix="/stac", tags=["SpatioTemporal Asset Catalog"] + stac.router, prefix=stac_prefix, tags=["SpatioTemporal Asset Catalog"] ) ############################################################################### # Mosaic endpoints +mosaicjson_prefix = global_prefix + "/mosaicjson" if not api_settings.disable_mosaic: - mosaic = MosaicTilerFactory(router_prefix="/mosaicjson") - app.include_router(mosaic.router, prefix="/mosaicjson", tags=["MosaicJSON"]) + mosaic = MosaicTilerFactory(router_prefix=mosaicjson_prefix) + app.include_router(mosaic.router, prefix=mosaicjson_prefix, tags=["MosaicJSON"]) ############################################################################### # TileMatrixSets endpoints tms = TMSFactory() -app.include_router(tms.router, tags=["Tiling Schemes"]) +app.include_router(tms.router, prefix=global_prefix, tags=["Tiling Schemes"]) ############################################################################### # Algorithms endpoints algorithms = AlgorithmFactory() -app.include_router(algorithms.router, tags=["Algorithms"]) +app.include_router(algorithms.router, prefix=global_prefix, tags=["Algorithms"]) add_exception_handlers(app, DEFAULT_STATUS_CODES) add_exception_handlers(app, MOSAIC_STATUS_CODES) @@ -139,7 +162,7 @@ app.add_middleware( CacheControlMiddleware, cachecontrol=api_settings.cachecontrol, - exclude_path={r"/healthz"}, + exclude_path={global_prefix + r"/healthz"}, ) if api_settings.debug: @@ -150,7 +173,10 @@ app.add_middleware(LowerCaseQueryStringMiddleware) -@app.get( +router = APIRouter(prefix=global_prefix) + + +@router.get( "/healthz", description="Health Check.", summary="Health Check.", @@ -162,7 +188,7 @@ def ping(): return {"ping": "pong!"} -@app.get("/", response_class=HTMLResponse, include_in_schema=False) +@router.get("/", response_class=HTMLResponse, include_in_schema=False) def landing(request: Request): """TiTiler landing page.""" data = { @@ -231,3 +257,6 @@ def landing(request: Request): "urlparams": str(request.url.query), }, ) + + +app.include_router(router) diff --git a/src/titiler/application/titiler/application/settings.py b/src/titiler/application/titiler/application/settings.py index 81aa86587..2c2061b7a 100644 --- a/src/titiler/application/titiler/application/settings.py +++ b/src/titiler/application/titiler/application/settings.py @@ -12,6 +12,7 @@ class ApiSettings(BaseSettings): cors_allow_methods: str = "GET" cachecontrol: str = "public, max-age=3600" root_path: str = "" + path_prefix: str = "" debug: bool = False disable_cog: bool = False diff --git a/src/titiler/core/pyproject.toml b/src/titiler/core/pyproject.toml index 58c69844e..8025def23 100644 --- a/src/titiler/core/pyproject.toml +++ b/src/titiler/core/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "numpy", "pydantic~=2.0", "rasterio", - "rio-tiler>=6.0,<7.0", + "rio-tiler@https://github.com/SenteraLLC/rio-tiler/archive/refs/heads/6.1.0-sentera-changes-dev.tar.gz", "morecantile>=5.0,<6.0", "simplejson", "typing_extensions>=4.6.1", diff --git a/src/titiler/core/titiler/core/factory.py b/src/titiler/core/titiler/core/factory.py index c90ad4694..06bc4e6eb 100644 --- a/src/titiler/core/titiler/core/factory.py +++ b/src/titiler/core/titiler/core/factory.py @@ -1,6 +1,8 @@ """TiTiler Router factories.""" import abc +import json +from base64 import b64decode from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union from urllib.parse import urlencode @@ -249,6 +251,11 @@ def add_route_dependencies( route.dependencies.extend(dependencies) # type: ignore +def get_feature(aoi: str) -> Feature: + """Base64 encoded GeoJSON Feature.""" + return json.loads(b64decode(aoi)) + + @dataclass class TilerFactory(BaseTilerFactory): """Tiler Factory. @@ -510,6 +517,10 @@ def tile( description="Row (Y) index of the tile on the selected TileMatrix. It cannot exceed the MatrixWidth-1 for the selected TileMatrix.", ), ], + aoi: Annotated[ + Union[str, None], + "Area of interest to crop the tile.", + ] = None, tileMatrixSetId: Annotated[ Literal[tuple(self.supported_tms.list())], f"Identifier selecting one of the TileMatrixSetId supported (default: '{self.default_tms}')", @@ -534,6 +545,7 @@ def tile( env=Depends(self.environment_dependency), ): """Create map tile from a dataset.""" + feature = get_feature(aoi) if aoi else None tms = self.supported_tms.get(tileMatrixSetId) with rasterio.Env(**env): with self.reader(src_path, tms=tms, **reader_params) as src_dst: @@ -542,6 +554,7 @@ def tile( y, z, tilesize=scale * 256, + aoi=feature, buffer=buffer, **layer_params, **dataset_params, diff --git a/src/titiler/extensions/pyproject.toml b/src/titiler/extensions/pyproject.toml index 9ed887e87..d2fff7e80 100644 --- a/src/titiler/extensions/pyproject.toml +++ b/src/titiler/extensions/pyproject.toml @@ -29,7 +29,9 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.14.1" + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", ] [project.optional-dependencies] diff --git a/src/titiler/mosaic/pyproject.toml b/src/titiler/mosaic/pyproject.toml index 5ebdcfae4..a92ca305f 100644 --- a/src/titiler/mosaic/pyproject.toml +++ b/src/titiler/mosaic/pyproject.toml @@ -29,7 +29,9 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "titiler.core==0.14.1", + # Configured for volume-mount path of Lambda Dockerfile. For local + # development, update accordingly for your local filesystem path: + "titiler.core @ file:///titiler/src/titiler/core", "cogeo-mosaic>=7.0,<8.0", ] From 523b76e81122a2d8e4a86202abc5be794b4f119c Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Tue, 17 Oct 2023 14:06:05 -0500 Subject: [PATCH 2/4] Add reproject query param for rio-tiler::reproject_method --- src/titiler/core/titiler/core/dependencies.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/titiler/core/titiler/core/dependencies.py b/src/titiler/core/titiler/core/dependencies.py index 3ebf11eaf..9401a582f 100644 --- a/src/titiler/core/titiler/core/dependencies.py +++ b/src/titiler/core/titiler/core/dependencies.py @@ -10,7 +10,7 @@ from rasterio.crs import CRS from rio_tiler.colormap import cmap, parse_color from rio_tiler.errors import MissingAssets, MissingBands -from rio_tiler.types import ColorMapType, RIOResampling +from rio_tiler.types import ColorMapType, RIOResampling, WarpResampling from typing_extensions import Annotated ColorMapName = Enum( # type: ignore @@ -344,6 +344,13 @@ class DatasetParams(DefaultDependency): alias="resampling", description="Resampling method.", ), + ] = "nearest", + reproject_method: Annotated[ + WarpResampling, + Query( + alias="reproject", + description="Reprojection method.", + ), ] = "nearest" def __post_init__(self): From bbea83e16c83f3ddc0373c7b32c64e40ef0eb90a Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Tue, 17 Oct 2023 14:13:16 -0500 Subject: [PATCH 3/4] Remove problematic comma --- src/titiler/core/titiler/core/dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/titiler/core/titiler/core/dependencies.py b/src/titiler/core/titiler/core/dependencies.py index 9401a582f..9727acf01 100644 --- a/src/titiler/core/titiler/core/dependencies.py +++ b/src/titiler/core/titiler/core/dependencies.py @@ -344,7 +344,7 @@ class DatasetParams(DefaultDependency): alias="resampling", description="Resampling method.", ), - ] = "nearest", + ] = "nearest" reproject_method: Annotated[ WarpResampling, Query( From 961a272d62eef45c158501242a1f032541c5d283 Mon Sep 17 00:00:00 2001 From: Dan Schoppe Date: Wed, 18 Oct 2023 10:58:33 -0500 Subject: [PATCH 4/4] Try specifying api_gateway_base_path to Mangum --- deployment/aws/lambda/handler.py | 5 ++- .../application/titiler/application/main.py | 36 +++++++------------ .../titiler/application/settings.py | 1 - 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/deployment/aws/lambda/handler.py b/deployment/aws/lambda/handler.py index 4e66c7b2e..933807be5 100644 --- a/deployment/aws/lambda/handler.py +++ b/deployment/aws/lambda/handler.py @@ -5,8 +5,11 @@ from mangum import Mangum from titiler.application.main import app +from titiler.application.settings import ApiSettings logging.getLogger("mangum.lifespan").setLevel(logging.ERROR) logging.getLogger("mangum.http").setLevel(logging.ERROR) -handler = Mangum(app, lifespan="auto") +api_settings = ApiSettings() + +handler = Mangum(app, api_gateway_base_path=api_settings.root_path, lifespan="auto") diff --git a/src/titiler/application/titiler/application/main.py b/src/titiler/application/titiler/application/main.py index fe371458f..122be9f0e 100644 --- a/src/titiler/application/titiler/application/main.py +++ b/src/titiler/application/titiler/application/main.py @@ -4,7 +4,7 @@ import os import jinja2 -from fastapi import APIRouter, FastAPI +from fastapi import FastAPI from rio_tiler.io import STACReader from starlette.middleware.cors import CORSMiddleware from starlette.requests import Request @@ -63,9 +63,6 @@ ) # type:ignore api_settings = ApiSettings() -global_prefix = api_settings.path_prefix -logger.debug(f"Root path: {api_settings.root_path}") -logger.debug(f"Path prefix: {global_prefix}") app = FastAPI( title=api_settings.name, @@ -87,10 +84,9 @@ ############################################################################### # Simple Dataset endpoints (e.g Cloud Optimized GeoTIFF) -cog_prefix = global_prefix + "/cog" if not api_settings.disable_cog: cog = TilerFactory( - router_prefix=cog_prefix, + router_prefix="/cog", extensions=[ cogValidateExtension(), cogViewerExtension(), @@ -98,41 +94,39 @@ ], ) - app.include_router(cog.router, prefix=cog_prefix, tags=["Cloud Optimized GeoTIFF"]) + app.include_router(cog.router, prefix="/cog", tags=["Cloud Optimized GeoTIFF"]) ############################################################################### # STAC endpoints -stac_prefix = global_prefix + "/stac" if not api_settings.disable_stac: stac = MultiBaseTilerFactory( reader=STACReader, - router_prefix=stac_prefix, + router_prefix="/stac", extensions=[ stacViewerExtension(), ], ) app.include_router( - stac.router, prefix=stac_prefix, tags=["SpatioTemporal Asset Catalog"] + stac.router, prefix="/stac", tags=["SpatioTemporal Asset Catalog"] ) ############################################################################### # Mosaic endpoints -mosaicjson_prefix = global_prefix + "/mosaicjson" if not api_settings.disable_mosaic: - mosaic = MosaicTilerFactory(router_prefix=mosaicjson_prefix) - app.include_router(mosaic.router, prefix=mosaicjson_prefix, tags=["MosaicJSON"]) + mosaic = MosaicTilerFactory(router_prefix="/mosaicjson") + app.include_router(mosaic.router, prefix="/mosaicjson", tags=["MosaicJSON"]) ############################################################################### # TileMatrixSets endpoints tms = TMSFactory() -app.include_router(tms.router, prefix=global_prefix, tags=["Tiling Schemes"]) +app.include_router(tms.router, tags=["Tiling Schemes"]) ############################################################################### # Algorithms endpoints algorithms = AlgorithmFactory() -app.include_router(algorithms.router, prefix=global_prefix, tags=["Algorithms"]) +app.include_router(algorithms.router, tags=["Algorithms"]) add_exception_handlers(app, DEFAULT_STATUS_CODES) add_exception_handlers(app, MOSAIC_STATUS_CODES) @@ -162,7 +156,7 @@ app.add_middleware( CacheControlMiddleware, cachecontrol=api_settings.cachecontrol, - exclude_path={global_prefix + r"/healthz"}, + exclude_path={r"/healthz"}, ) if api_settings.debug: @@ -173,10 +167,7 @@ app.add_middleware(LowerCaseQueryStringMiddleware) -router = APIRouter(prefix=global_prefix) - - -@router.get( +@app.get( "/healthz", description="Health Check.", summary="Health Check.", @@ -188,7 +179,7 @@ def ping(): return {"ping": "pong!"} -@router.get("/", response_class=HTMLResponse, include_in_schema=False) +@app.get("/", response_class=HTMLResponse, include_in_schema=False) def landing(request: Request): """TiTiler landing page.""" data = { @@ -257,6 +248,3 @@ def landing(request: Request): "urlparams": str(request.url.query), }, ) - - -app.include_router(router) diff --git a/src/titiler/application/titiler/application/settings.py b/src/titiler/application/titiler/application/settings.py index 2c2061b7a..81aa86587 100644 --- a/src/titiler/application/titiler/application/settings.py +++ b/src/titiler/application/titiler/application/settings.py @@ -12,7 +12,6 @@ class ApiSettings(BaseSettings): cors_allow_methods: str = "GET" cachecontrol: str = "public, max-age=3600" root_path: str = "" - path_prefix: str = "" debug: bool = False disable_cog: bool = False