diff --git a/app.py b/app.py index ad525d19..c2bf290a 100644 --- a/app.py +++ b/app.py @@ -122,6 +122,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # ingestor config requires references to other resources, but can be shared between ingest api and bulk ingestor ingestor_config = ingest_config( + stage=veda_app_settings.stage_name(), stac_db_security_group_id=db_security_group.security_group_id, stac_api_url=stac_api.stac_api.url, raster_api_url=raster_api.raster_api.url, diff --git a/ingest_api/infrastructure/config.py b/ingest_api/infrastructure/config.py index 823cf616..3a2a587a 100644 --- a/ingest_api/infrastructure/config.py +++ b/ingest_api/infrastructure/config.py @@ -41,8 +41,8 @@ class IngestorConfig(BaseSettings): description="ID of Security Group used by pgSTAC DB" ) - raster_data_access_role_arn: AwsArn = Field( # type: ignore - description="ARN of AWS Role used to validate access to S3 data" + raster_data_access_role_arn: Optional[AwsArn] = Field( # type: ignore + None, description="ARN of AWS Role used to validate access to S3 data" ) stac_api_url: str = Field(description="URL of STAC API used to serve STAC Items") diff --git a/ingest_api/infrastructure/construct.py b/ingest_api/infrastructure/construct.py index 0b195a0d..662e90ac 100644 --- a/ingest_api/infrastructure/construct.py +++ b/ingest_api/infrastructure/construct.py @@ -1,6 +1,6 @@ import os import typing -from typing import Dict, Optional +from typing import Dict, Optional, Union from aws_cdk import CfnOutput, Duration, RemovalPolicy, Stack from aws_cdk import aws_apigateway as apigateway @@ -36,10 +36,6 @@ def __init__( super().__init__(scope, construct_id, **kwargs) self.table = self.build_table() - self.data_access_role = iam.Role.from_role_arn( - self, "data-access-role", config.raster_data_access_role_arn - ) - self.user_pool = cognito.UserPool.from_user_pool_id( self, "cognito-user-pool", config.userpool_id ) @@ -55,7 +51,6 @@ def __init__( "JWKS_URL": self.jwks_url, "NO_PYDANTIC_SSM_SETTINGS": "1", "STAC_URL": config.stac_api_url, - "DATA_ACCESS_ROLE_ARN": config.raster_data_access_role_arn, "USERPOOL_ID": config.userpool_id, "CLIENT_ID": config.client_id, "CLIENT_SECRET": config.client_secret, @@ -65,16 +60,23 @@ def __init__( "COGNITO_DOMAIN": config.cognito_domain, } + build_api_lambda_params = { + "table": self.table, + "user_pool": self.user_pool, + "db_secret": db_secret, + "db_vpc": db_vpc, + "db_security_group": db_security_group, + } + + if config.raster_data_access_role_arn: + lambda_env["DATA_ACCESS_ROLE_ARN"] = config.raster_data_access_role_arn + build_api_lambda_params["data_access_role"] = iam.Role.from_role_arn( + self, "data-access-role", config.raster_data_access_role_arn + ) + build_api_lambda_params["env"] = lambda_env + # create lambda - self.api_lambda = self.build_api_lambda( - table=self.table, - env=lambda_env, - data_access_role=self.data_access_role, - user_pool=self.user_pool, - db_secret=db_secret, - db_vpc=db_vpc, - db_security_group=db_security_group, - ) + self.api_lambda = self.build_api_lambda(**build_api_lambda_params) # create API self.api: aws_apigatewayv2_alpha.HttpApi = self.build_api( @@ -111,11 +113,11 @@ def build_api_lambda( *, table: dynamodb.ITable, env: Dict[str, str], - data_access_role: iam.IRole, user_pool: cognito.IUserPool, db_secret: secretsmanager.ISecret, db_vpc: ec2.IVpc, db_security_group: ec2.ISecurityGroup, + data_access_role: Union[iam.IRole, None] = None, code_dir: str = "./", ) -> apigateway.LambdaRestApi: stack_name = Stack.of(self).stack_name @@ -153,10 +155,11 @@ def build_api_lambda( log_format="JSON", ) table.grant_read_write_data(handler) - data_access_role.grant( - handler.grant_principal, - "sts:AssumeRole", - ) + if data_access_role: + data_access_role.grant( + handler.grant_principal, + "sts:AssumeRole", + ) handler.add_to_role_policy( iam.PolicyStatement( @@ -260,13 +263,15 @@ def __init__( "DYNAMODB_TABLE": table.table_name, "NO_PYDANTIC_SSM_SETTINGS": "1", "STAC_URL": config.stac_api_url, - "DATA_ACCESS_ROLE_ARN": config.raster_data_access_role_arn, "USERPOOL_ID": config.userpool_id, "CLIENT_ID": config.client_id, "CLIENT_SECRET": config.client_secret, "RASTER_URL": config.raster_api_url, } + if config.raster_data_access_role_arn: + lambda_env["DATA_ACCESS_ROLE_ARN"] = config.raster_data_access_role_arn + db_security_group = ec2.SecurityGroup.from_security_group_id( self, "db-security-group", diff --git a/ingest_api/runtime/handler.py b/ingest_api/runtime/handler.py index 102f98b0..d6494d78 100644 --- a/ingest_api/runtime/handler.py +++ b/ingest_api/runtime/handler.py @@ -4,5 +4,14 @@ from mangum import Mangum from src.main import app +from src.monitoring import logger, metrics, tracer handler = Mangum(app, lifespan="off", api_gateway_base_path=app.root_path) + +# Add tracing +handler.__name__ = "handler" # tracer requires __name__ to be set +handler = tracer.capture_lambda_handler(handler) +# Add logging +handler = logger.inject_lambda_context(handler, clear_state=True) +# Add metrics last to properly flush metrics. +handler = metrics.log_metrics(handler, capture_cold_start_metric=True) diff --git a/ingest_api/runtime/src/config.py b/ingest_api/runtime/src/config.py index 12e8f8d9..2183bbda 100644 --- a/ingest_api/runtime/src/config.py +++ b/ingest_api/runtime/src/config.py @@ -15,7 +15,7 @@ class Settings(BaseSettings): description="URL of JWKS, e.g. https://cognito-idp.{region}.amazonaws.com/{userpool_id}/.well-known/jwks.json" # noqa ) - data_access_role_arn: AwsArn = Field( # type: ignore + data_access_role_arn: Optional[AwsArn] = Field( # type: ignore description="ARN of AWS Role used to validate access to S3 data" ) @@ -23,7 +23,7 @@ class Settings(BaseSettings): userpool_id: str = Field(description="The Cognito Userpool used for authentication") - cognito_domain: AnyHttpUrl = Field( + cognito_domain: Optional[AnyHttpUrl] = Field( description="The base url of the Cognito domain for authorization and token urls" ) client_id: str = Field(description="The Cognito APP client ID") diff --git a/ingest_api/runtime/src/main.py b/ingest_api/runtime/src/main.py index e9ac102b..9b5fa6d2 100644 --- a/ingest_api/runtime/src/main.py +++ b/ingest_api/runtime/src/main.py @@ -10,7 +10,7 @@ from src.doc import DESCRIPTION from src.monitoring import LoggerRouteHandler, logger, metrics, tracer -from fastapi import APIRouter, Depends, FastAPI, HTTPException +from fastapi import Depends, FastAPI, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from fastapi.security import OAuth2PasswordRequestForm @@ -32,9 +32,9 @@ "clientId": settings.client_id, "usePkceWithAuthorizationCodeGrant": True, }, - router=APIRouter(route_class=LoggerRouteHandler), ) +app.router.route_class = LoggerRouteHandler collection_publisher = CollectionPublisher() item_publisher = ItemPublisher() diff --git a/ingest_api/runtime/src/monitoring.py b/ingest_api/runtime/src/monitoring.py index 472ba110..2eec20ee 100644 --- a/ingest_api/runtime/src/monitoring.py +++ b/ingest_api/runtime/src/monitoring.py @@ -1,19 +1,22 @@ """Observability utils""" +import json from typing import Callable -from aws_lambda_powertools import Logger, Metrics, Tracer +from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 +from src.config import settings from fastapi import Request, Response from fastapi.routing import APIRoute -logger: Logger = Logger(service="raster-api", namespace="veda-backend") -metrics: Metrics = Metrics(service="raster-api", namespace="veda-backend") +logger: Logger = Logger(service="ingest-api", namespace="veda-backend") +metrics: Metrics = Metrics(namespace="veda-backend") +metrics.set_default_dimensions(environment=settings.stage, service="ingest-api") tracer: Tracer = Tracer() class LoggerRouteHandler(APIRoute): - """Extension of base APIRoute to add context to log statements, as well as record usage metricss""" + """Extension of base APIRoute to add context to log statements, as well as record usage metrics""" def get_route_handler(self) -> Callable: """Overide route handler method to add logs, metrics, tracing""" @@ -21,20 +24,32 @@ def get_route_handler(self) -> Callable: async def route_handler(request: Request) -> Response: # Add fastapi context to logs + body = await request.body() + try: + body_json = json.loads(body) + except json.decoder.JSONDecodeError: + body_json = None ctx = { "path": request.url.path, + "path_params": request.path_params, + "body": body_json, "route": self.path, "method": request.method, } logger.append_keys(fastapi=ctx) logger.info("Received request") - metrics.add_metric( - name="/".join( - str(request.url.path).split("/")[:2] - ), # enough detail to capture search IDs, but not individual tile coords + + with single_metric( + name="RequestCount", unit=MetricUnit.Count, value=1, - ) + default_dimensions=metrics.default_dimensions, + namespace="veda-backend", + ) as metric: + metric.add_dimension( + name="route", value=f"{request.method} {self.path}" + ) + tracer.put_annotation(key="path", value=request.url.path) tracer.capture_method(original_route_handler)(request) return await original_route_handler(request) diff --git a/ingest_api/runtime/src/validators.py b/ingest_api/runtime/src/validators.py index 0abee3ba..ff057d5c 100644 --- a/ingest_api/runtime/src/validators.py +++ b/ingest_api/runtime/src/validators.py @@ -9,8 +9,10 @@ def get_s3_credentials(): from src.main import settings - print("Fetching S3 Credentials...") + if not settings.data_access_role_arn: + return {} + print("Fetching S3 Credentials...") response = boto3.client("sts").assume_role( RoleArn=settings.data_access_role_arn, RoleSessionName="stac-ingestor-data-validation", diff --git a/raster_api/infrastructure/construct.py b/raster_api/infrastructure/construct.py index 3b8300fb..e8a43a86 100644 --- a/raster_api/infrastructure/construct.py +++ b/raster_api/infrastructure/construct.py @@ -80,6 +80,8 @@ def __init__( "VEDA_RASTER_ROOT_PATH", veda_raster_settings.raster_root_path ) + veda_raster_function.add_environment("VEDA_RASTER_STAGE", stage) + # Optional AWS S3 requester pays global setting if veda_raster_settings.raster_aws_request_payer: veda_raster_function.add_environment( diff --git a/raster_api/runtime/src/config.py b/raster_api/runtime/src/config.py index c26d0b89..305d85cc 100644 --- a/raster_api/runtime/src/config.py +++ b/raster_api/runtime/src/config.py @@ -57,6 +57,7 @@ class ApiSettings(BaseSettings): cachecontrol: str = "public, max-age=3600" debug: bool = False root_path: Optional[str] = None + stage: Optional[str] = None # MosaicTiler settings enable_mosaic_search: bool = False diff --git a/raster_api/runtime/src/monitoring.py b/raster_api/runtime/src/monitoring.py index 472ba110..7237806d 100644 --- a/raster_api/runtime/src/monitoring.py +++ b/raster_api/runtime/src/monitoring.py @@ -1,19 +1,24 @@ """Observability utils""" +import json from typing import Callable -from aws_lambda_powertools import Logger, Metrics, Tracer +from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 +from src.config import ApiSettings from fastapi import Request, Response from fastapi.routing import APIRoute +settings = ApiSettings() + logger: Logger = Logger(service="raster-api", namespace="veda-backend") -metrics: Metrics = Metrics(service="raster-api", namespace="veda-backend") +metrics: Metrics = Metrics(namespace="veda-backend") +metrics.set_default_dimensions(environment=settings.stage, service="raster-api") tracer: Tracer = Tracer() class LoggerRouteHandler(APIRoute): - """Extension of base APIRoute to add context to log statements, as well as record usage metricss""" + """Extension of base APIRoute to add context to log statements, as well as record usage metrics""" def get_route_handler(self) -> Callable: """Overide route handler method to add logs, metrics, tracing""" @@ -21,22 +26,36 @@ def get_route_handler(self) -> Callable: async def route_handler(request: Request) -> Response: # Add fastapi context to logs + body = await request.body() + try: + body_json = json.loads(body) + except json.decoder.JSONDecodeError: + body_json = None + ctx = { "path": request.url.path, + "path_params": request.path_params, + "body": body_json, "route": self.path, "method": request.method, } logger.append_keys(fastapi=ctx) logger.info("Received request") - metrics.add_metric( - name="/".join( - str(request.url.path).split("/")[:2] - ), # enough detail to capture search IDs, but not individual tile coords + + with single_metric( + name="RequestCount", unit=MetricUnit.Count, value=1, - ) + default_dimensions=metrics.default_dimensions, + namespace="veda-backend", + ) as metric: + metric.add_dimension( + name="route", value=f"{request.method} {self.path}" + ) + tracer.put_annotation(key="path", value=request.url.path) tracer.capture_method(original_route_handler)(request) + return await original_route_handler(request) return route_handler diff --git a/routes/infrastructure/construct.py b/routes/infrastructure/construct.py index 8ef0cafc..35c87129 100755 --- a/routes/infrastructure/construct.py +++ b/routes/infrastructure/construct.py @@ -55,6 +55,7 @@ def __init__( cache_policy=cf.CachePolicy.CACHING_DISABLED, ), certificate=domain_cert, + enable_logging=True, domain_names=[f"{stage}.{veda_route_settings.domain_hosted_zone_name}"] if veda_route_settings.domain_hosted_zone_name else None, diff --git a/stac_api/infrastructure/construct.py b/stac_api/infrastructure/construct.py index 6359ddfe..0f3ba03b 100644 --- a/stac_api/infrastructure/construct.py +++ b/stac_api/infrastructure/construct.py @@ -84,6 +84,8 @@ def __init__( "VEDA_STAC_ROOT_PATH", veda_stac_settings.stac_root_path ) + lambda_function.add_environment("VEDA_STAC_STAGE", stage) + integration_kwargs = dict(handler=lambda_function) if veda_stac_settings.custom_host: integration_kwargs[ diff --git a/stac_api/runtime/src/app.py b/stac_api/runtime/src/app.py index e3044afb..5e68080d 100644 --- a/stac_api/runtime/src/app.py +++ b/stac_api/runtime/src/app.py @@ -8,7 +8,7 @@ from src.config import post_request_model as POSTModel from src.extension import TiTilerExtension -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from fastapi.responses import ORJSONResponse from stac_fastapi.pgstac.db import close_db_connection, connect_to_db from starlette.middleware.cors import CORSMiddleware @@ -19,7 +19,7 @@ from .api import VedaStacApi from .core import VedaCrudClient -from .monitoring import logger, metrics, tracer +from .monitoring import LoggerRouteHandler, logger, metrics, tracer try: from importlib.resources import files as resources_files # type: ignore @@ -49,6 +49,7 @@ search_post_request_model=POSTModel, response_class=ORJSONResponse, middlewares=[CompressionMiddleware], + router=APIRouter(route_class=LoggerRouteHandler), ) app = api.app diff --git a/stac_api/runtime/src/config.py b/stac_api/runtime/src/config.py index bdda7574..d2f1a524 100644 --- a/stac_api/runtime/src/config.py +++ b/stac_api/runtime/src/config.py @@ -56,6 +56,7 @@ class _ApiSettings(pydantic.BaseSettings): debug: bool = False root_path: Optional[str] = None pgstac_secret_arn: Optional[str] + stage: Optional[str] = None @pydantic.validator("cors_origins") def parse_cors_origin(cls, v): diff --git a/stac_api/runtime/src/monitoring.py b/stac_api/runtime/src/monitoring.py index 67fa4a43..1e3b3d61 100644 --- a/stac_api/runtime/src/monitoring.py +++ b/stac_api/runtime/src/monitoring.py @@ -1,19 +1,24 @@ """Observability utils""" +import json from typing import Callable -from aws_lambda_powertools import Logger, Metrics, Tracer +from aws_lambda_powertools import Logger, Metrics, Tracer, single_metric from aws_lambda_powertools.metrics import MetricUnit # noqa: F401 +from src.config import ApiSettings from fastapi import Request, Response from fastapi.routing import APIRoute +settings = ApiSettings() + logger: Logger = Logger(service="stac-api", namespace="veda-backend") -metrics: Metrics = Metrics(service="stac-api", namespace="veda-backend") +metrics: Metrics = Metrics(namespace="veda-backend") +metrics.set_default_dimensions(environment=settings.stage, service="stac-api") tracer: Tracer = Tracer() class LoggerRouteHandler(APIRoute): - """Extension of base APIRoute to add context to log statements, as well as record usage metricss""" + """Extension of base APIRoute to add context to log statements, as well as record usage metrics""" def get_route_handler(self) -> Callable: """Overide route handler method to add logs, metrics, tracing""" @@ -21,20 +26,33 @@ def get_route_handler(self) -> Callable: async def route_handler(request: Request) -> Response: # Add fastapi context to logs + body = await request.body() + try: + body_json = json.loads(body) + except json.decoder.JSONDecodeError: + body_json = None + ctx = { "path": request.url.path, + "path_params": request.path_params, + "body": body_json, "route": self.path, "method": request.method, } logger.append_keys(fastapi=ctx) logger.info("Received request") - metrics.add_metric( - name="/".join( - str(request.url).split("/")[:2] - ), # enough detail to capture search IDs, but not individual tile coords + + with single_metric( + name="RequestCount", unit=MetricUnit.Count, value=1, - ) + default_dimensions=metrics.default_dimensions, + namespace="veda-backend", + ) as metric: + metric.add_dimension( + name="route", value=f"{request.method} {self.path}" + ) + tracer.put_annotation(key="path", value=request.url.path) tracer.capture_method(original_route_handler)(request) return await original_route_handler(request)