From 85694ffae8bd881924a2db5491c88dec092972e5 Mon Sep 17 00:00:00 2001 From: emileten Date: Mon, 11 Sep 2023 18:40:00 +0900 Subject: [PATCH 1/2] first commit, draft move --- infrastructure/aws/cdk/__init__.py | 1 - infrastructure/aws/cdk/app.py | 493 ++---------------------- infrastructure/aws/cdk/config.py | 211 +++++----- infrastructure/aws/cdk/pgStacInfra.py | 186 +++++++++ infrastructure/aws/cdk/vpc.py | 50 +++ infrastructure/aws/requirements-cdk.txt | 8 +- 6 files changed, 372 insertions(+), 577 deletions(-) delete mode 100644 infrastructure/aws/cdk/__init__.py create mode 100644 infrastructure/aws/cdk/pgStacInfra.py create mode 100644 infrastructure/aws/cdk/vpc.py diff --git a/infrastructure/aws/cdk/__init__.py b/infrastructure/aws/cdk/__init__.py deleted file mode 100644 index 4955682..0000000 --- a/infrastructure/aws/cdk/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""AWS App.""" diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b327440..bbf8cf0 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -1,461 +1,46 @@ -""" -CDK Stack definition code for EOAPI -""" -import json -import os -from typing import Any - -from aws_cdk import App, CfnOutput, CustomResource, Duration, RemovalPolicy, Stack, Tags -from aws_cdk import aws_apigatewayv2_alpha as apigw -from aws_cdk import aws_ec2 as ec2 -from aws_cdk import aws_iam as iam -from aws_cdk import aws_lambda -from aws_cdk import aws_logs as logs -from aws_cdk import aws_rds as rds -from aws_cdk import aws_secretsmanager as secretsmanager -from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration -from config import ( - eoAPISettings, - eoDBSettings, - eoRasterSettings, - eoSTACSettings, - eoVectorSettings, -) -from constructs import Construct - -eoapi_settings = eoAPISettings() - - -class BootstrappedDb(Construct): - """ - Given an RDS database, connect to DB and create a database, user, and - password - """ - - def __init__( - self, - scope: Construct, - id: str, - db: rds.DatabaseInstance, - new_dbname: str, - new_username: str, - secrets_prefix: str, - pgstac_version: str, - enable_context: bool = False, - enable_mosaic_index: bool = False, - context_dir: str = "../../", - ) -> None: - """Update RDS database.""" - super().__init__(scope, id) - - # TODO: Utilize a singleton function. - handler = aws_lambda.Function( - self, - "DatabaseBootstrapper", - handler="handler.handler", - runtime=aws_lambda.Runtime.PYTHON_3_10, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.db", - build_args={"PYTHON_VERSION": "3.10", "PGSTAC_VERSION": pgstac_version}, - platform="linux/amd64", - ), - timeout=Duration.minutes(5), - vpc=db.vpc, - allow_public_subnet=True, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - - self.secret = secretsmanager.Secret( - self, - id, - secret_name=os.path.join( - secrets_prefix, id.replace(" ", "_"), self.node.addr - ), - generate_secret_string=secretsmanager.SecretStringGenerator( - secret_string_template=json.dumps( - { - "dbname": new_dbname, - "engine": "postgres", - "port": 5432, - "host": db.instance_endpoint.hostname, - "username": new_username, - }, - ), - generate_string_key="password", - exclude_punctuation=True, - ), - description=f"Deployed by {Stack.of(self).stack_name}", - ) - - self.resource = CustomResource( - scope=scope, - id="BootstrappedDbResource", - service_token=handler.function_arn, - properties={ - # By setting pgstac_version in the properties assures - # that Create/Update events will be passed to the service token - "pgstac_version": pgstac_version, - "context": enable_context, - "mosaic_index": enable_mosaic_index, - "conn_secret_arn": db.secret.secret_arn, - "new_user_secret_arn": self.secret.secret_arn, - }, - # We do not need to run the custom resource on STAC Delete - # Custom Resource are not physical resources so it's OK to `Retain` it - removal_policy=RemovalPolicy.RETAIN, - ) - - # Allow lambda to... - # read new user secret - self.secret.grant_read(handler) - # read database secret - db.secret.grant_read(handler) - # connect to database - db.connections.allow_from(handler, port_range=ec2.Port.tcp(5432)) - - def is_required_by(self, construct: Construct): - """Register required services.""" - return construct.node.add_dependency(self.resource) - - -class eoAPIconstruct(Stack): - """Earth Observation API CDK application""" - - def __init__( # noqa: C901 - self, - scope: Construct, - id: str, - stage: str, - name: str, - context_dir: str = "../../", - **kwargs: Any, - ) -> None: - """Define stack.""" - super().__init__(scope, id, **kwargs) - - # vpc = ec2.Vpc(self, f"{id}-vpc", nat_gateways=0) - - vpc = ec2.Vpc( - self, - f"{id}-vpc", - subnet_configuration=[ - ec2.SubnetConfiguration( - name="ingress", - cidr_mask=24, - subnet_type=ec2.SubnetType.PUBLIC, - ), - ec2.SubnetConfiguration( - name="application", - cidr_mask=24, - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, - ), - ec2.SubnetConfiguration( - name="rds", - cidr_mask=28, - subnet_type=ec2.SubnetType.PRIVATE_ISOLATED, - ), - ], - nat_gateways=1, - ) - print( - """The eoAPI stack use AWS NatGateway for the Raster service so it can reach the internet. -This might incurs some cost (https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html).""" - ) - - interface_endpoints = [ - ( - "SecretsManager Endpoint", - ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER, - ), - ( - "CloudWatch Logs Endpoint", - ec2.InterfaceVpcEndpointAwsService.CLOUDWATCH_LOGS, - ), - ] - for (key, service) in interface_endpoints: - vpc.add_interface_endpoint(key, service=service) - - gateway_endpoints = [("S3", ec2.GatewayVpcEndpointAwsService.S3)] - for (key, service) in gateway_endpoints: - vpc.add_gateway_endpoint(key, service=service) - - eodb_settings = eoDBSettings() - db = rds.DatabaseInstance( - self, - f"{id}-postgres-db", - vpc=vpc, - engine=rds.DatabaseInstanceEngine.POSTGRES, - instance_type=ec2.InstanceType.of( - ec2.InstanceClass.BURSTABLE3, - ec2.InstanceSize(eodb_settings.instance_size), - ), - database_name="postgres", - # should set the subnet to `PRIVATE_ISOLATED` but then we need either a bastion host to connect to the db - # or an API to ingest/delete data in the DB - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), - backup_retention=Duration.days(7), - deletion_protection=eoapi_settings.stage.lower() == "production", - removal_policy=RemovalPolicy.SNAPSHOT - if eoapi_settings.stage.lower() == "production" - else RemovalPolicy.DESTROY, - ) - - setup_db = BootstrappedDb( - self, - "STAC DB for eoapi", - db=db, - new_dbname=eodb_settings.dbname, - new_username=eodb_settings.user, - secrets_prefix=os.path.join(stage, name), - pgstac_version=eodb_settings.pgstac_version, - enable_context=eodb_settings.context, - enable_mosaic_index=eodb_settings.mosaic_index, - context_dir=context_dir, - ) - - CfnOutput( - self, - f"{id}-database-secret-arn", - value=db.secret.secret_arn, - description="Arn of the SecretsManager instance holding the connection info for Postgres DB", - ) - - # eoapi.raster - if "raster" in eoapi_settings.functions: - db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( - "host" - ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( - "dbname" - ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( - "username" - ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( - "password" - ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( - "port" - ).to_string(), - } - - eoraster_settings = eoRasterSettings() - env = eoraster_settings.env or {} - if "DB_MAX_CONN_SIZE" not in env: - env["DB_MAX_CONN_SIZE"] = "1" - - eoraster_function = aws_lambda.Function( - self, - f"{id}-raster-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.raster", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - vpc_subnets=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), - allow_public_subnet=True, - handler="handler.handler", - memory_size=eoraster_settings.memory, - timeout=Duration.seconds(eoraster_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eoraster_function.add_environment(key=k, value=str(v)) - - eoraster_function.add_to_role_policy( - iam.PolicyStatement( - actions=["s3:GetObject"], - resources=[ - f"arn:aws:s3:::{bucket}/{eoraster_settings.key}" - for bucket in eoraster_settings.buckets - ], - ) - ) - - db.connections.allow_from(eoraster_function, port_range=ec2.Port.tcp(5432)) - - raster_api = apigw.HttpApi( - self, - f"{id}-raster-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-raster-integration", - eoraster_function, - ), - ) - CfnOutput(self, "eoAPI-raster", value=raster_api.url.strip("/")) - - setup_db.is_required_by(eoraster_function) - - # eoapi.stac - if "stac" in eoapi_settings.functions: - db_secrets = { - "POSTGRES_HOST_READER": setup_db.secret.secret_value_from_json( - "host" - ).to_string(), - "POSTGRES_HOST_WRITER": setup_db.secret.secret_value_from_json( - "host" - ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( - "dbname" - ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( - "username" - ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( - "password" - ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( - "port" - ).to_string(), - } - - eostac_settings = eoSTACSettings() - env = eostac_settings.env or {} - if "DB_MAX_CONN_SIZE" not in env: - env["DB_MAX_CONN_SIZE"] = "1" - if "DB_MIN_CONN_SIZE" not in env: - env["DB_MIN_CONN_SIZE"] = "1" - - eostac_function = aws_lambda.Function( - self, - f"{id}-stac-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.stac", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - handler="handler.handler", - memory_size=eostac_settings.memory, - timeout=Duration.seconds(eostac_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eostac_function.add_environment(key=k, value=str(v)) - - # If raster is deployed we had the TITILER_ENDPOINT env to add the Proxy extension - if "raster" in eoapi_settings.functions: - eostac_function.add_environment( - key="TITILER_ENDPOINT", value=raster_api.url.strip("/") - ) - - db.connections.allow_from(eostac_function, port_range=ec2.Port.tcp(5432)) - - stac_api = apigw.HttpApi( - self, - f"{id}-stac-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-stac-integration", - eostac_function, - ), - ) - CfnOutput(self, "eoAPI-stac", value=stac_api.url.strip("/")) - - setup_db.is_required_by(eostac_function) - - # eoapi.vector - if "vector" in eoapi_settings.functions: - db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( - "host" - ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( - "dbname" - ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( - "username" - ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( - "password" - ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( - "port" - ).to_string(), - } - - eovector_settings = eoVectorSettings() - env = eovector_settings.env or {} - - if "DB_MAX_CONN_SIZE" not in env: - env["DB_MAX_CONN_SIZE"] = "1" - if "DB_MIN_CONN_SIZE" not in env: - env["DB_MIN_CONN_SIZE"] = "1" - - eovector_function = aws_lambda.Function( - self, - f"{id}-vector-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.vector", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - handler="handler.handler", - memory_size=eovector_settings.memory, - timeout=Duration.seconds(eovector_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eovector_function.add_environment(key=k, value=str(v)) - - db.connections.allow_from(eovector_function, port_range=ec2.Port.tcp(5432)) - - vector_api = apigw.HttpApi( - self, - f"{id}-vector-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-vector-integration", - eovector_function, - ), - ) - CfnOutput(self, "eoAPI-vector", value=vector_api.url.strip("/")) - - setup_db.is_required_by(eovector_function) +import yaml +from aws_cdk import App +from config import Config +from eoapi_template import pgStacInfra, vpc app = App() - -eoapi_stack = eoAPIconstruct( - app, - f"{eoapi_settings.name}-{eoapi_settings.stage}", - eoapi_settings.name, - eoapi_settings.stage, - env={ - "account": os.environ["CDK_DEFAULT_ACCOUNT"], - "region": os.environ["CDK_DEFAULT_REGION"], - }, +try: + with open("config.yaml") as f: + config = yaml.safe_load(f) + config = ( + {} if config is None else config + ) # if config is empty, set it to an empty dict + config = Config(**config) +except FileNotFoundError: + # if no config at the expected path, using defaults + config = Config() + +vpc_stack = vpc.VpcStack( + tags=config.tags, + scope=app, + id=config.build_service_name("pgSTAC-vpc"), + nat_gateway_count=config.nat_gateway_count, ) -# Tag infrastructure -for key, value in { - "Project": eoapi_settings.name, - "Stack": eoapi_settings.stage, - "Owner": eoapi_settings.owner, - "Client": eoapi_settings.client, -}.items(): - if value: - Tags.of(eoapi_stack).add(key, value) - -app.synth() +pgstac_infra_stack = pgStacInfra.pgStacInfraStack( + scope=app, + tags=config.tags, + id=config.build_service_name("pgSTAC-infra"), + vpc=vpc_stack.vpc, + stac_api_lambda_name=config.build_service_name("STAC API"), + titiler_pgstac_api_lambda_name=config.build_service_name("titiler pgSTAC API"), + stage=config.stage, + db_allocated_storage=config.db_allocated_storage, + public_db_subnet=config.public_db_subnet, + db_instance_type=config.db_instance_type, + bastion_host_allow_ip_list=config.bastion_host_allow_ip_list, + bastion_host_create_elastic_ip=config.bastion_host_create_elastic_ip, + bastion_host_user_data=yaml.dump(config.bastion_host_user_data), + titiler_buckets=config.titiler_buckets, + data_access_role_arn=config.data_access_role_arn, + auth_provider_jwks_url=config.auth_provider_jwks_url, +) +app.synth() \ No newline at end of file diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 6c033b6..bb7fcff 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -1,120 +1,95 @@ -"""eoAPI Configs.""" - -from enum import Enum -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import pydantic - - -class functionName(str, Enum): - """Function names.""" - - stac = "stac" - raster = "raster" - vector = "vector" - - -class eoAPISettings(pydantic.BaseSettings): - """Application settings""" - - name: str = "eoapi" - stage: str = "production" - owner: Optional[str] - client: Optional[str] - functions: List[functionName] = [functionName.stac, functionName.raster] - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "CDK_EOAPI_" - use_enum_values = True - - -class eoDBSettings(pydantic.BaseSettings): - """Application settings""" - - dbname: str = "eoapi" - user: str = "eouser" - - # Define PGSTAC VERSION - pgstac_version: str - instance_size: str = "SMALL" - context: bool = True - mosaic_index: bool = True - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "CDK_EOAPI_DB_" - - -class eoSTACSettings(pydantic.BaseSettings): - """Application settings""" - - env: Dict = {} - - timeout: int = 10 - memory: int = 256 - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "CDK_EOAPI_STAC_" - - -class eoRasterSettings(pydantic.BaseSettings): - """Application settings""" - - # Default options are optimized for CloudOptimized GeoTIFF - # For more information on GDAL env see: https://gdal.org/user/configoptions.html - # or https://developmentseed.org/titiler/advanced/performance_tuning/ - env: Dict = { - "CPL_VSIL_CURL_ALLOWED_EXTENSIONS": ".tif,.TIF,.tiff", - "GDAL_CACHEMAX": "200", # 200 mb - "GDAL_DISABLE_READDIR_ON_OPEN": "EMPTY_DIR", - "GDAL_INGESTED_BYTES_AT_OPEN": "32768", - "GDAL_HTTP_MERGE_CONSECUTIVE_RANGES": "YES", - "GDAL_HTTP_MULTIPLEX": "YES", - "GDAL_HTTP_VERSION": "2", - "PYTHONWARNINGS": "ignore", - "VSI_CACHE": "TRUE", - "VSI_CACHE_SIZE": "5000000", # 5 MB (per file-handle) - "DB_MIN_CONN_SIZE": "1", - "DB_MAX_CONN_SIZE": "1", - } - - # S3 bucket names where TiTiler could do HEAD and GET Requests - # specific private and public buckets MUST be added if you want to use s3:// urls - # You can whitelist all bucket by setting `*`. - # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html - buckets: List = ["*"] - - # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif") - key: str = "*" - - timeout: int = 10 - memory: int = 3008 - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "CDK_EOAPI_RASTER_" - - -class eoVectorSettings(pydantic.BaseSettings): - """Application settings""" - - env: Dict = {} - - timeout: int = 10 - memory: int = 512 - - class Config: - """model config""" - - env_file = ".env" - env_prefix = "CDK_EOAPI_VECTOR_" +from aws_cdk import aws_ec2 +from pydantic_core.core_schema import FieldValidationInfo +from pydantic_settings import BaseSettings + +DEFAULT_PROJECT_ID = "eoapi" +DEFAULT_STAGE = "staging" +DEFAULT_NAT_GATEWAY_COUNT = 1 + + +class Config(BaseSettings): + project_id: Optional[str] = pydantic.Field( + description="Project ID", default=DEFAULT_PROJECT_ID + ) + stage: Optional[str] = pydantic.Field( + description="Stage of deployment", default=DEFAULT_STAGE + ) + tags: Optional[Dict[str, str]] = pydantic.Field( + description="""Tags to apply to resources. If none provided, + will default to the defaults defined in `default_tags`. + Note that if tags are passed to the CDK CLI via `--tags`, + they will override any tags defined here.""", + default=None, + ) + auth_provider_jwks_url: Optional[str] = pydantic.Field( + description="""Auth Provider JSON Web Key Set URL for + ingestion authentication. If not provided, + no authentication will be required.""", + default=None, + ) + data_access_role_arn: Optional[str] = pydantic.Field( + description="""Role ARN for data access, that will be + used by the STAC ingestor for validation of assets + located in S3 and for the tiler application to access + assets located in S3. If none, the role will be + created at runtime with full S3 read access. If + provided, the existing role must be configured to + allow the tiler and STAC ingestor lambda roles to + assume it. See https://github.com/developmentseed/eoapi-cdk""", + default=None, + ) + db_instance_type: Optional[str] = pydantic.Field( + description="Database instance type", default="t3.micro" + ) + db_allocated_storage: Optional[int] = pydantic.Field( + description="Allocated storage for the database", default=5 + ) + public_db_subnet: Optional[bool] = pydantic.Field( + description="Whether to put the database in a public subnet", default=False + ) + nat_gateway_count: Optional[int] = pydantic.Field( + description="Number of NAT gateways to create", + default=DEFAULT_NAT_GATEWAY_COUNT, + ) + bastion_host_create_elastic_ip: Optional[bool] = pydantic.Field( + description="Whether to create an elastic IP for the bastion host", + default=False, + ) + bastion_host_allow_ip_list: Optional[List[str]] = pydantic.Field( + description="""YAML file containing list of IP addresses to + allow SSH access to the bastion host""", + default=[], + ) + bastion_host_user_data: Optional[ + Union[Dict[str, Any], aws_ec2.UserData] + ] = pydantic.Field( + description="Path to file containing user data for the bastion host", + default=aws_ec2.UserData.for_linux(), + ) + titiler_buckets: Optional[List[str]] = pydantic.Field( + description="""Path to YAML file containing list of + buckets to grant access to the titiler API""", + default=[], + ) + + @pydantic.field_validator("tags") + def default_tags(cls, v, info: FieldValidationInfo): + return v or {"project_id": info.data["project_id"], "stage": info.data["stage"]} + + @pydantic.field_validator("nat_gateway_count") + def validate_nat_gateway_count(cls, v, info: FieldValidationInfo): + if not info.data["public_db_subnet"] and v <= 0: + raise ValueError( + """if the database and its associated services instances + are to be located in the private subnet of the VPC, NAT + gateways are needed to allow egress from the services + and therefore `nat_gateway_count` has to be > 0.""" + ) + else: + return v + + def build_service_name(self, service_id: str) -> str: + return f"{self.project_id}-{self.stage}-{service_id}" \ No newline at end of file diff --git a/infrastructure/aws/cdk/pgStacInfra.py b/infrastructure/aws/cdk/pgStacInfra.py new file mode 100644 index 0000000..487131e --- /dev/null +++ b/infrastructure/aws/cdk/pgStacInfra.py @@ -0,0 +1,186 @@ +from typing import Optional, Union + +import boto3 +from aws_cdk import Stack, aws_ec2, aws_iam, aws_rds +from constructs import Construct +from eoapi_cdk import ( + BastionHost, + PgStacApiLambda, + PgStacDatabase, + StacIngestor, + TitilerPgstacApiLambda, +) + + +class pgStacInfraStack(Stack): + def __init__( + self, + scope: Construct, + id: str, + vpc: aws_ec2.Vpc, + stage: str, + db_allocated_storage: int, + public_db_subnet: bool, + db_instance_type: str, + stac_api_lambda_name: str, + titiler_pgstac_api_lambda_name: str, + bastion_host_allow_ip_list: list, + bastion_host_create_elastic_ip: bool, + titiler_buckets: list, + data_access_role_arn: Optional[str], + auth_provider_jwks_url: Optional[str], + bastion_host_user_data: Union[str, aws_ec2.UserData], + **kwargs, + ) -> None: + super().__init__(scope, id, **kwargs) + + pgstac_db = PgStacDatabase( + self, + "pgstac-db", + vpc=vpc, + engine=aws_rds.DatabaseInstanceEngine.postgres( + version=aws_rds.PostgresEngineVersion.VER_14 + ), + vpc_subnets=aws_ec2.SubnetSelection( + subnet_type=aws_ec2.SubnetType.PUBLIC + if public_db_subnet + else aws_ec2.SubnetType.PRIVATE_ISOLATED + ), + allocated_storage=db_allocated_storage, + instance_type=aws_ec2.InstanceType(db_instance_type), + ) + + stac_api_lambda = PgStacApiLambda( + self, + "pgstac-api", + api_env={"NAME": stac_api_lambda_name, "description": f"{stage} STAC API"}, + vpc=vpc, + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=aws_ec2.SubnetSelection( + subnet_type=aws_ec2.SubnetType.PUBLIC + if public_db_subnet + else aws_ec2.SubnetType.PRIVATE_WITH_EGRESS + ), + ) + + TitilerPgstacApiLambda( + self, + "titiler-pgstac-api", + api_env={ + "NAME": titiler_pgstac_api_lambda_name, + "description": f"{stage} titiler pgstac API", + }, + vpc=vpc, + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=aws_ec2.SubnetSelection( + subnet_type=aws_ec2.SubnetType.PUBLIC + if public_db_subnet + else aws_ec2.SubnetType.PRIVATE_WITH_EGRESS + ), + buckets=titiler_buckets, + ) + + BastionHost( + self, + "bastion-host", + vpc=vpc, + db=pgstac_db.db, + ipv4_allowlist=bastion_host_allow_ip_list, + user_data=aws_ec2.UserData.custom(bastion_host_user_data) + if bastion_host_user_data + else aws_ec2.UserData.for_linux(), + create_elastic_ip=bastion_host_create_elastic_ip, + ) + + if data_access_role_arn: + # importing provided role from arn. + # the stac ingestor will try to assume it when called, + # so it must be listed in the data access role trust policy. + data_access_role = aws_iam.Role.from_role_arn( + self, + "data-access-role", + role_arn=data_access_role_arn, + ) + else: + data_access_role = self._create_data_access_role() + + stac_ingestor_env = {"REQUESTER_PAYS": "True"} + + if auth_provider_jwks_url: + stac_ingestor_env["JWKS_URL"] = auth_provider_jwks_url + + stac_ingestor = StacIngestor( + self, + "stac-ingestor", + stac_url=stac_api_lambda.url, + stage=stage, + vpc=vpc, + data_access_role=data_access_role, + stac_db_secret=pgstac_db.pgstac_secret, + stac_db_security_group=pgstac_db.db.connections.security_groups[0], + subnet_selection=aws_ec2.SubnetSelection( + subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS + ), + api_env=stac_ingestor_env, + ) + + # we can only do that if the role is created here. + # If injecting a role, that role's trust relationship + # must be already set up, or set up after this deployment. + if not data_access_role_arn: + data_access_role = self._grant_assume_role_with_principal_pattern( + data_access_role, stac_ingestor.handler_role.role_name + ) + + def _create_data_access_role(self) -> aws_iam.Role: + """ + Creates an IAM role with full S3 read access. + """ + + data_access_role = aws_iam.Role( + self, + "data-access-role", + assumed_by=aws_iam.ServicePrincipal("lambda.amazonaws.com"), + ) + + data_access_role.add_to_policy( + aws_iam.PolicyStatement( + actions=[ + "s3:Get*", + ], + resources=["*"], + effect=aws_iam.Effect.ALLOW, + ) + ) + return data_access_role + + def _grant_assume_role_with_principal_pattern( + self, + role_to_assume: aws_iam.Role, + principal_pattern: str, + account_id: str = boto3.client("sts").get_caller_identity().get("Account"), + ) -> aws_iam.Role: + """ + Grants assume role permissions to the role of the given + account with the given name pattern. Default account + is the current account. + """ + + role_to_assume.assume_role_policy.add_statements( + aws_iam.PolicyStatement( + effect=aws_iam.Effect.ALLOW, + principals=[aws_iam.AnyPrincipal()], + actions=["sts:AssumeRole"], + conditions={ + "StringLike": { + "aws:PrincipalArn": [ + f"arn:aws:iam::{account_id}:role/{principal_pattern}" + ] + } + }, + ) + ) + + return role_to_assume \ No newline at end of file diff --git a/infrastructure/aws/cdk/vpc.py b/infrastructure/aws/cdk/vpc.py new file mode 100644 index 0000000..0f0f8a5 --- /dev/null +++ b/infrastructure/aws/cdk/vpc.py @@ -0,0 +1,50 @@ +from aws_cdk import Stack, aws_ec2 +from constructs import Construct + + +class VpcStack(Stack): + def __init__( + self, scope: Construct, id: str, nat_gateway_count: int, **kwargs + ) -> None: + super().__init__(scope, id, **kwargs) + + self.vpc = aws_ec2.Vpc( + self, + "vpc", + subnet_configuration=[ + aws_ec2.SubnetConfiguration( + name="ingress", subnet_type=aws_ec2.SubnetType.PUBLIC, cidr_mask=24 + ), + aws_ec2.SubnetConfiguration( + name="application", + subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS, + cidr_mask=24, + ), + aws_ec2.SubnetConfiguration( + name="rds", + subnet_type=aws_ec2.SubnetType.PRIVATE_ISOLATED, + cidr_mask=24, + ), + ], + nat_gateways=nat_gateway_count, + ) + + self.vpc.add_gateway_endpoint( + "DynamoDbEndpoint", service=aws_ec2.GatewayVpcEndpointAwsService.DYNAMODB + ) + + self.vpc.add_interface_endpoint( + "SecretsManagerEndpoint", + service=aws_ec2.InterfaceVpcEndpointAwsService.SECRETS_MANAGER, + ) + + self.export_value( + self.vpc.select_subnets(subnet_type=aws_ec2.SubnetType.PUBLIC) + .subnets[0] + .subnet_id + ) + self.export_value( + self.vpc.select_subnets(subnet_type=aws_ec2.SubnetType.PUBLIC) + .subnets[1] + .subnet_id + ) \ No newline at end of file diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index 9dd5762..565c22a 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,9 +1,9 @@ # aws cdk aws-cdk-lib==2.94.0 -aws_cdk-aws_apigatewayv2_alpha==2.94.0a0 -aws_cdk-aws_apigatewayv2_integrations_alpha==2.94.0a0 constructs>=10.0.0 - # pydantic settings -pydantic~=1.0 +pydantic==2.0.2 python-dotenv +eoapi-cdk==5.4.0 +boto3==1.24.15 +pydantic-settings==2.0.1 \ No newline at end of file From ffed2b453e4de995bcde4b64a91cd9cf0bdb3e4e Mon Sep 17 00:00:00 2001 From: emileten Date: Wed, 13 Sep 2023 16:41:24 +0900 Subject: [PATCH 2/2] update --- infrastructure/aws/.env.example | 19 --- infrastructure/aws/cdk/app.py | 41 +----- infrastructure/aws/cdk/config.py | 93 +++++++++++- infrastructure/aws/cdk/pgStacInfra.py | 188 +++++++++++++++++++----- infrastructure/aws/cdk/vpc.py | 17 ++- infrastructure/aws/config.yaml | 12 ++ infrastructure/aws/requirements-cdk.txt | 21 ++- 7 files changed, 280 insertions(+), 111 deletions(-) delete mode 100644 infrastructure/aws/.env.example create mode 100644 infrastructure/aws/config.yaml diff --git a/infrastructure/aws/.env.example b/infrastructure/aws/.env.example deleted file mode 100644 index 057c721..0000000 --- a/infrastructure/aws/.env.example +++ /dev/null @@ -1,19 +0,0 @@ -# STACK -CDK_EOAPI_NAME=eoAPI -CDK_EOAPI_FUNCTIONS='["stac","raster","vector"]' - -# DB -CDK_EOAPI_DB_PGSTAC_VERSION="0.7.1" -CDK_EOAPI_DB_PGSTAC_MOSAIC_INDEX=TRUE - -# STAC API -CDK_EOAPI_STAC_TIMEOUT=20 -CDK_EOAPI_STAC_MEMORY=1024 - -# Raster API -CDK_EOAPI_RASTER_TIMEOUT=20 -CDK_EOAPI_RASTER_MEMORY=3008 - -# Vector -CDK_EOAPI_VECTOR_TIMEOUT=20 -CDK_EOAPI_VECTOR_MEMORY=1024 diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index bbf8cf0..daa258b 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -1,46 +1,17 @@ -import yaml from aws_cdk import App -from config import Config -from eoapi_template import pgStacInfra, vpc +from config import build_app_config +import pgStacInfra, vpc app = App() -try: - with open("config.yaml") as f: - config = yaml.safe_load(f) - config = ( - {} if config is None else config - ) # if config is empty, set it to an empty dict - config = Config(**config) -except FileNotFoundError: - # if no config at the expected path, using defaults - config = Config() - -vpc_stack = vpc.VpcStack( - tags=config.tags, - scope=app, - id=config.build_service_name("pgSTAC-vpc"), - nat_gateway_count=config.nat_gateway_count, -) +app_config = build_app_config() +vpc_stack = vpc.VpcStack(scope=app, app_config=app_config) pgstac_infra_stack = pgStacInfra.pgStacInfraStack( scope=app, - tags=config.tags, - id=config.build_service_name("pgSTAC-infra"), vpc=vpc_stack.vpc, - stac_api_lambda_name=config.build_service_name("STAC API"), - titiler_pgstac_api_lambda_name=config.build_service_name("titiler pgSTAC API"), - stage=config.stage, - db_allocated_storage=config.db_allocated_storage, - public_db_subnet=config.public_db_subnet, - db_instance_type=config.db_instance_type, - bastion_host_allow_ip_list=config.bastion_host_allow_ip_list, - bastion_host_create_elastic_ip=config.bastion_host_create_elastic_ip, - bastion_host_user_data=yaml.dump(config.bastion_host_user_data), - titiler_buckets=config.titiler_buckets, - data_access_role_arn=config.data_access_role_arn, - auth_provider_jwks_url=config.auth_provider_jwks_url, + app_config=app_config, ) -app.synth() \ No newline at end of file +app.synth() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index bb7fcff..7c4bdf6 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -1,16 +1,17 @@ from typing import Any, Dict, List, Optional, Union import pydantic +import yaml from aws_cdk import aws_ec2 from pydantic_core.core_schema import FieldValidationInfo from pydantic_settings import BaseSettings -DEFAULT_PROJECT_ID = "eoapi" -DEFAULT_STAGE = "staging" +DEFAULT_PROJECT_ID = "eoapi-template-demo" +DEFAULT_STAGE = "test" DEFAULT_NAT_GATEWAY_COUNT = 1 -class Config(BaseSettings): +class AppConfig(BaseSettings): project_id: Optional[str] = pydantic.Field( description="Project ID", default=DEFAULT_PROJECT_ID ) @@ -54,6 +55,12 @@ class Config(BaseSettings): description="Number of NAT gateways to create", default=DEFAULT_NAT_GATEWAY_COUNT, ) + bastion_host: Optional[bool] = pydantic.Field( + description="""Whether to create a bastion host. It can typically + be used to make administrative connections to the database if + `public_db_subnet` is False""", + default=True, + ) bastion_host_create_elastic_ip: Optional[bool] = pydantic.Field( description="Whether to create an elastic IP for the bastion host", default=False, @@ -74,6 +81,39 @@ class Config(BaseSettings): buckets to grant access to the titiler API""", default=[], ) + acm_certificate_arn: Optional[str] = pydantic.Field( + description="""ARN of ACM certificate to use for + custom domain names. If provided, + CDNs are created for all the APIs""", + default=None, + ) + stac_api_custom_domain: Optional[str] = pydantic.Field( + description="""Custom domain name for the STAC API. + Must provide `acm_certificate_arn`""", + default=None, + ) + titiler_pgstac_api_custom_domain: Optional[str] = pydantic.Field( + description="""Custom domain name for the titiler pgstac API. + Must provide `acm_certificate_arn`""", + default=None, + ) + stac_ingestor_api_custom_domain: Optional[str] = pydantic.Field( + description="""Custom domain name for the STAC ingestor API. + Must provide `acm_certificate_arn`""", + default=None, + ) + tipg_api_custom_domain: Optional[str] = pydantic.Field( + description="""Custom domain name for the tipg API. + Must provide `acm_certificate_arn`""", + default=None, + ) + stac_browser_version: Optional[str] = pydantic.Field( + description="""Version of the Radiant Earth STAC browser to deploy. + If none provided, no STAC browser will be deployed. + If provided, `stac_api_custom_domain` must be provided + as it will be used as a backend.""", + default=None, + ) @pydantic.field_validator("tags") def default_tags(cls, v, info: FieldValidationInfo): @@ -91,5 +131,50 @@ def validate_nat_gateway_count(cls, v, info: FieldValidationInfo): else: return v + @pydantic.field_validator("stac_browser_version") + def validate_stac_browser_version(cls, v, info: FieldValidationInfo): + if v is not None and info.data["stac_api_custom_domain"] is None: + raise ValueError( + """If a STAC browser version is provided, + a custom domain must be provided for the STAC API""" + ) + else: + return v + + @pydantic.field_validator("acm_certificate_arn") + def validate_acm_certificate_arn(cls, v, info: FieldValidationInfo): + if v is None and any( + [ + info.data["stac_api_custom_domain"], + info.data["titiler_pgstac_api_custom_domain"], + info.data["stac_ingestor_api_custom_domain"], + info.data["tipg_api_custom_domain"], + ] + ): + raise ValueError( + """If any custom domain is provided, + an ACM certificate ARN must be provided""" + ) + else: + return v + def build_service_name(self, service_id: str) -> str: - return f"{self.project_id}-{self.stage}-{service_id}" \ No newline at end of file + return f"{self.project_id}-{self.stage}-{service_id}" + + +def build_app_config() -> AppConfig: + """Builds the AppConfig object from config.yaml file if exists, + otherwise use defaults""" + try: + with open("config.yaml") as f: + print("Loading config from config.yaml") + app_config = yaml.safe_load(f) + app_config = ( + {} if app_config is None else app_config + ) # if config is empty, set it to an empty dict + app_config = AppConfig(**app_config) + except FileNotFoundError: + # if no config at the expected path, using defaults + app_config = AppConfig() + + return app_config diff --git a/infrastructure/aws/cdk/pgStacInfra.py b/infrastructure/aws/cdk/pgStacInfra.py index 487131e..ac18e9d 100644 --- a/infrastructure/aws/cdk/pgStacInfra.py +++ b/infrastructure/aws/cdk/pgStacInfra.py @@ -1,38 +1,46 @@ -from typing import Optional, Union - import boto3 -from aws_cdk import Stack, aws_ec2, aws_iam, aws_rds +import yaml +from aws_cdk import ( + RemovalPolicy, + Stack, + aws_certificatemanager, + aws_ec2, + aws_iam, + aws_rds, + aws_s3, + aws_lambda +) +from aws_cdk.aws_apigateway import DomainNameOptions +from aws_cdk.aws_apigatewayv2_alpha import DomainName from constructs import Construct from eoapi_cdk import ( BastionHost, PgStacApiLambda, PgStacDatabase, + StacBrowser, StacIngestor, + TiPgApiLambda, TitilerPgstacApiLambda, ) +from config import AppConfig + +APIS_RUNTIME_DIR = 'runtime' class pgStacInfraStack(Stack): def __init__( self, scope: Construct, - id: str, vpc: aws_ec2.Vpc, - stage: str, - db_allocated_storage: int, - public_db_subnet: bool, - db_instance_type: str, - stac_api_lambda_name: str, - titiler_pgstac_api_lambda_name: str, - bastion_host_allow_ip_list: list, - bastion_host_create_elastic_ip: bool, - titiler_buckets: list, - data_access_role_arn: Optional[str], - auth_provider_jwks_url: Optional[str], - bastion_host_user_data: Union[str, aws_ec2.UserData], + app_config: AppConfig, **kwargs, ) -> None: - super().__init__(scope, id, **kwargs) + super().__init__( + scope, + id=app_config.build_service_name("pgSTAC-infra"), + tags=app_config.tags, + **kwargs, + ) pgstac_db = PgStacDatabase( self, @@ -43,79 +51,142 @@ def __init__( ), vpc_subnets=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC - if public_db_subnet + if app_config.public_db_subnet else aws_ec2.SubnetType.PRIVATE_ISOLATED ), - allocated_storage=db_allocated_storage, - instance_type=aws_ec2.InstanceType(db_instance_type), + allocated_storage=app_config.db_allocated_storage, + instance_type=aws_ec2.InstanceType(app_config.db_instance_type), + api_code={'entry': f'{APIS_RUNTIME_DIR}/db', 'index': 'handler.py', 'handler': 'handler'}, ) stac_api_lambda = PgStacApiLambda( self, "pgstac-api", - api_env={"NAME": stac_api_lambda_name, "description": f"{stage} STAC API"}, + api_env={ + "NAME": app_config.build_service_name("STAC API"), + "description": f"{app_config.stage} STAC API", + }, + api_code={'entry': f'{APIS_RUNTIME_DIR}/stac', 'index': 'handler.py', 'handler': 'handler'}, vpc=vpc, db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC - if public_db_subnet + if app_config.public_db_subnet else aws_ec2.SubnetType.PRIVATE_WITH_EGRESS ), + stac_api_domain_name=DomainName( + self, + "stac-api-domain-name", + domain_name=app_config.stac_api_custom_domain, + certificate=aws_certificatemanager.Certificate.from_certificate_arn( + self, + "stac-api-cdn-certificate", + certificate_arn=app_config.acm_certificate_arn, + ), + ) + if app_config.stac_api_custom_domain + else None, ) TitilerPgstacApiLambda( self, "titiler-pgstac-api", api_env={ - "NAME": titiler_pgstac_api_lambda_name, - "description": f"{stage} titiler pgstac API", + "NAME": app_config.build_service_name("titiler pgSTAC API"), + "description": f"{app_config.stage} titiler pgstac API", }, + python_lambda_options={'runtime':f'{APIS_RUNTIME_DIR}/raster','index':'handler.py', 'handler':'handler','memorySize': 3008, 'architecture': aws_lambda.Architecture.X86_64}, vpc=vpc, db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, subnet_selection=aws_ec2.SubnetSelection( subnet_type=aws_ec2.SubnetType.PUBLIC - if public_db_subnet + if app_config.public_db_subnet else aws_ec2.SubnetType.PRIVATE_WITH_EGRESS ), - buckets=titiler_buckets, + buckets=app_config.titiler_buckets, + titiler_pgstac_api_domain_name=DomainName( + self, + "titiler-pgstac-api-domain-name", + domain_name=app_config.titiler_pgstac_api_custom_domain, + certificate=aws_certificatemanager.Certificate.from_certificate_arn( + self, + "titiler-pgstac-api-cdn-certificate", + certificate_arn=app_config.acm_certificate_arn, + ), + ) + if app_config.titiler_pgstac_api_custom_domain + else None, ) - BastionHost( + TiPgApiLambda( self, - "bastion-host", + "tipg-api", + api_env={ + "NAME": app_config.build_service_name("tipg API"), + "description": f"{app_config.stage} tipg API", + }, + api_code={'entry': f'{APIS_RUNTIME_DIR}/vector', 'index': 'handler.py', 'handler': 'handler'}, vpc=vpc, db=pgstac_db.db, - ipv4_allowlist=bastion_host_allow_ip_list, - user_data=aws_ec2.UserData.custom(bastion_host_user_data) - if bastion_host_user_data - else aws_ec2.UserData.for_linux(), - create_elastic_ip=bastion_host_create_elastic_ip, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=aws_ec2.SubnetSelection( + subnet_type=aws_ec2.SubnetType.PUBLIC + if app_config.public_db_subnet + else aws_ec2.SubnetType.PRIVATE_WITH_EGRESS + ), + tipg_api_domain_name=DomainName( + self, + "tipg-api-domain-name", + domain_name=app_config.tipg_api_custom_domain, + certificate=aws_certificatemanager.Certificate.from_certificate_arn( + self, + "tipg-api-cdn-certificate", + certificate_arn=app_config.acm_certificate_arn, + ), + ) + if app_config.tipg_api_custom_domain + else None, ) - if data_access_role_arn: + if app_config.bastion_host: + BastionHost( + self, + "bastion-host", + vpc=vpc, + db=pgstac_db.db, + ipv4_allowlist=app_config.bastion_host_allow_ip_list, + user_data=aws_ec2.UserData.custom( + yaml.dump(app_config.bastion_host_user_data) + ) + if app_config.bastion_host_user_data is not None + else aws_ec2.UserData.for_linux(), + create_elastic_ip=app_config.bastion_host_create_elastic_ip, + ) + + if app_config.data_access_role_arn: # importing provided role from arn. # the stac ingestor will try to assume it when called, # so it must be listed in the data access role trust policy. data_access_role = aws_iam.Role.from_role_arn( self, "data-access-role", - role_arn=data_access_role_arn, + role_arn=app_config.data_access_role_arn, ) else: data_access_role = self._create_data_access_role() stac_ingestor_env = {"REQUESTER_PAYS": "True"} - if auth_provider_jwks_url: - stac_ingestor_env["JWKS_URL"] = auth_provider_jwks_url + if app_config.auth_provider_jwks_url: + stac_ingestor_env["JWKS_URL"] = app_config.auth_provider_jwks_url stac_ingestor = StacIngestor( self, "stac-ingestor", stac_url=stac_api_lambda.url, - stage=stage, + stage=app_config.stage, vpc=vpc, data_access_role=data_access_role, stac_db_secret=pgstac_db.pgstac_secret, @@ -124,12 +195,49 @@ def __init__( subnet_type=aws_ec2.SubnetType.PRIVATE_WITH_EGRESS ), api_env=stac_ingestor_env, + api_code={'entry': f'{APIS_RUNTIME_DIR}/ingestor', 'index': 'handler.py', 'handler': 'handler'}, + ingestor_domain_name_options=DomainNameOptions( + domain_name=app_config.stac_ingestor_api_custom_domain, + certificate=aws_certificatemanager.Certificate.from_certificate_arn( + self, + "stac-ingestor-api-cdn-certificate", + certificate_arn=app_config.acm_certificate_arn, + ), + ) + if app_config.stac_ingestor_api_custom_domain + else None, ) + if app_config.stac_browser_version: + stac_browser_bucket = aws_s3.Bucket( + self, + "stac-browser-bucket", + bucket_name=app_config.build_service_name("stac-browser"), + removal_policy=RemovalPolicy.DESTROY, + auto_delete_objects=True, + website_index_document="index.html", + public_read_access=True, + block_public_access=aws_s3.BlockPublicAccess( + block_public_acls=False, + block_public_policy=False, + ignore_public_acls=False, + restrict_public_buckets=False, + ), + object_ownership=aws_s3.ObjectOwnership.OBJECT_WRITER, + ) + StacBrowser( + self, + "stac-browser", + github_repo_tag=app_config.stac_browser_version, + stac_catalog_url=f"https://{app_config.stac_api_custom_domain}", + website_index_document="index.html", + bucket_arn=stac_browser_bucket.bucket_arn, + ) + # we can only do that if the role is created here. # If injecting a role, that role's trust relationship # must be already set up, or set up after this deployment. - if not data_access_role_arn: + if not app_config.data_access_role_arn: data_access_role = self._grant_assume_role_with_principal_pattern( data_access_role, stac_ingestor.handler_role.role_name ) @@ -183,4 +291,4 @@ def _grant_assume_role_with_principal_pattern( ) ) - return role_to_assume \ No newline at end of file + return role_to_assume diff --git a/infrastructure/aws/cdk/vpc.py b/infrastructure/aws/cdk/vpc.py index 0f0f8a5..bc723f4 100644 --- a/infrastructure/aws/cdk/vpc.py +++ b/infrastructure/aws/cdk/vpc.py @@ -1,12 +1,17 @@ from aws_cdk import Stack, aws_ec2 from constructs import Construct +from config import AppConfig + class VpcStack(Stack): - def __init__( - self, scope: Construct, id: str, nat_gateway_count: int, **kwargs - ) -> None: - super().__init__(scope, id, **kwargs) + def __init__(self, scope: Construct, app_config: AppConfig, **kwargs) -> None: + super().__init__( + scope, + id=app_config.build_service_name("pgSTAC-vpc"), + tags=app_config.tags, + **kwargs + ) self.vpc = aws_ec2.Vpc( self, @@ -26,7 +31,7 @@ def __init__( cidr_mask=24, ), ], - nat_gateways=nat_gateway_count, + nat_gateways=app_config.nat_gateway_count, ) self.vpc.add_gateway_endpoint( @@ -47,4 +52,4 @@ def __init__( self.vpc.select_subnets(subnet_type=aws_ec2.SubnetType.PUBLIC) .subnets[1] .subnet_id - ) \ No newline at end of file + ) diff --git a/infrastructure/aws/config.yaml b/infrastructure/aws/config.yaml new file mode 100644 index 0000000..7c4978d --- /dev/null +++ b/infrastructure/aws/config.yaml @@ -0,0 +1,12 @@ +project_id: "eoapi" +stage: "dev" +tags: + project: "eoapi" +public_db_subnet: True # DB in public subnet +bastion_host: False # no bastion host +acm_certificate_arn: some_arn # for custom domain names +stac_api_custom_domain: stac.eoapi.dev # domain name for STAC API +titiler_pgstac_api_custom_domain: raster.eoapi.dev # .... +stac_ingestor_api_custom_domain: ingestor.eoapi.dev +tipg_api_custom_domain: vector.eoapi.dev +stac_browser_version: v3.1.0 # Radiant Earth GH tag \ No newline at end of file diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index 565c22a..50ae0b8 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,9 +1,16 @@ -# aws cdk -aws-cdk-lib==2.94.0 -constructs>=10.0.0 -# pydantic settings -pydantic==2.0.2 -python-dotenv +aws-cdk-lib>=2.75.0 +aws_cdk.aws_cognito_identitypool_alpha>=2.75.0a0 +aws-cdk.aws-apigatewayv2-alpha==2.95.1a0 eoapi-cdk==5.4.0 +constructs>=10.0.0,<11.0.0 +pydantic==2.0.2 +pydantic-settings==2.0.1 +black==22.3.0 boto3==1.24.15 -pydantic-settings==2.0.1 \ No newline at end of file +boto3-stubs[cognito-idp,cognito-identity] +flake8==4.0.1 +click==8.1.3 +requests==2.28.0 +python-dotenv==1.0.0 +pyyaml==6.0 +types-PyYAML==6.0.12.10