diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68828f9..56af3a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks # Don't run pre-commit on notebooks folder -exclude: ^files-api/notebooks +exclude: ^notebooks repos: - repo: local @@ -13,18 +13,17 @@ repos: name: Generate and Assert that OpenAPI Schema is up to date entry: | bash -c ' + THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - pushd "$THIS_DIR/files-api" - uv run ./scripts/generate-openapi.py generate-and-diff - --existing-spec ./openapi.json - --output-spec ./openapi.json + uv run $THIS_DIR/scripts/generate-openapi.py generate-and-diff \ + --existing-spec ./openapi.json \ + --output-spec ./openapi.json \ --fail-on-diff - popd ' language: system # run this hook if openapi.json or any of the src/*.py files change (since those files generate openapi.json) - files: ^files-api/openapi\.json$|^files-api/src/.*\.py$ + files: ^openapi\.json$|^src/.*\.py$ pass_filenames: false always_run: false @@ -38,23 +37,21 @@ repos: THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - pushd "$THIS_DIR/files-api" - # generate the OpenAPI spec from the latest fastapi app code; - uv run ./scripts/generate-openapi.py generate --output-spec openapi.json; + uv run $THIS_DIR/scripts/generate-openapi.py generate --output-spec openapi.json; # Determine which ref to use: remote if available, otherwise local - if git show refs/heads/main:files-api/openapi.json > /dev/null 2>&1; then - OPENAPI_REF="refs/heads/main:files-api/openapi.json" + if git show refs/heads/main:openapi.json > /dev/null 2>&1; then + OPENAPI_REF="refs/heads/main:openapi.json" else - OPENAPI_REF="origin/main:files-api/openapi.json" + OPENAPI_REF="origin/main:openapi.json" fi # Load the OpenAPI schema from the determined ref git show $OPENAPI_REF > ./openapi-main.json # set the openapi-main.json to be deleted when this hook finishes; - trap "rm $THIS_DIR/files-api/openapi-main.json" EXIT; + trap "rm $THIS_DIR/openapi-main.json" EXIT; # compare the recently generated OpenAPI schema to the one in main and fail if ; # the recently generated one would introduce breaking changes; @@ -64,12 +61,10 @@ repos: /data/openapi-main.json \ /data/openapi.json \ --fail-on ERR - - popd ' language: system # run this hook if openapi.json or any of the src/*.py files change (since those files generate openapi.json) - files: ^files-api/openapi\.json$|^files-api/src/.*\.py$ + files: ^openapi\.json$|^src/.*\.py$ pass_filenames: false always_run: false verbose: false diff --git a/README.md b/README.md index 34238ff..d47fd09 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ This project is a more polished version of the [cloud-engineering-project](https * Setup load testing with Locust * We wrote API Contract using OpenAPI spec, auto-generated from FastAPI code, with pre-commit hooks using `oasdiff` to catch breaking changes and OpenAPI Generator to create a Python client SDK for the API. * Serverless Deployment: Deployed the app using AWS CDK with Docker on AWS Lambda and exposed it via API Gateway. + > *NOTE: After deploying, make sure to update the secret in Secrets Manager with your OpenAI API key.* + * Using AWS Secrets Manager to store OpenAI API keys and + * Using [`AWS-Parameters-and-Secrets-Lambda-Extension`](https://docs.aws.amazon.com/lambda/latest/dg/with-secrets-manager.html) to securely fetch it inside the Lambda function. * CI/CD Pipeline: Automated testing and deployment using GitHub Actions. * Observability & Monitoring: * Setup in-depth logging on AWS CloudWatch using loguru. @@ -32,6 +35,27 @@ This project is a more polished version of the [cloud-engineering-project](https * Custom Metrics with AWS CloudWatch Metrics using `aws-embedded-metrics`. +## TODO + +- [x] Added Secret Manager to store OpenAI API key securely. +- [ ] Setup the Dockerfile with the recommended way of using [uv in Docker](https://docs.astral.sh/uv/guides/integration/docker/). +- [ ] Implement API versioning strategy (like v1 in the path). +- [ ] Implement authentication (API keys or AWS Cognito) and secure Swagger UI page and possiblly the API endpoints as well. + - [ ] Add rate limiting to the API using API Gateway +- [ ] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. + - [ ] Implement multi-environment deployment pipeline (dev/prod) with approval gates +- [ ] Observability & Monitoring improvements: + - [ ] Use OpenTelemetry for tracing instead of AWS X-Ray, [ref](https://aws.amazon.com/blogs/mt/aws-x-ray-sdks-daemon-migration-to-opentelemetry/). + - [ ] Setup Grafana dashboards with CloudWatch data sources for enhanced monitoring + - [ ] Replace Cloudwatch with Grafana Stack -- logs, metrics and traces +- [ ] Container Orchestration: + - [ ] Containerize the app and deploy it using Application Load Balancer + - [ ] ECS Fargate (Serverless) + - [ ] Amazon EC2 + - [ ] Setup auto-scaling based on request load or CPU/memory usage + - [ ] Deploy on Kubernetes EKS (Kubernetes) +- [ ] Add custom domain with Route53 and ACM for HTTPS (`https://api.myapp.com/v1/`) + ## Setup Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), [aws-cli v2](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) and [node.js](https://nodejs.org/en/download) before running. @@ -182,11 +206,6 @@ Install [`uv`](https://docs.astral.sh/uv/getting-started/installation/), [aws-cl Similary, you can view other metrics like Lambda Invocations, Duration, Errors, Throttles etc. and for S3 as well. -## TODO - -- [ ] Setup the Dockerfile with the recommended way of using [uv in Docker](https://docs.astral.sh/uv/guides/integration/docker/). - - ## Contributing Contributions are welcome! diff --git a/docs/images/deployed-api.png b/docs/images/deployed-api.png index 832ae2c..074ca70 100644 Binary files a/docs/images/deployed-api.png and b/docs/images/deployed-api.png differ diff --git a/infra.py b/infra.py index 6132125..a188d22 100644 --- a/infra.py +++ b/infra.py @@ -2,7 +2,13 @@ from pathlib import Path import aws_cdk as cdk -from aws_cdk import Stack, aws_apigateway as apigw, aws_lambda as _lambda, aws_s3 as s3 +from aws_cdk import ( + Stack, + aws_apigateway as apigw, + aws_lambda as _lambda, + aws_s3 as s3, + aws_secretsmanager as secretsmanager, +) from constructs import Construct THIS_DIR = Path(__file__).parent @@ -25,6 +31,23 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: removal_policy=cdk.RemovalPolicy.DESTROY, ) + # Create a Secret to store OpenAI API Key + openai_api_secret_key = secretsmanager.Secret( + self, + id="OpenAIApiSecretKey", + description="OpenAI API Key to generate text, images, and audio using OpenAI's API", + secret_name="files-api/openai-api-key", + # secret_string_value=..., + # ^^^AWS discourages to pass the secret value directly in the CDK code as the value will be included in the + # output of the cdk as part of synthesis, and will appear in the CloudFormation template in the console + removal_policy=cdk.RemovalPolicy.DESTROY, + ) + # ^^^The recommended way is to leave this field empty and manually add the secret value in the Secrets Manager console after deploying the stack. + # AWS Secrets Manager will automatically create a placeholder/empty secret for you + # The secret exists in AWS, but initially has no value (or a generated random value depending on the context). + + # This way, the secret value never appears in code, outputs, or CloudFormation templates. + # Create a Lambda function & Lambda Layer files_api_lambda_layer = _lambda.LayerVersion( self, @@ -54,6 +77,17 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: removal_policy=cdk.RemovalPolicy.DESTROY, ) + # Add AWS parameters and secrets Lambda extension to read secrets from Secrets Manager + # ref: https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_lambda.html + # ref: https://docs.aws.amazon.com/lambda/latest/dg/with-secrets-manager.html + secrets_manager_lambda_extension_layer = _lambda.LayerVersion.from_layer_version_arn( + self, + id="SecretsManagerExtensionLayer", + layer_version_arn=f"arn:aws:lambda:{self.region}:345057560386:layer:AWS-Parameters-and-Secrets-Lambda-Extension-Arm64:23", + ) + # ^^^I found the layer ARN here from the AWS docs: + # https://docs.aws.amazon.com/systems-manager/latest/userguide/ps-integration-lambda-extensions.html#ps-integration-lambda-extensions-add + files_api_lambda = _lambda.Function( self, id="FilesApiLambda", @@ -68,7 +102,7 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: path=(THIS_DIR / "src").as_posix(), exclude=["tests/*", ".venv", "*.pyc", "__pycache__", ".git"], ), - layers=[files_api_lambda_layer], + layers=[files_api_lambda_layer, secrets_manager_lambda_extension_layer], tracing=_lambda.Tracing.ACTIVE, environment={ "S3_BUCKET_NAME": files_api_bucket.bucket_name, @@ -76,13 +110,26 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: "AWS_EMF_NAMESPACE": "files-api", "AWS_XRAY_TRACING_NAME": "Files API", "AWS_XRAY_DAEMON_CONTEXT_MISSING": "RUNTIME_ERROR", - "OPENAI_API_KEY": os.environ["OPENAI_API_KEY"], + # "OPENAI_API_KEY": os.environ["OPENAI_API_KEY"], + "OPENAI_API_SECRET_NAME": openai_api_secret_key.secret_name, + # AWS Parameters and Secrets Lambda Extension configuration + # You can find all the supported environment variables here: + # https://docs.aws.amazon.com/lambda/latest/dg/with-secrets-manager.html + "PARAMETERS_SECRETS_EXTENSION_HTTP_PORT": "2773", + # ^^^Port on which the AWS Parameters and Secrets Lambda Extension listens by default + "PARAMETERS_SECRETS_EXTENSION_CACHE_ENABLED": "TRUE", + # Enable caching of secrets to reduce latency and cost + "SECRETS_MANAGER_TTL": "300", # Cache secrets for 300 seconds (5 minutes) + # Time-to-live for cached secrets. }, ) # Grant the Lambda function permissions to read/write to the S3 bucket files_api_bucket.grant_read_write(files_api_lambda) + # Grant the Lambda function permissions to read the OpenAI API Key secret + openai_api_secret_key.grant_read(files_api_lambda) + # Setup API Gateway with resources and methods # The LambdaRestApi construct by default creates a `test-invoke-stage` stage for the API diff --git a/openapi.json b/openapi.json index 96d1ec6..1029e60 100644 --- a/openapi.json +++ b/openapi.json @@ -3,15 +3,16 @@ "info": { "title": "Files API", "summary": "Store and Retrieve Files.", - "description": " \n\n| Helpful Links | Notes |\n| --- | --- |\n| [MLOps Club](https://mlops-club.org) | ![MLOps Club](https://img.shields.io/badge/Memember%20of-MLOps%20Club-05998B?style=for-the-badge) |\n| [Project Repo](https://github.com/avr2002/cloud-engineering-project) | `avr2002/cloud-engineering-project` |\n", + "description": " \n \n \n
\n", "contact": { "name": "Amit Vikram Raj", "url": "https://www.linkedin.com/in/avr27/", "email": "raj.amitvikram@gmail.com" }, "license": { - "name": "Apache 2.0", - "identifier": "MIT" + "name": "MIT License", + "identifier": "MIT", + "url": "https://github.com/avr2002/files-api/blob/main/LICENSE" }, "version": "v1" }, diff --git a/pyproject.toml b/pyproject.toml index 0dea802..0dc9077 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "files-api" -version = "0.1.2" +version = "0.1.3" description = "Files API to upload and download files from S3 using FastAPI." readme = "README.md" authors = [ diff --git a/run b/run index 08fea73..eb20a29 100755 --- a/run +++ b/run @@ -1,25 +1,28 @@ #!/usr/bin/env bash -set -euox pipefail +set -euo pipefail export AWS_PROFILE=sandbox export AWS_REGION=us-west-2 export AWS_DEFAULT_REGION=${AWS_REGION} # aws account id/hash used to make a unique, but consistent bucket name -export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text || echo "") -# Cache AWS account ID for 10 hours using bash-cache -# AWS_ACCOUNT_ID=$(bc::cache 10h aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) +# Only create bucket name if AWS_ACCOUNT_ID is available +# Suppress errors if profile doesn't exist (e.g., in CI/CD) +if [ -n "$AWS_ACCOUNT_ID" ]; then + AWS_ACCOUNT_ID_HASH=$(echo -n "${AWS_ACCOUNT_ID}" | sha256sum | cut -c5-8) + export S3_BUCKET_NAME="python-aws-cloud-course-bucket-${AWS_ACCOUNT_ID_HASH}" # python-aws-cloud-course-bucket-6721 +else + echo "WARNING: Could not retrieve AWS Account ID. Profile '$AWS_PROFILE' may not exist." + echo "S3_BUCKET_NAME environment variable will not be set." +fi -# Cache AWS account ID for 10 hours using our custom function -# AWS_ACCOUNT_ID=$(get-cached-aws-account-id 10) -AWS_ACCOUNT_ID_HASH=$(echo -n "${AWS_ACCOUNT_ID}" | sha256sum | cut -c5-8) -export S3_BUCKET_NAME="python-aws-cloud-course-bucket-${AWS_ACCOUNT_ID_HASH}" # python-aws-cloud-course-bucket-6721 -THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ########################## @@ -40,12 +43,12 @@ cdk-deploy () { exit 1 fi - if [ ! -f "$THIS_DIR/.openai.env" ]; then - echo "No OpenAI environment file found. Please create a .openai.env file with the OpenAI API key." - return 1 - fi + # If S3_BUCKET_NAME is not set, exit with error + if [ -z "${S3_BUCKET_NAME:-}" ]; then + echo "ERROR: S3_BUCKET_NAME is not set. Cannot deploy CDK stack." + exit 1 + fi - source "$THIS_DIR/.openai.env" cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION } @@ -311,7 +314,7 @@ lint:ci() { # remove all files generated by tests, builds, or operating this codebase clean() { - rm -rf dist build coverage.xml test-reports + rm -rf dist build coverage.xml test-reports/ find . \ -type d \ \( \ @@ -329,9 +332,6 @@ clean() { -name "*.pyc" \ -not -path "*env/*" \ -exec rm {} + - - # Clean AWS account ID cache - rm -f /tmp/.aws_account_id_cache_* } diff --git a/src/files_api/aws_lambda_handler.py b/src/files_api/aws_lambda_handler.py index 685c2c9..e3497db 100644 --- a/src/files_api/aws_lambda_handler.py +++ b/src/files_api/aws_lambda_handler.py @@ -4,13 +4,57 @@ Repository: https://github.com/jordaneremieff/mangum """ +import json +import os +import urllib.request + from mangum import Mangum from files_api.main import create_app +# Use the AWS-Parameters-and-Secrets-Lambda-Extension to retrieve secrets from Secrets Manager + + +# ref: https://docs.aws.amazon.com/systems-manager/latest/userguide/ps-integration-lambda-extensions.html +def get_secret_from_extension(secret_name: str) -> str: + """Retrieves a secret value from the Secrets Manager extension.""" + # The extension runs on localhost port 2773 by default + _extension_routing_port: str = os.environ["PARAMETERS_SECRETS_EXTENSION_HTTP_PORT"] + + # AWS_SESSION_TOKEN is required for authentication with the extension with the Secrets Manager + _aws_session_auth_token: str = os.environ["AWS_SESSION_TOKEN"] + + endpoint = f"http://localhost:{_extension_routing_port}/secretsmanager/get?secretId={secret_name}" + + # Use the session token to authenticate with the extension + req = urllib.request.Request(url=endpoint) + req.add_header("X-Aws-Parameters-Secrets-Token", _aws_session_auth_token) + + # Request/Respone Syntax: https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html + with urllib.request.urlopen(req) as response: + secret_response = response.read().decode("utf-8") + # The response is a JSON string containing the secret value + secret_data = json.loads(secret_response) + return secret_data["SecretString"] + + +# Cache the secret - only fetch once per cold start +_CACHED_OPENAI_API_KEY: str | None = None + + +# Create the FastAPI application APP = create_app() def handler(event, context): + global _CACHED_OPENAI_API_KEY # noqa: PLW0603 + # ^^^Using the global statement to update `_CACHED_OPENAI_API_KEY` is discouraged + + # Only fetch if not already cached + if _CACHED_OPENAI_API_KEY is None: + # Export the OpenAI API Key from secerets manager as an environment variable + _CACHED_OPENAI_API_KEY = get_secret_from_extension(secret_name=os.environ["OPENAI_API_SECRET_NAME"]) + os.environ["OPENAI_API_KEY"] = _CACHED_OPENAI_API_KEY + response = Mangum(APP)(event, context) return response diff --git a/src/files_api/generate_files/__init__.py b/src/files_api/generate_files/__init__.py new file mode 100644 index 0000000..2f48c94 --- /dev/null +++ b/src/files_api/generate_files/__init__.py @@ -0,0 +1 @@ +"""Generate AI files using OpenAI and Gemini API.""" diff --git a/src/files_api/generate_files.py b/src/files_api/generate_files/openai.py similarity index 100% rename from src/files_api/generate_files.py rename to src/files_api/generate_files/openai.py diff --git a/src/files_api/main.py b/src/files_api/main.py index 743832b..ca33177 100644 --- a/src/files_api/main.py +++ b/src/files_api/main.py @@ -41,11 +41,13 @@ def create_app(settings: Union[Settings, None] = None) -> FastAPI: \ \ - - | Helpful Links | Notes | - | --- | --- | - | [MLOps Club](https://mlops-club.org) | ![MLOps Club](https://img.shields.io/badge/Memember%20of-MLOps%20Club-05998B?style=for-the-badge) | - | [Project Repo](https://github.com/avr2002/cloud-engineering-project) | `avr2002/cloud-engineering-project` | + \ + \ + + \ + \ + +
""" ), contact={ @@ -53,7 +55,11 @@ def create_app(settings: Union[Settings, None] = None) -> FastAPI: "url": "https://www.linkedin.com/in/avr27/", "email": "raj.amitvikram@gmail.com", }, - license_info={"name": "Apache 2.0", "identifier": "MIT"}, + license_info={ + "name": "MIT License", + "identifier": "MIT", + "url": "https://github.com/avr2002/files-api/blob/main/LICENSE", + }, docs_url="/", # its easier to find the docs when they live on the base url redoc_url="/redoc", root_path="/prod", # adding stage name to the root path diff --git a/src/files_api/routes.py b/src/files_api/routes.py index dbeab28..ce15c11 100644 --- a/src/files_api/routes.py +++ b/src/files_api/routes.py @@ -19,7 +19,7 @@ from fastapi.responses import StreamingResponse from loguru import logger -from files_api.generate_files import ( +from files_api.generate_files.openai import ( generate_image, generate_text_to_speech, get_text_chat_completion, @@ -410,6 +410,12 @@ async def generate_file_using_openai( image_response = requests.get(image_url) # pylint: disable=missing-timeout file_content_bytes = image_response.content + # For Gemini image generation: + # image_bytes = await generate_image(prompt=body.prompt) + # if image_bytes is None: + # raise ValueError("Failed to generate image from Gemini") + # file_content_bytes = image_bytes + logger.debug("Image file generated successfully, image_url: {image_url}", image_url=image_url) else: response_format = body.file_path.split(".")[-1] diff --git a/uv.lock b/uv.lock index f09b631..954e9a5 100644 --- a/uv.lock +++ b/uv.lock @@ -930,7 +930,7 @@ wheels = [ [[package]] name = "files-api" -version = "0.1.2" +version = "0.1.3" source = { editable = "." } dependencies = [ { name = "aws-embedded-metrics" },