From 34bef784d7cb1851dbf606cb593c0a3bbad9e453 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 14:31:53 +0530 Subject: [PATCH 01/23] feat: enabled access logs for API Gateway with custom log format --- infra.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/infra.py b/infra.py index 79222c2..bdb7c21 100644 --- a/infra.py +++ b/infra.py @@ -1,4 +1,5 @@ import hashlib +import json import os from pathlib import Path @@ -7,6 +8,7 @@ Stack, aws_apigateway as apigw, aws_lambda as _lambda, + aws_logs as logs, aws_s3 as s3, aws_ssm as ssm, ) @@ -168,9 +170,18 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Setup API Gateway with resources and methods - # The LambdaRestApi construct by default creates a `test-invoke-stage` stage for the API + # Log group for API Gateway access logs + api_gw_access_log_group_prod = logs.LogGroup( + self, + id="FilesApiGwAccessLogGroup", + log_group_name="/aws/apigateway/access-logs/files-api/prod", + retention=logs.RetentionDays.ONE_MONTH, + removal_policy=cdk.RemovalPolicy.DESTROY, + ) + + # The LambdaRestApi L2 construct by default creates a `test-invoke-stage` stage for the API - # I disabled the proxy integration(proxy=False) because on the root path it defined "ANY" method by default which we do not want. + # I disabled the proxy integration(proxy=False) because on the root path it defined "ANY" method by default which we do not want. # For the root path we only want to allow "GET" method to access the OpenAPI docs page. files_api_gw = apigw.LambdaRestApi( self, @@ -184,7 +195,16 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: deploy_options=apigw.StageOptions( stage_name="prod", tracing_enabled=True, + metrics_enabled=True, + logging_level=apigw.MethodLoggingLevel.INFO, + access_log_destination=apigw.LogGroupLogDestination(log_group=api_gw_access_log_group_prod), + # access_log_format=apigw.AccessLogFormat.clf(), # Common Log Format for access logs + # access_log_format=apigw.AccessLogFormat.json_with_standard_fields(...) # Pre-defined JSON format with standard fields + access_log_format=apigw_custom_access_log_format(), # Custom JSON format for access logs ), + # Setting cloudWatchRole to true ensures CDK creates the necessary IAM role for logging + cloud_watch_role=True, + cloud_watch_role_removal_policy=cdk.RemovalPolicy.DESTROY, # endpoint_configuration=apigw.EndpointConfiguration( # types=[apigw.EndpointType.REGIONAL], # Use REGIONAL endpoint type for cost-effectiveness # ), @@ -220,20 +240,59 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # self, # id="FilesApiGatewayUrl", # value=files_api_gw.url, - # description="Files-API API Gateway URL", + # description="Files-API API Gateway Invoke URL", # ) # Print an output saying manually create the SSM SecureString parameter cdk.CfnOutput( self, id="ManualSSMParameterCreationNotice", - value="Please remember to manually create the SSM Parameter Store SecureString parameter" - " '/files-api/openai-api-key' with your OpenAI API Key before deploying the stack." - f" You can create it here: https://{self.region}.console.aws.amazon.com/systems-manager/parameters/", + value="Please remember to manually create a SecureString parameter in AWS SSM Parameter Store with name" + " '/files-api/openai-api-key' with your OpenAI API Key after the first deployment of this stack.\n" + f"You can create it here: https://{self.region}.console.aws.amazon.com/systems-manager/parameters/", description="Manual SSM Parameter Creation Notice", ) +def apigw_custom_access_log_format() -> apigw.AccessLogFormat: + """ + Custom API Gateway Access Log Format based on Alex DeBrie's blog post. + + Ref: https://www.alexdebrie.com/posts/api-gateway-access-logs/#access-logging-fields + """ + # ref: Refer to Alex DeBrie's blog post for custom access log format: + return apigw.AccessLogFormat.custom( + json.dumps( + { # Request Information + "requestTime": apigw.AccessLogField.context_request_time, + "requestId": apigw.AccessLogField.context_request_id, + # There is slight difference in requestId & extendedRequestId: Clients can override the requestID + # but not the extendedRequestId, which may be helpful for troubleshooting & debugging purposes + "extendedRequestId": apigw.AccessLogField.context_extended_request_id, + "httpMethod": apigw.AccessLogField.context_http_method, + "path": apigw.AccessLogField.context_path, + "resourcePath": apigw.AccessLogField.context_resource_path, + "status": apigw.AccessLogField.context_status, + "responseLatency": apigw.AccessLogField.context_response_latency, # in milliseconds + "xrayTraceId": apigw.AccessLogField.context_xray_trace_id, + # Integration Information + # AWS Endpoint Request ID: The requestID generated by Lambda function invocation + # "integrationRequestId": apigw.AccessLogField.context_integration_request_id, + "integrationRequestId": "$context.integration.requestId", + # Integration Response Status Code: Status code returned by the AWS Lambda function + "functionResponseStatus": apigw.AccessLogField.context_integration_status, + # Latency of the integration, like Lambda function, in milliseconds + "integrationLatency": apigw.AccessLogField.context_integration_latency, + # Status code returned by the AWS Lambda Service and not the backend Lambda function code + "integrationServiceStatus": apigw.AccessLogField.context_integration_status, + # User Identity Information + "ip": apigw.AccessLogField.context_identity_source_ip, + "userAgent": apigw.AccessLogField.context_identity_user_agent, + } + ), + ) + + ############### # --- App --- # ############### From 8f70ffb034a67c0affbf8797549aba1598677a5b Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 14:53:51 +0530 Subject: [PATCH 02/23] fix: added paranthesis to function calls for `apigw.AccessLogField` construct --- infra.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/infra.py b/infra.py index bdb7c21..25538fb 100644 --- a/infra.py +++ b/infra.py @@ -264,30 +264,30 @@ def apigw_custom_access_log_format() -> apigw.AccessLogFormat: return apigw.AccessLogFormat.custom( json.dumps( { # Request Information - "requestTime": apigw.AccessLogField.context_request_time, - "requestId": apigw.AccessLogField.context_request_id, + "requestTime": apigw.AccessLogField.context_request_time(), + "requestId": apigw.AccessLogField.context_request_id(), # There is slight difference in requestId & extendedRequestId: Clients can override the requestID # but not the extendedRequestId, which may be helpful for troubleshooting & debugging purposes - "extendedRequestId": apigw.AccessLogField.context_extended_request_id, - "httpMethod": apigw.AccessLogField.context_http_method, - "path": apigw.AccessLogField.context_path, - "resourcePath": apigw.AccessLogField.context_resource_path, - "status": apigw.AccessLogField.context_status, - "responseLatency": apigw.AccessLogField.context_response_latency, # in milliseconds - "xrayTraceId": apigw.AccessLogField.context_xray_trace_id, + "extendedRequestId": apigw.AccessLogField.context_extended_request_id(), + "httpMethod": apigw.AccessLogField.context_http_method(), + "path": apigw.AccessLogField.context_path(), + "resourcePath": apigw.AccessLogField.context_resource_path(), + "status": apigw.AccessLogField.context_status(), + "responseLatency": apigw.AccessLogField.context_response_latency(), # in milliseconds + "xrayTraceId": apigw.AccessLogField.context_xray_trace_id(), # Integration Information # AWS Endpoint Request ID: The requestID generated by Lambda function invocation # "integrationRequestId": apigw.AccessLogField.context_integration_request_id, "integrationRequestId": "$context.integration.requestId", # Integration Response Status Code: Status code returned by the AWS Lambda function - "functionResponseStatus": apigw.AccessLogField.context_integration_status, + "functionResponseStatus": apigw.AccessLogField.context_integration_status(), # Latency of the integration, like Lambda function, in milliseconds - "integrationLatency": apigw.AccessLogField.context_integration_latency, + "integrationLatency": apigw.AccessLogField.context_integration_latency(), # Status code returned by the AWS Lambda Service and not the backend Lambda function code - "integrationServiceStatus": apigw.AccessLogField.context_integration_status, + "integrationServiceStatus": apigw.AccessLogField.context_integration_status(), # User Identity Information - "ip": apigw.AccessLogField.context_identity_source_ip, - "userAgent": apigw.AccessLogField.context_identity_user_agent, + "ip": apigw.AccessLogField.context_identity_source_ip(), + "userAgent": apigw.AccessLogField.context_identity_user_agent(), } ), ) From de1e784e054cb753d80e18e6bc07cf4c4c958a97 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 15:09:44 +0530 Subject: [PATCH 03/23] feat: create explicit log group for Lambda function with retention policy --- infra.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/infra.py b/infra.py index 25538fb..5b8d4be 100644 --- a/infra.py +++ b/infra.py @@ -122,6 +122,17 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # ^^^I found the layer ARN here from the AWS docs: # https://docs.aws.amazon.com/systems-manager/latest/userguide/ps-integration-lambda-extensions.html#ps-integration-lambda-extensions-add + # Log group for Lambda function + # Lambda by default creates a log group of format /aws/lambda/ + # But here we are explicitly creating it to set retention and removal policy + files_api_lambda_log_group = logs.LogGroup( + self, + id="FilesApiLambdaLogGroup", + log_group_name="/aws/lambda/files-api", + retention=logs.RetentionDays.ONE_MONTH, + removal_policy=cdk.RemovalPolicy.DESTROY, + ) + files_api_lambda = _lambda.Function( self, id="FilesApiLambda", @@ -138,6 +149,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ), # Add Lambda Layers for dependencies and AWS Secrets Manager extension layers=[files_api_lambda_layer, secrets_manager_lambda_extension_layer], + # Specify the log group for the Lambda function + log_group=files_api_lambda_log_group, # Enable X-Ray Tracing for the Lambda function tracing=_lambda.Tracing.ACTIVE, environment={ @@ -168,6 +181,9 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Grant the Lambda function permissions to read the OpenAI API Key secret from SSM Parameter Store ssm_openai_api_secret_key.grant_read(files_api_lambda) + # Grant the Lambda function permissions to write logs to CloudWatch Logs + files_api_lambda_log_group.grant_write(files_api_lambda) + # Setup API Gateway with resources and methods # Log group for API Gateway access logs From 15496cf82bb893a0b2d24bdc5059068a9b4a2bf9 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 15:19:48 +0530 Subject: [PATCH 04/23] feat: turned off API Gateway execution logs --- infra.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/infra.py b/infra.py index 5b8d4be..b39b426 100644 --- a/infra.py +++ b/infra.py @@ -212,7 +212,11 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: stage_name="prod", tracing_enabled=True, metrics_enabled=True, - logging_level=apigw.MethodLoggingLevel.INFO, + # logging_level=apigw.MethodLoggingLevel.INFO, + # ^^^API Gateway Execution Logs - it is recommended to turn it off in production + # Execution logs capture detailed information about API request processing lifecycle. It should be + # enabled only for debugging purposes as it can quite verbose and incur additional cloudwatch costs + # and may expose sensitive information. access_log_destination=apigw.LogGroupLogDestination(log_group=api_gw_access_log_group_prod), # access_log_format=apigw.AccessLogFormat.clf(), # Common Log Format for access logs # access_log_format=apigw.AccessLogFormat.json_with_standard_fields(...) # Pre-defined JSON format with standard fields From a1e4eab1d0f1df3dbf0015a6e739776dcc754f27 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 16:38:28 +0530 Subject: [PATCH 05/23] updated docs --- docs/aws/API-Gateway.md | 80 ++++++++++++++++++++++++++++ docs/{ => aws}/CDK-Asset-Hash.md | 0 docs/aws/CDK-CI-CD-Github-Actions.md | 52 ++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 docs/aws/API-Gateway.md rename docs/{ => aws}/CDK-Asset-Hash.md (100%) create mode 100644 docs/aws/CDK-CI-CD-Github-Actions.md diff --git a/docs/aws/API-Gateway.md b/docs/aws/API-Gateway.md new file mode 100644 index 0000000..4a327b9 --- /dev/null +++ b/docs/aws/API-Gateway.md @@ -0,0 +1,80 @@ +Optimizing tool selection...Looking at your infra.py, I can see you have a basic Lambda REST API setup. Here are the key production patterns and concepts you should learn: + +## API Gateway Deployment Patterns to Study + +**1. API Types & When to Use Them** +- REST API vs HTTP API (cost/feature tradeoffs) +- WebSocket APIs for real-time communication +- Regional vs Edge-Optimized vs Private endpoints + +**2. Authentication & Authorization** +- IAM authorization +- Lambda authorizers (custom auth logic) +- Cognito User Pools integration +- API Keys and Usage Plans +- OAuth 2.0 / JWT validation + +**3. Traffic Management** +- Throttling and rate limiting (per-client quotas) +- Usage plans for different customer tiers +- API Keys management +- Request/Response validation + +**4. Deployment Strategies** +- Stage variables for environment configuration +- Canary deployments (gradual traffic shifting) +- Multiple stages (dev/staging/prod) +- Stage-specific configuration + +**5. Observability & Monitoring** +- CloudWatch Logs (access logs vs execution logs) +- X-Ray tracing integration (you have this!) +- Custom CloudWatch metrics +- Alarms for error rates and latency + +**6. Cost Optimization** +- Caching strategies at API Gateway level +- Regional vs Edge endpoints +- HTTP API vs REST API cost comparison + +**7. Security Best Practices** +- WAF (Web Application Firewall) integration +- Resource policies to restrict access +- TLS/SSL certificate management +- CORS configuration +- Request validation to block malformed requests + +**8. Domain & DNS** +- Custom domain names +- Certificate management (ACM) +- Base path mappings +- API versioning strategies + +## Recommended Resources + +**AWS Documentation:** +- [API Gateway Best Practices](https://docs.aws.amazon.com/apigateway/latest/developerguide/best-practices.html) +- [Choosing Between HTTP and REST APIs](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-vs-rest.html) +- [API Gateway Security Best Practices](https://docs.aws.amazon.com/apigateway/latest/developerguide/security-best-practices.html) + +**Architecture Patterns:** +- [AWS Serverless Application Lens](https://docs.aws.amazon.com/wellarchitected/latest/serverless-applications-lens/) - comprehensive guide +- [API Gateway Deployment Best Practices whitepaper](https://aws.amazon.com/blogs/compute/best-practices-for-organizing-larger-serverless-applications/) + +**CDK Specific:** +- [AWS CDK API Gateway L2 Constructs](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_apigateway-readme.html) +- CDK Patterns for API Gateway: https://github.com/cdk-patterns/serverless + +**Hands-On Learning:** +- AWS Workshops: https://catalog.workshops.aws/ +- Serverless Land: https://serverlessland.com/patterns?framework=CDK + +**Key Topics for Your Current Setup:** +1. Add request/response validation models +2. Implement proper error responses with Gateway Responses +3. Add CloudWatch access logging +4. Consider usage plans if this is a public API +5. Implement custom domain with Route53 +6. Add Lambda authorizer if you need custom auth + +Start with the AWS Well-Architected Serverless Lens - it covers all five pillars (operational excellence, security, reliability, performance, cost) specifically for serverless applications. \ No newline at end of file diff --git a/docs/CDK-Asset-Hash.md b/docs/aws/CDK-Asset-Hash.md similarity index 100% rename from docs/CDK-Asset-Hash.md rename to docs/aws/CDK-Asset-Hash.md diff --git a/docs/aws/CDK-CI-CD-Github-Actions.md b/docs/aws/CDK-CI-CD-Github-Actions.md new file mode 100644 index 0000000..af72e63 --- /dev/null +++ b/docs/aws/CDK-CI-CD-Github-Actions.md @@ -0,0 +1,52 @@ +Here's a learning path for CI/CD with AWS CDK: + +## Core Concepts to Learn + +**1. AWS CDK Fundamentals** +- CDK Pipelines construct +- Stacks vs Stages vs Applications +- CDK synthesis and bootstrapping +- Cross-account/cross-region deployments + +**2. CI/CD Patterns** +- Pipeline stages: Source → Build → Test → Deploy +- Blue/Green and Canary deployments +- Self-mutating pipelines (pipelines that update themselves) +- Manual approval gates + +**3. AWS Services Integration** +- CodePipeline, CodeBuild, CodeCommit +- GitHub Actions with OIDC for AWS +- AWS Secrets Manager for credentials +- CloudWatch for monitoring pipeline metrics + +**4. Testing Strategies** +- CDK assertions and snapshot testing +- Integration tests in pipeline stages +- Security scanning (cdk-nag) +- Infrastructure validation pre-deployment + +## Recommended Resources + +**Official AWS Documentation:** +- [CDK Pipelines Documentation](https://docs.aws.amazon.com/cdk/v2/guide/cdk_pipeline.html) +- [CDK Workshop - CI/CD Module](https://cdkworkshop.com/) +- [AWS CDK Examples - Pipelines](https://github.com/aws-samples/aws-cdk-examples) + +**Video Courses:** +- AWS re:Invent sessions on "CDK Pipelines" (YouTube) +- A Cloud Guru / Pluralsight CDK courses + +**Hands-On:** +- [CDK Patterns - CI/CD Patterns](https://cdkpatterns.com/) +- AWS Well-Architected Labs for CI/CD + +**GitHub Actions specific:** +- [AWS Actions for GitHub](https://github.com/aws-actions) +- [Configure AWS Credentials Action](https://github.com/aws-actions/configure-aws-credentials) + +**Best Practices:** +- AWS Well-Architected Framework - Operational Excellence Pillar +- CDK Best Practices guide + +Start with the official CDK Pipelines documentation, then work through a hands-on tutorial to deploy your files-api project through a pipeline. \ No newline at end of file From b7a9f7d55f5c44c65bbb2ab20f794f3fb8280735 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 16:38:48 +0530 Subject: [PATCH 06/23] updated TODO --- README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index e08d62f..d56a89d 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,17 @@ This project is a more polished version of the [cloud-engineering-project](https - [x] Setup the Dockerfile with the recommended way of using [uv in Docker](https://docs.astral.sh/uv/guides/integration/docker/). - [x] CDK rebuilds the Lambda Layer Docker image on every deployment. Is it possible to cache it locally and only rebuild when there are changes to files like `pyproject.toml` or `uv.lock`? - [ ] Try Docker multi-stage builds and configure [watch](https://docs.astral.sh/uv/guides/integration/docker/#configuring-watch-with-docker-compose) with docker compose. -- [ ] Implement API versioning strategy (like v1 in the path). - [ ] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. -- [ ] Deployment Stratgies like Blue-Green, Canary deployments, etc. -- [ ] Implement authentication (API keys or AWS Cognito) and secure Swagger UI page and possiblly the API endpoints as well. - - [ ] Add rate limiting to the API using API Gateway - - [ ] Implement multi-environment deployment pipeline (dev/prod) with approval gates +- [ ] API Gateway Improvements: + - [ ] Implement API versioning strategy (like v1 in the path). + - [ ] API Gateway with multiple stages (dev/prod). [`aws-samples/aws-cdk-examples/api-stages-lambda`](https://github.com/aws-samples/aws-cdk-examples/tree/main/python/api-stages-lambda) + - [ ] API Throttling and Usage Plans to manage API consumption and prevent abuse. + - [ ] Deployment Stratgies like Blue-Green, Canary deployments, etc. + - [ ] Implement authentication (API keys with Usage Plans/AWS Cognito/custom JWT/IAM Auth) and secure Swagger UI page and possiblly the API endpoints as well. + - [ ] Add rate limiting to the API using API Gateway + - [ ] Implement multi-environment deployment pipeline (dev/prod) with approval gates - [ ] Observability & Monitoring improvements: + - [ ] Cloudwatch Dashboard using CDK to visualize custom metrics, logs and traces in one place. [*ref*](https://github.com/cdk-patterns/serverless/tree/main/the-cloudwatch-dashboard/python) - [ ] Use OpenTelemetry for tracing instead of AWS X-Ray, [ref](https://aws.amazon.com/blogs/mt/aws-x-ray-sdks-daemon-migration-to-opentelemetry/). - [ ] Setup Grafana dashboards with CloudWatch data sources for enhanced monitoring - [ ] Replace Cloudwatch with Grafana Stack -- logs, metrics and traces From 9694c0251bc5ece375a3e0b9e44290b97afe6830 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Fri, 23 Jan 2026 17:32:49 +0530 Subject: [PATCH 07/23] updated function docs --- infra.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/infra.py b/infra.py index b39b426..21933bb 100644 --- a/infra.py +++ b/infra.py @@ -212,11 +212,12 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: stage_name="prod", tracing_enabled=True, metrics_enabled=True, - # logging_level=apigw.MethodLoggingLevel.INFO, - # ^^^API Gateway Execution Logs - it is recommended to turn it off in production + # API Gateway Execution Logs: it is recommended to turn it off in production # Execution logs capture detailed information about API request processing lifecycle. It should be - # enabled only for debugging purposes as it can quite verbose and incur additional cloudwatch costs - # and may expose sensitive information. + # enabled only for debugging purposes as it can quite verbose and incur additional cloudwatch costs and may expose sensitive information. + # AWS auto creates a log group for execution logs with format: API-Gateway-Execution-Logs/{rest-api-id}/{stage-name} + logging_level=None, # apigw.MethodLoggingLevel.INFO, # Set to INFO or ERROR to enable execution logging + # Access Logs: Access logs capture traffic-related information about the requests coming into API Gateway. access_log_destination=apigw.LogGroupLogDestination(log_group=api_gw_access_log_group_prod), # access_log_format=apigw.AccessLogFormat.clf(), # Common Log Format for access logs # access_log_format=apigw.AccessLogFormat.json_with_standard_fields(...) # Pre-defined JSON format with standard fields @@ -278,7 +279,9 @@ def apigw_custom_access_log_format() -> apigw.AccessLogFormat: """ Custom API Gateway Access Log Format based on Alex DeBrie's blog post. - Ref: https://www.alexdebrie.com/posts/api-gateway-access-logs/#access-logging-fields + Ref: + - Alex DeBrie's blog post: https://www.alexdebrie.com/posts/api-gateway-access-logs/#access-logging-fields + - My article: https://ericriddoch.notion.site/Deep-Dive-Log-Correlation-Setting-up-Access-and-Execution-Logs-in-our-API-19a29335f6d880149ec2e1875e8b8761?pvs=143 """ # ref: Refer to Alex DeBrie's blog post for custom access log format: return apigw.AccessLogFormat.custom( From 1abc8bef34b63af467fda593065536f848099d30 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:40:07 +0530 Subject: [PATCH 08/23] feat: createad CDK Stack to create IAM role for OIDC auth in github workflows --- github_oidc_infra.py | 141 +++++++++++++++++++++++++++++++++++++++++++ run | 22 +++++++ 2 files changed, 163 insertions(+) create mode 100644 github_oidc_infra.py diff --git a/github_oidc_infra.py b/github_oidc_infra.py new file mode 100644 index 0000000..5552ee1 --- /dev/null +++ b/github_oidc_infra.py @@ -0,0 +1,141 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "aws-cdk-lib>=2.233.0", +# "constructs>=10.4.4", +# ] +# /// +"""Creating an IAM Role for GitHub OIDC to allow GitHub Actions to assume the role and deploy CDK stack.""" + +import os +import re + +import aws_cdk as cdk +from aws_cdk import Stack, aws_iam as iam +from constructs import Construct + +""" +Information on GitHub OIDC provider: + +- official docs by GitHub: https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services +- where to find the GitHub Actions thumbprints for the OIDC provider: https://github.blog/changelog/2023-06-27-github-actions-update-on-oidc-integration-with-aws/ +- video on setting this up: https://www.youtube.com/watch?v=USIVWqXVv_U +""" + + +class GitHubActionsOIDCRoleStack(Stack): + """Stack to create an IAM Role for GitHub OIDC.""" + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + # # define parameter or context variable for GitHub repository name and owner for the cdk app + # github_repo_parameter = cdk.CfnParameter( + # self, + # "GitHubRepo", + # type="String", + # description="GitHub repository name in the format 'owner/repo'. E.g., 'amitvraj/my-repo'", + # allowed_pattern=".+/.+", + # default=None, # User must provide value + # ) + + # Use context variable for GitHub repository name and owner + github_repo = self.node.try_get_context("github_repo") + if not github_repo: + raise ValueError("GitHub repository must be provided in the context in the format 'owner/repo'.") + + if github_repo: + # Match the pattern 'owner/repo' + if not re.match(r"^[^/]+/[^/]+$", github_repo): + raise ValueError("GitHub repository must be in the format 'owner/repo'.") + + # Create a OIDC Provider resource if not already existing + github_provider: iam.IOpenIdConnectProvider | iam.OidcProviderNative + + # Check if we should create a new OIDC provider or use an existing one + create_oidc_provider_str = self.node.try_get_context("create_oidc_provider") + + # Context values from CLI are strings, so we need to check the string value + if create_oidc_provider_str.lower() == "false": + oidc_provider_arn = f"arn:aws:iam::{self.account}:oidc-provider/token.actions.githubusercontent.com" + # Import existing OIDC provider + # github_provider = iam.OpenIdConnectProvider.from_open_id_connect_provider_arn( + # self, + # id="GitHubOIDCProvider", + # open_id_connect_provider_arn=oidc_provider_arn, + # ) + else: + # Create a new OIDC provider - This can be ONLY ONE per AWS account + github_provider = iam.OidcProviderNative( + self, + id="GitHubOIDCProvider", + url="https://token.actions.githubusercontent.com", + client_ids=["sts.amazonaws.com"], + thumbprints=[ + # https://github.blog/changelog/2023-06-27-github-actions-update-on-oidc-integration-with-aws/ + "6938fd4d98bab03faadb97b34396831e3780aea1", + "1c58a3a8518e8759bf075b76b750d4f2df264fcd", + ], + ) + oidc_provider_arn = github_provider.oidc_provider_arn + + # Create the IAM Role for GitHub Actions OIDC + github_oidc_role = iam.Role( + self, + "GitHubActionsOIDCRole", + assumed_by=iam.FederatedPrincipal( + # federated="arn:aws:iam::" + self.account + ":oidc-provider/token.actions.githubusercontent.com", + federated=oidc_provider_arn, + conditions={ + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + "token.actions.githubusercontent.com:sub": f"repo:{github_repo}:*", + } + }, + assume_role_action="sts:AssumeRoleWithWebIdentity", + ), + role_name="GitHubActionsOIDCRole", + description="IAM Role for GitHub Actions to assume via OIDC for deploying CDK stack.", + ) + + # Attach necessary policies to the role (e.g., AdministratorAccess, PowerUserAccess, or custom policies) + github_oidc_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("PowerUserAccess")) + + # Output the ARN of the IAM Role + cdk.CfnOutput( + self, + "GitHubActionsOIDCRoleARNOutput", + value=github_oidc_role.role_arn, + description="ARN of the IAM Role for GitHub Actions OIDC", + ) + + +############### +# --- App --- # +############### + +# CDK App +app = cdk.App() + +cdk.Tags.of(app).add("x-project", "files-api") + + +GitHubActionsOIDCRoleStack( + app, + construct_id="GitHubActionsOIDCRoleStack", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + env=cdk.Environment( + account=os.getenv("CDK_DEFAULT_ACCOUNT"), + region=os.getenv("CDK_DEFAULT_REGION"), + ), + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + # env=cdk.Environment(account='123456789012', region='us-east-1'), + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html +) + +app.synth() diff --git a/run b/run index d5305f7..90e4641 100755 --- a/run +++ b/run @@ -50,6 +50,28 @@ cdk-deploy () { cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION } +cdk-deploy:github_oidc_stack () { + set -x + local github_repo=${1} # avr2002/files-api + local create_oidc_provider=${2:-false} # boolean to indicate whether to create a new OIDC provider + + if [ -z "$github_repo" ]; then + echo "ERROR: GitHub repository not specified. Usage: cdk-deploy:github_oidc_stack . E.g., avr2002/files-api" + exit 1 + fi + + # Check if create_oidc_provider is a boolean + if [[ "$create_oidc_provider" != "true" && "$create_oidc_provider" != "false" ]]; then + echo "ERROR: create_oidc_provider must be 'true' or 'false'." + exit 1 + fi + + cdk deploy --app 'uv run --script github_oidc_infra.py' '*' \ + --context github_repo="$github_repo" \ + --context create_oidc_provider="$create_oidc_provider" \ + --profile $AWS_PROFILE --region $AWS_REGION +} + cdk-destroy () { cdk destroy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION } From e28713dc7993d561fc96643185899e2849baf7f9 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:40:24 +0530 Subject: [PATCH 09/23] docs: on cdk context values and parameters --- docs/aws/CDK-ContextValues-and-Parameters.md | 152 +++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 docs/aws/CDK-ContextValues-and-Parameters.md diff --git a/docs/aws/CDK-ContextValues-and-Parameters.md b/docs/aws/CDK-ContextValues-and-Parameters.md new file mode 100644 index 0000000..6f40c73 --- /dev/null +++ b/docs/aws/CDK-ContextValues-and-Parameters.md @@ -0,0 +1,152 @@ +## CDK Parameters vs Context Values + +### CDK Parameters (`CfnParameter`) + +**What they are:** +- CloudFormation parameters that prompt for values at **deployment time** +- Defined in your CDK code but values provided when running `cdk deploy` +- Part of the CloudFormation template itself + +**Characteristics:** +- ✅ Can change values between deployments without changing code +- ✅ Interactive prompts during deployment +- ✅ Good for values that differ per deployment +- ❌ Can't use in synthesis-time logic (if statements, loops) +- ❌ Limited to CloudFormation types (String, Number, List, etc.) +- ❌ Can't be used to conditionally create resources + +**Use when:** +- Value changes frequently between deployments +- Different operators need different values +- You want deployment-time flexibility +- Building reusable CloudFormation templates + +**Example:** Database password, instance sizes that operators choose + +--- + +### CDK Context Values + +**What they are:** +- Configuration values available at **synthesis time** (when running `cdk synth`) +- Can be provided via CLI, `cdk.json`, or `cdk.context.json` +- Become hardcoded in the synthesized CloudFormation template + +**Characteristics:** +- ✅ Available during synthesis - can use in if/else logic +- ✅ Can conditionally create/skip resources +- ✅ Can be any data type (strings, objects, arrays) +- ✅ Cached in `cdk.context.json` for consistency +- ❌ Require re-synthesis to change values +- ❌ Not visible in CloudFormation console + +**How to provide:** +```bash +# Via CLI +cdk deploy -c github_repo=owner/repo + +# Via cdk.json +{ + "context": { + "github_repo": "owner/repo" + } +} +``` + +**Use when:** +- Value is known at development time +- Need to use value in synthesis logic +- Building environment-specific stacks +- Value is configuration, not secret + +**Example:** Environment names, feature flags, account IDs, repo names + +--- + +## Key Differences + +| Aspect | Parameters | Context | +|--------|-----------|---------| +| **When resolved** | Deploy time | Synth time | +| **Can use in if/else** | ❌ No | ✅ Yes | +| **Interactive prompts** | ✅ Yes | ❌ No | +| **Visible in CFN console** | ✅ Yes | ❌ No | +| **Change without re-synth** | ✅ Yes | ❌ No | +| **Conditionally create resources** | ❌ No | ✅ Yes | + +--- + +## For Your GitHub Repo Name + +**Recommendation: Use Context** 🎯 + +**Why:** +1. **Known at development time** - repo name doesn't change deployment-to-deployment +2. **Better DX** - No interactive prompts, can be in version control +3. **Simpler** - Can be in `cdk.json` or environment variable +4. **Validation** - Can validate format during synthesis, not deployment + +**Current issue with your Parameter approach:** +```python +if not github_repo_parameter.value_as_string: + raise ValueError(...) # This won't work! +``` +Parameters are **tokens** at synthesis time, so you can't check if they're empty. The check always passes, then fails at CloudFormation deployment with a cryptic error. + +--- + +## Recommended Implementation Patterns + +**Pattern 1: Context via cdk.json** +```python +github_repo = self.node.try_get_context("github_repo") +if not github_repo: + raise ValueError("Context value 'github_repo' is required") +``` + +```json +// cdk.json +{ + "context": { + "github_repo": "avr2002/files-api" + } +} +``` + +**Pattern 2: Environment Variable** +```python +github_repo = os.environ.get("GITHUB_REPO") +if not github_repo: + raise ValueError("GITHUB_REPO environment variable required") +``` + +**Pattern 3: CLI Context** +```bash +cdk deploy -c github_repo=avr2002/files-api +``` + +--- + +## When Parameters ARE Better + +Use Parameters when you need: +- Operators to make runtime decisions (e.g., instance type selection) +- Different values for same template across accounts +- CloudFormation StackSets with different parameter values +- True infrastructure-as-a-template (not infrastructure-as-code) + +Example: A reusable template deployed by multiple teams with their own values. + +--- + +## Learning Resources + +**Official Docs:** +- [CDK Context](https://docs.aws.amazon.com/cdk/v2/guide/context.html) +- [CDK Parameters](https://docs.aws.amazon.com/cdk/v2/guide/parameters.html) +- [CDK Best Practices - Parameters vs Context](https://docs.aws.amazon.com/cdk/v2/guide/best-practices.html#best-practices-apps) + +**Key Quote from AWS:** +> "Generally, we recommend against using AWS CloudFormation parameters with the AWS CDK. [...] Use context values or environment variables instead." + +For your use case, switch to context - it's the CDK-native way. \ No newline at end of file From e3c979ef806aee1419b8c49bd212b72d6249420b Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:40:36 +0530 Subject: [PATCH 10/23] updated gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 4a8696a..e8cffe8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ cdk.context.json backup/ notebooks/code/ +**/*cache*/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 348fe891f9d2579ef7b6ac9dfec8a9ae244a8043 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:41:15 +0530 Subject: [PATCH 11/23] added cdk deps to infra script for uv --- infra.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/infra.py b/infra.py index 21933bb..22a51d5 100644 --- a/infra.py +++ b/infra.py @@ -1,3 +1,10 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "aws-cdk-lib>=2.233.0", +# "constructs>=10.4.4", +# ] +# /// import hashlib import json import os From fc1e0c4f40230e0e59aee27a8cfb3876f5db426e Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:50:17 +0530 Subject: [PATCH 12/23] feat: wrote github workflow to deploy cdk stack via CI --- .github/workflows/build-test-publish.yaml | 55 ++++++++++++++++++++--- run | 14 ++++++ 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index 14e97c7..a3792b0 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -49,6 +49,7 @@ jobs: git tag "v$VERSION" lint-format-and-static-code-checks: + name: Lint, Format, and Static Code Quality Checks runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -68,7 +69,8 @@ jobs: - name: Run oasdiff, Lint, Format, and other static code quality checks run: SKIP=no-commit-to-branch uvx --from pre-commit pre-commit run --all-files - build-wheel-and-sdist: + build-wheel: + name: Build Wheel Distribution and Upload as Artifact runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -87,8 +89,9 @@ jobs: path: ./dist/* execute-tests: + name: Execute Tests against Wheel needs: - - build-wheel-and-sdist + - build-wheel runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -111,12 +114,50 @@ jobs: uv pip install --group test ./run run-tests - publish: + deploy: + name: Deploy to AWS needs: - - execute-tests - - build-wheel-and-sdist - - lint-format-and-static-code-checks - - check-version + [ + execute-tests, + build-wheel, + lint-format-and-static-code-checks, + check-version, + ] + runs-on: ubuntu-latest + # IDEAL WAY to do this would be deploy to dev/staging on PR merge, then either + # Deploy to prod on merge to main or have a manual approval step. + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-west-2 + + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + + - name: Setup Node + uses: actions/setup-node@v6 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + # ^^^needed for building and deploying Lambda Layer + + - name: Install AWS CDK CLI + run: npm install -g aws-cdk + + - name: CDK Synthesize + run: bash +x ./run cdk-synth:ci + + - name: CDK Deploy + run: bash +x ./run cdk-deploy:ci + + publish: + name: Publish Version Tag + needs: [deploy] runs-on: ubuntu-latest # if - this is a merge to main or push directly to the main branch if: github.event_name == 'push' && github.ref == 'refs/heads/main' diff --git a/run b/run index 90e4641..9054943 100755 --- a/run +++ b/run @@ -77,6 +77,20 @@ cdk-destroy () { } +cdk-synth:ci () { + cdk synth --app 'uv run --script infra.py' --region $AWS_REGION +} + +cdk-deploy:ci () { + # If S3_BUCKET_NAME is not set, exit with error + if [ -z "${S3_BUCKET_NAME:-}" ]; then + echo "ERROR: S3_BUCKET_NAME is not set. Cannot deploy CDK stack." + exit 1 + fi + + cdk deploy --app 'uv run --script infra.py' '*' --region $AWS_REGION +} + ############################ # --- Run App --- # # ############################ From dcc193b72a48187ff7fadfbab227f9226a673e43 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:52:10 +0530 Subject: [PATCH 13/23] chore: version bump to 0.2.0 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64e4e70..835383a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "files-api" -version = "0.1.5" +version = "0.2.0" description = "Files API to upload and download files from S3 using FastAPI." readme = "README.md" authors = [ diff --git a/uv.lock b/uv.lock index ac36dc6..66ccfe8 100644 --- a/uv.lock +++ b/uv.lock @@ -930,7 +930,7 @@ wheels = [ [[package]] name = "files-api" -version = "0.1.5" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "aws-embedded-metrics" }, From 73621a6b5fab626361df6d869fba332743a61ef0 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 14:57:21 +0530 Subject: [PATCH 14/23] fix: add permissions for id-token and contents in deploy job --- .github/workflows/build-test-publish.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index a3792b0..a55591c 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -123,6 +123,9 @@ jobs: lint-format-and-static-code-checks, check-version, ] + permissions: + id-token: write + contents: read runs-on: ubuntu-latest # IDEAL WAY to do this would be deploy to dev/staging on PR merge, then either # Deploy to prod on merge to main or have a manual approval step. From 87945a35b1045df3de0f1329adf2f8df90d0a551 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:20:01 +0530 Subject: [PATCH 15/23] fix: update assume role conditions for GitHub OIDC integration --- github_oidc_infra.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/github_oidc_infra.py b/github_oidc_infra.py index 5552ee1..8b9de1a 100644 --- a/github_oidc_infra.py +++ b/github_oidc_infra.py @@ -89,8 +89,10 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: conditions={ "StringEquals": { "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + }, + "StringLike": { "token.actions.githubusercontent.com:sub": f"repo:{github_repo}:*", - } + }, }, assume_role_action="sts:AssumeRoleWithWebIdentity", ), From 8627808ca10d51e79534f4d51528bcb2cb82641e Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:20:34 +0530 Subject: [PATCH 16/23] refactor: moved aws configure step before installing cdk --- .github/workflows/build-test-publish.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index a55591c..dffdad9 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -130,12 +130,6 @@ jobs: # IDEAL WAY to do this would be deploy to dev/staging on PR merge, then either # Deploy to prod on merge to main or have a manual approval step. steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} - aws-region: us-west-2 - - name: Checkout Repository uses: actions/checkout@v4 @@ -149,6 +143,12 @@ jobs: uses: docker/setup-buildx-action@v3 # ^^^needed for building and deploying Lambda Layer + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-west-2 + - name: Install AWS CDK CLI run: npm install -g aws-cdk From d76ffd98e48010495e6e13fc347f577188c7f9e8 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:31:01 +0530 Subject: [PATCH 17/23] refactor: update CDK commands for GitHub Actions compatibility --- .github/workflows/build-test-publish.yaml | 4 +-- run | 41 ++++++++++++++--------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index dffdad9..d0db1cd 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -153,10 +153,10 @@ jobs: run: npm install -g aws-cdk - name: CDK Synthesize - run: bash +x ./run cdk-synth:ci + run: bash +x ./run cdk-synth - name: CDK Deploy - run: bash +x ./run cdk-deploy:ci + run: bash +x ./run cdk-deploy publish: name: Publish Version Tag diff --git a/run b/run index 9054943..e86c0cf 100755 --- a/run +++ b/run @@ -7,7 +7,14 @@ export AWS_REGION=us-west-2 export AWS_DEFAULT_REGION=${AWS_REGION} # aws account id/hash used to make a unique, but consistent bucket name -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text || echo "") +if [ -z "${GITHUB_ACTIONS:-}" ]; then + # not running in GitHub Actions + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) +else + # running in GitHub Actions + # retrieve AWS Account ID from environment variable set by GitHub Actions OIDC token + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text || echo "") +fi # Only create bucket name if AWS_ACCOUNT_ID is available # Suppress errors if profile doesn't exist (e.g., in CI/CD) @@ -34,6 +41,16 @@ cdk-bootstrap () { cdk bootstrap "aws://${AWS_ACCOUNT_ID}/${AWS_REGION}" } + +cdk-synth () { + if [ -z "${GITHUB_ACTIONS:-}" ]; then + cdk synth --app 'uv run --script infra.py' --profile $AWS_PROFILE --region $AWS_REGION + else: + cdk synth --app 'uv run --script infra.py' --region $AWS_REGION + fi +} + + cdk-deploy () { # Check if Docker is running if ! docker info >/dev/null 2>&1; then @@ -47,9 +64,14 @@ cdk-deploy () { exit 1 fi - cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION + if [ -z "${GITHUB_ACTIONS:-}" ]; then + cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION + else + cdk deploy --app 'uv run --script infra.py' '*' --region $AWS_REGION + fi } + cdk-deploy:github_oidc_stack () { set -x local github_repo=${1} # avr2002/files-api @@ -72,25 +94,12 @@ cdk-deploy:github_oidc_stack () { --profile $AWS_PROFILE --region $AWS_REGION } + cdk-destroy () { cdk destroy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION } -cdk-synth:ci () { - cdk synth --app 'uv run --script infra.py' --region $AWS_REGION -} - -cdk-deploy:ci () { - # If S3_BUCKET_NAME is not set, exit with error - if [ -z "${S3_BUCKET_NAME:-}" ]; then - echo "ERROR: S3_BUCKET_NAME is not set. Cannot deploy CDK stack." - exit 1 - fi - - cdk deploy --app 'uv run --script infra.py' '*' --region $AWS_REGION -} - ############################ # --- Run App --- # # ############################ From 1cd9c047a05093c9135de7ecfc9978f26a51858f Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:38:18 +0530 Subject: [PATCH 18/23] fix: using correct bash flag to check if GITHUB_ACTIONS var is set and not empty --- run | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/run b/run index e86c0cf..9f54284 100755 --- a/run +++ b/run @@ -7,13 +7,13 @@ export AWS_REGION=us-west-2 export AWS_DEFAULT_REGION=${AWS_REGION} # aws account id/hash used to make a unique, but consistent bucket name -if [ -z "${GITHUB_ACTIONS:-}" ]; then - # not running in GitHub Actions - AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) -else +if [[ -n "${GITHUB_ACTIONS:-}" ]]; then # running in GitHub Actions # retrieve AWS Account ID from environment variable set by GitHub Actions OIDC token AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text || echo "") +else + # not running in GitHub Actions + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) fi # Only create bucket name if AWS_ACCOUNT_ID is available @@ -43,10 +43,12 @@ cdk-bootstrap () { cdk-synth () { - if [ -z "${GITHUB_ACTIONS:-}" ]; then - cdk synth --app 'uv run --script infra.py' --profile $AWS_PROFILE --region $AWS_REGION - else: + # Check if varibale is set and not empty + if [ -n "${GITHUB_ACTIONS:-}" ]; then + # running in GitHub Actions cdk synth --app 'uv run --script infra.py' --region $AWS_REGION + else: + cdk synth --app 'uv run --script infra.py' --profile $AWS_PROFILE --region $AWS_REGION fi } @@ -64,10 +66,11 @@ cdk-deploy () { exit 1 fi - if [ -z "${GITHUB_ACTIONS:-}" ]; then - cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION - else + if [ -n "${GITHUB_ACTIONS:-}" ]; then + # running in GitHub Actions cdk deploy --app 'uv run --script infra.py' '*' --region $AWS_REGION + else + cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION fi } From 4be08f316d6f8daa6927600bbc4fb6cebb86fd1a Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:41:40 +0530 Subject: [PATCH 19/23] fix: fixed if-else syntax in cdk-synth function --- run | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run b/run index 9f54284..8a16996 100755 --- a/run +++ b/run @@ -43,11 +43,11 @@ cdk-bootstrap () { cdk-synth () { - # Check if varibale is set and not empty + # Check if variable is set and not empty if [ -n "${GITHUB_ACTIONS:-}" ]; then # running in GitHub Actions cdk synth --app 'uv run --script infra.py' --region $AWS_REGION - else: + else cdk synth --app 'uv run --script infra.py' --profile $AWS_PROFILE --region $AWS_REGION fi } From a6f66756492d8a947f5484681c5747f76f1f0598 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:44:41 +0530 Subject: [PATCH 20/23] fix: conditionally set AWS_PROFILE only when not in GitHub Actions --- run | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/run b/run index 8a16996..366257d 100755 --- a/run +++ b/run @@ -2,7 +2,11 @@ set -euo pipefail -export AWS_PROFILE=sandbox +# Only set AWS_PROFILE when not running in GitHub Actions +if [[ -z "${GITHUB_ACTIONS:-}" ]]; then + export AWS_PROFILE=sandbox +fi + export AWS_REGION=us-west-2 export AWS_DEFAULT_REGION=${AWS_REGION} From e2dec58da2e55fe7aba7c8c6d3a740a5ec7e9e53 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:49:25 +0530 Subject: [PATCH 21/23] fix: conditionally unset AWS_PROFILE in run-tests function --- run | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run b/run index 366257d..a706fa3 100755 --- a/run +++ b/run @@ -311,8 +311,10 @@ test:ci() { run-tests() { PYTEST_EXIT_STATUS=0 - # Unset AWS credentials - unset AWS_PROFILE + # Unset AWS credentials (only if set) + if [[ -n "${AWS_PROFILE:-}" ]]; then + unset AWS_PROFILE + fi # Disable AWS X-Ray and AWS EMF export AWS_XRAY_SDK_ENABLED="false" From be334106fef1ff312f933cef4c0b1f395ea58524 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 15:57:23 +0530 Subject: [PATCH 22/23] fix: ci was failing because of unbound variable error in execute-tests job (with set -u flag in script) --- run | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run b/run index a706fa3..c444312 100755 --- a/run +++ b/run @@ -26,7 +26,7 @@ if [ -n "$AWS_ACCOUNT_ID" ]; then AWS_ACCOUNT_ID_HASH=$(echo -n "${AWS_ACCOUNT_ID}" | sha256sum | cut -c5-8) export S3_BUCKET_NAME="python-aws-cloud-course-bucket-${AWS_ACCOUNT_ID_HASH}" # python-aws-cloud-course-bucket-6721 else - echo "WARNING: Could not retrieve AWS Account ID. Profile '$AWS_PROFILE' may not exist." + echo "WARNING: Could not retrieve AWS Account ID. Profile '${AWS_PROFILE:-}' may not exist." echo "S3_BUCKET_NAME environment variable will not be set." fi @@ -309,6 +309,7 @@ test:ci() { # (example) ./run.sh test tests/test_states_info.py::test__slow_add run-tests() { + set -x PYTEST_EXIT_STATUS=0 # Unset AWS credentials (only if set) From d56901a559a9dda3806f34c83c390aef6331dfb5 Mon Sep 17 00:00:00 2001 From: avr2002 Date: Sat, 24 Jan 2026 16:02:42 +0530 Subject: [PATCH 23/23] fix: updated aws-deploy job condition to run on merge to main --- .github/workflows/build-test-publish.yaml | 3 ++- README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index d0db1cd..6767631 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -70,7 +70,7 @@ jobs: run: SKIP=no-commit-to-branch uvx --from pre-commit pre-commit run --all-files build-wheel: - name: Build Wheel Distribution and Upload as Artifact + name: Build Wheel Dist & Upload as Artifact runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -129,6 +129,7 @@ jobs: runs-on: ubuntu-latest # IDEAL WAY to do this would be deploy to dev/staging on PR merge, then either # Deploy to prod on merge to main or have a manual approval step. + if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'true' steps: - name: Checkout Repository uses: actions/checkout@v4 diff --git a/README.md b/README.md index d56a89d..c5014ff 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ This project is a more polished version of the [cloud-engineering-project](https - [x] Setup the Dockerfile with the recommended way of using [uv in Docker](https://docs.astral.sh/uv/guides/integration/docker/). - [x] CDK rebuilds the Lambda Layer Docker image on every deployment. Is it possible to cache it locally and only rebuild when there are changes to files like `pyproject.toml` or `uv.lock`? - [ ] Try Docker multi-stage builds and configure [watch](https://docs.astral.sh/uv/guides/integration/docker/#configuring-watch-with-docker-compose) with docker compose. -- [ ] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. +- [x] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. - [ ] API Gateway Improvements: - [ ] Implement API versioning strategy (like v1 in the path). - [ ] API Gateway with multiple stages (dev/prod). [`aws-samples/aws-cdk-examples/api-stages-lambda`](https://github.com/aws-samples/aws-cdk-examples/tree/main/python/api-stages-lambda)