diff --git a/.github/workflows/build-test-publish.yaml b/.github/workflows/build-test-publish.yaml index 14e97c7..6767631 100644 --- a/.github/workflows/build-test-publish.yaml +++ b/.github/workflows/build-test-publish.yaml @@ -49,6 +49,7 @@ jobs: git tag "v$VERSION" lint-format-and-static-code-checks: + name: Lint, Format, and Static Code Quality Checks runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -68,7 +69,8 @@ jobs: - name: Run oasdiff, Lint, Format, and other static code quality checks run: SKIP=no-commit-to-branch uvx --from pre-commit pre-commit run --all-files - build-wheel-and-sdist: + build-wheel: + name: Build Wheel Dist & Upload as Artifact runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -87,8 +89,9 @@ jobs: path: ./dist/* execute-tests: + name: Execute Tests against Wheel needs: - - build-wheel-and-sdist + - build-wheel runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -111,12 +114,54 @@ jobs: uv pip install --group test ./run run-tests - publish: + deploy: + name: Deploy to AWS needs: - - execute-tests - - build-wheel-and-sdist - - lint-format-and-static-code-checks - - check-version + [ + execute-tests, + build-wheel, + lint-format-and-static-code-checks, + check-version, + ] + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + # IDEAL WAY to do this would be deploy to dev/staging on PR merge, then either + # Deploy to prod on merge to main or have a manual approval step. + if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' && github.event.inputs.deploy == 'true' + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + + - name: Setup Node + uses: actions/setup-node@v6 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + # ^^^needed for building and deploying Lambda Layer + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_OIDC_ROLE_TO_ASSUME }} + aws-region: us-west-2 + + - name: Install AWS CDK CLI + run: npm install -g aws-cdk + + - name: CDK Synthesize + run: bash +x ./run cdk-synth + + - name: CDK Deploy + run: bash +x ./run cdk-deploy + + publish: + name: Publish Version Tag + needs: [deploy] runs-on: ubuntu-latest # if - this is a merge to main or push directly to the main branch if: github.event_name == 'push' && github.ref == 'refs/heads/main' diff --git a/.gitignore b/.gitignore index 4a8696a..e8cffe8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ cdk.context.json backup/ notebooks/code/ +**/*cache*/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index e08d62f..c5014ff 100644 --- a/README.md +++ b/README.md @@ -43,13 +43,17 @@ This project is a more polished version of the [cloud-engineering-project](https - [x] Setup the Dockerfile with the recommended way of using [uv in Docker](https://docs.astral.sh/uv/guides/integration/docker/). - [x] CDK rebuilds the Lambda Layer Docker image on every deployment. Is it possible to cache it locally and only rebuild when there are changes to files like `pyproject.toml` or `uv.lock`? - [ ] Try Docker multi-stage builds and configure [watch](https://docs.astral.sh/uv/guides/integration/docker/#configuring-watch-with-docker-compose) with docker compose. -- [ ] Implement API versioning strategy (like v1 in the path). -- [ ] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. -- [ ] Deployment Stratgies like Blue-Green, Canary deployments, etc. -- [ ] Implement authentication (API keys or AWS Cognito) and secure Swagger UI page and possiblly the API endpoints as well. - - [ ] Add rate limiting to the API using API Gateway - - [ ] Implement multi-environment deployment pipeline (dev/prod) with approval gates +- [x] Setup CI/CD pipeline to deploy the API to AWS using GitHub Actions. +- [ ] API Gateway Improvements: + - [ ] Implement API versioning strategy (like v1 in the path). + - [ ] API Gateway with multiple stages (dev/prod). [`aws-samples/aws-cdk-examples/api-stages-lambda`](https://github.com/aws-samples/aws-cdk-examples/tree/main/python/api-stages-lambda) + - [ ] API Throttling and Usage Plans to manage API consumption and prevent abuse. + - [ ] Deployment Stratgies like Blue-Green, Canary deployments, etc. + - [ ] Implement authentication (API keys with Usage Plans/AWS Cognito/custom JWT/IAM Auth) and secure Swagger UI page and possiblly the API endpoints as well. + - [ ] Add rate limiting to the API using API Gateway + - [ ] Implement multi-environment deployment pipeline (dev/prod) with approval gates - [ ] Observability & Monitoring improvements: + - [ ] Cloudwatch Dashboard using CDK to visualize custom metrics, logs and traces in one place. [*ref*](https://github.com/cdk-patterns/serverless/tree/main/the-cloudwatch-dashboard/python) - [ ] Use OpenTelemetry for tracing instead of AWS X-Ray, [ref](https://aws.amazon.com/blogs/mt/aws-x-ray-sdks-daemon-migration-to-opentelemetry/). - [ ] Setup Grafana dashboards with CloudWatch data sources for enhanced monitoring - [ ] Replace Cloudwatch with Grafana Stack -- logs, metrics and traces diff --git a/docs/aws/API-Gateway.md b/docs/aws/API-Gateway.md new file mode 100644 index 0000000..4a327b9 --- /dev/null +++ b/docs/aws/API-Gateway.md @@ -0,0 +1,80 @@ +Optimizing tool selection...Looking at your infra.py, I can see you have a basic Lambda REST API setup. Here are the key production patterns and concepts you should learn: + +## API Gateway Deployment Patterns to Study + +**1. API Types & When to Use Them** +- REST API vs HTTP API (cost/feature tradeoffs) +- WebSocket APIs for real-time communication +- Regional vs Edge-Optimized vs Private endpoints + +**2. Authentication & Authorization** +- IAM authorization +- Lambda authorizers (custom auth logic) +- Cognito User Pools integration +- API Keys and Usage Plans +- OAuth 2.0 / JWT validation + +**3. Traffic Management** +- Throttling and rate limiting (per-client quotas) +- Usage plans for different customer tiers +- API Keys management +- Request/Response validation + +**4. Deployment Strategies** +- Stage variables for environment configuration +- Canary deployments (gradual traffic shifting) +- Multiple stages (dev/staging/prod) +- Stage-specific configuration + +**5. Observability & Monitoring** +- CloudWatch Logs (access logs vs execution logs) +- X-Ray tracing integration (you have this!) +- Custom CloudWatch metrics +- Alarms for error rates and latency + +**6. Cost Optimization** +- Caching strategies at API Gateway level +- Regional vs Edge endpoints +- HTTP API vs REST API cost comparison + +**7. Security Best Practices** +- WAF (Web Application Firewall) integration +- Resource policies to restrict access +- TLS/SSL certificate management +- CORS configuration +- Request validation to block malformed requests + +**8. Domain & DNS** +- Custom domain names +- Certificate management (ACM) +- Base path mappings +- API versioning strategies + +## Recommended Resources + +**AWS Documentation:** +- [API Gateway Best Practices](https://docs.aws.amazon.com/apigateway/latest/developerguide/best-practices.html) +- [Choosing Between HTTP and REST APIs](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-vs-rest.html) +- [API Gateway Security Best Practices](https://docs.aws.amazon.com/apigateway/latest/developerguide/security-best-practices.html) + +**Architecture Patterns:** +- [AWS Serverless Application Lens](https://docs.aws.amazon.com/wellarchitected/latest/serverless-applications-lens/) - comprehensive guide +- [API Gateway Deployment Best Practices whitepaper](https://aws.amazon.com/blogs/compute/best-practices-for-organizing-larger-serverless-applications/) + +**CDK Specific:** +- [AWS CDK API Gateway L2 Constructs](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_apigateway-readme.html) +- CDK Patterns for API Gateway: https://github.com/cdk-patterns/serverless + +**Hands-On Learning:** +- AWS Workshops: https://catalog.workshops.aws/ +- Serverless Land: https://serverlessland.com/patterns?framework=CDK + +**Key Topics for Your Current Setup:** +1. Add request/response validation models +2. Implement proper error responses with Gateway Responses +3. Add CloudWatch access logging +4. Consider usage plans if this is a public API +5. Implement custom domain with Route53 +6. Add Lambda authorizer if you need custom auth + +Start with the AWS Well-Architected Serverless Lens - it covers all five pillars (operational excellence, security, reliability, performance, cost) specifically for serverless applications. \ No newline at end of file diff --git a/docs/CDK-Asset-Hash.md b/docs/aws/CDK-Asset-Hash.md similarity index 100% rename from docs/CDK-Asset-Hash.md rename to docs/aws/CDK-Asset-Hash.md diff --git a/docs/aws/CDK-CI-CD-Github-Actions.md b/docs/aws/CDK-CI-CD-Github-Actions.md new file mode 100644 index 0000000..af72e63 --- /dev/null +++ b/docs/aws/CDK-CI-CD-Github-Actions.md @@ -0,0 +1,52 @@ +Here's a learning path for CI/CD with AWS CDK: + +## Core Concepts to Learn + +**1. AWS CDK Fundamentals** +- CDK Pipelines construct +- Stacks vs Stages vs Applications +- CDK synthesis and bootstrapping +- Cross-account/cross-region deployments + +**2. CI/CD Patterns** +- Pipeline stages: Source → Build → Test → Deploy +- Blue/Green and Canary deployments +- Self-mutating pipelines (pipelines that update themselves) +- Manual approval gates + +**3. AWS Services Integration** +- CodePipeline, CodeBuild, CodeCommit +- GitHub Actions with OIDC for AWS +- AWS Secrets Manager for credentials +- CloudWatch for monitoring pipeline metrics + +**4. Testing Strategies** +- CDK assertions and snapshot testing +- Integration tests in pipeline stages +- Security scanning (cdk-nag) +- Infrastructure validation pre-deployment + +## Recommended Resources + +**Official AWS Documentation:** +- [CDK Pipelines Documentation](https://docs.aws.amazon.com/cdk/v2/guide/cdk_pipeline.html) +- [CDK Workshop - CI/CD Module](https://cdkworkshop.com/) +- [AWS CDK Examples - Pipelines](https://github.com/aws-samples/aws-cdk-examples) + +**Video Courses:** +- AWS re:Invent sessions on "CDK Pipelines" (YouTube) +- A Cloud Guru / Pluralsight CDK courses + +**Hands-On:** +- [CDK Patterns - CI/CD Patterns](https://cdkpatterns.com/) +- AWS Well-Architected Labs for CI/CD + +**GitHub Actions specific:** +- [AWS Actions for GitHub](https://github.com/aws-actions) +- [Configure AWS Credentials Action](https://github.com/aws-actions/configure-aws-credentials) + +**Best Practices:** +- AWS Well-Architected Framework - Operational Excellence Pillar +- CDK Best Practices guide + +Start with the official CDK Pipelines documentation, then work through a hands-on tutorial to deploy your files-api project through a pipeline. \ No newline at end of file diff --git a/docs/aws/CDK-ContextValues-and-Parameters.md b/docs/aws/CDK-ContextValues-and-Parameters.md new file mode 100644 index 0000000..6f40c73 --- /dev/null +++ b/docs/aws/CDK-ContextValues-and-Parameters.md @@ -0,0 +1,152 @@ +## CDK Parameters vs Context Values + +### CDK Parameters (`CfnParameter`) + +**What they are:** +- CloudFormation parameters that prompt for values at **deployment time** +- Defined in your CDK code but values provided when running `cdk deploy` +- Part of the CloudFormation template itself + +**Characteristics:** +- ✅ Can change values between deployments without changing code +- ✅ Interactive prompts during deployment +- ✅ Good for values that differ per deployment +- ❌ Can't use in synthesis-time logic (if statements, loops) +- ❌ Limited to CloudFormation types (String, Number, List, etc.) +- ❌ Can't be used to conditionally create resources + +**Use when:** +- Value changes frequently between deployments +- Different operators need different values +- You want deployment-time flexibility +- Building reusable CloudFormation templates + +**Example:** Database password, instance sizes that operators choose + +--- + +### CDK Context Values + +**What they are:** +- Configuration values available at **synthesis time** (when running `cdk synth`) +- Can be provided via CLI, `cdk.json`, or `cdk.context.json` +- Become hardcoded in the synthesized CloudFormation template + +**Characteristics:** +- ✅ Available during synthesis - can use in if/else logic +- ✅ Can conditionally create/skip resources +- ✅ Can be any data type (strings, objects, arrays) +- ✅ Cached in `cdk.context.json` for consistency +- ❌ Require re-synthesis to change values +- ❌ Not visible in CloudFormation console + +**How to provide:** +```bash +# Via CLI +cdk deploy -c github_repo=owner/repo + +# Via cdk.json +{ + "context": { + "github_repo": "owner/repo" + } +} +``` + +**Use when:** +- Value is known at development time +- Need to use value in synthesis logic +- Building environment-specific stacks +- Value is configuration, not secret + +**Example:** Environment names, feature flags, account IDs, repo names + +--- + +## Key Differences + +| Aspect | Parameters | Context | +|--------|-----------|---------| +| **When resolved** | Deploy time | Synth time | +| **Can use in if/else** | ❌ No | ✅ Yes | +| **Interactive prompts** | ✅ Yes | ❌ No | +| **Visible in CFN console** | ✅ Yes | ❌ No | +| **Change without re-synth** | ✅ Yes | ❌ No | +| **Conditionally create resources** | ❌ No | ✅ Yes | + +--- + +## For Your GitHub Repo Name + +**Recommendation: Use Context** 🎯 + +**Why:** +1. **Known at development time** - repo name doesn't change deployment-to-deployment +2. **Better DX** - No interactive prompts, can be in version control +3. **Simpler** - Can be in `cdk.json` or environment variable +4. **Validation** - Can validate format during synthesis, not deployment + +**Current issue with your Parameter approach:** +```python +if not github_repo_parameter.value_as_string: + raise ValueError(...) # This won't work! +``` +Parameters are **tokens** at synthesis time, so you can't check if they're empty. The check always passes, then fails at CloudFormation deployment with a cryptic error. + +--- + +## Recommended Implementation Patterns + +**Pattern 1: Context via cdk.json** +```python +github_repo = self.node.try_get_context("github_repo") +if not github_repo: + raise ValueError("Context value 'github_repo' is required") +``` + +```json +// cdk.json +{ + "context": { + "github_repo": "avr2002/files-api" + } +} +``` + +**Pattern 2: Environment Variable** +```python +github_repo = os.environ.get("GITHUB_REPO") +if not github_repo: + raise ValueError("GITHUB_REPO environment variable required") +``` + +**Pattern 3: CLI Context** +```bash +cdk deploy -c github_repo=avr2002/files-api +``` + +--- + +## When Parameters ARE Better + +Use Parameters when you need: +- Operators to make runtime decisions (e.g., instance type selection) +- Different values for same template across accounts +- CloudFormation StackSets with different parameter values +- True infrastructure-as-a-template (not infrastructure-as-code) + +Example: A reusable template deployed by multiple teams with their own values. + +--- + +## Learning Resources + +**Official Docs:** +- [CDK Context](https://docs.aws.amazon.com/cdk/v2/guide/context.html) +- [CDK Parameters](https://docs.aws.amazon.com/cdk/v2/guide/parameters.html) +- [CDK Best Practices - Parameters vs Context](https://docs.aws.amazon.com/cdk/v2/guide/best-practices.html#best-practices-apps) + +**Key Quote from AWS:** +> "Generally, we recommend against using AWS CloudFormation parameters with the AWS CDK. [...] Use context values or environment variables instead." + +For your use case, switch to context - it's the CDK-native way. \ No newline at end of file diff --git a/github_oidc_infra.py b/github_oidc_infra.py new file mode 100644 index 0000000..8b9de1a --- /dev/null +++ b/github_oidc_infra.py @@ -0,0 +1,143 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "aws-cdk-lib>=2.233.0", +# "constructs>=10.4.4", +# ] +# /// +"""Creating an IAM Role for GitHub OIDC to allow GitHub Actions to assume the role and deploy CDK stack.""" + +import os +import re + +import aws_cdk as cdk +from aws_cdk import Stack, aws_iam as iam +from constructs import Construct + +""" +Information on GitHub OIDC provider: + +- official docs by GitHub: https://docs.github.com/en/actions/security-for-github-actions/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services +- where to find the GitHub Actions thumbprints for the OIDC provider: https://github.blog/changelog/2023-06-27-github-actions-update-on-oidc-integration-with-aws/ +- video on setting this up: https://www.youtube.com/watch?v=USIVWqXVv_U +""" + + +class GitHubActionsOIDCRoleStack(Stack): + """Stack to create an IAM Role for GitHub OIDC.""" + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + # # define parameter or context variable for GitHub repository name and owner for the cdk app + # github_repo_parameter = cdk.CfnParameter( + # self, + # "GitHubRepo", + # type="String", + # description="GitHub repository name in the format 'owner/repo'. E.g., 'amitvraj/my-repo'", + # allowed_pattern=".+/.+", + # default=None, # User must provide value + # ) + + # Use context variable for GitHub repository name and owner + github_repo = self.node.try_get_context("github_repo") + if not github_repo: + raise ValueError("GitHub repository must be provided in the context in the format 'owner/repo'.") + + if github_repo: + # Match the pattern 'owner/repo' + if not re.match(r"^[^/]+/[^/]+$", github_repo): + raise ValueError("GitHub repository must be in the format 'owner/repo'.") + + # Create a OIDC Provider resource if not already existing + github_provider: iam.IOpenIdConnectProvider | iam.OidcProviderNative + + # Check if we should create a new OIDC provider or use an existing one + create_oidc_provider_str = self.node.try_get_context("create_oidc_provider") + + # Context values from CLI are strings, so we need to check the string value + if create_oidc_provider_str.lower() == "false": + oidc_provider_arn = f"arn:aws:iam::{self.account}:oidc-provider/token.actions.githubusercontent.com" + # Import existing OIDC provider + # github_provider = iam.OpenIdConnectProvider.from_open_id_connect_provider_arn( + # self, + # id="GitHubOIDCProvider", + # open_id_connect_provider_arn=oidc_provider_arn, + # ) + else: + # Create a new OIDC provider - This can be ONLY ONE per AWS account + github_provider = iam.OidcProviderNative( + self, + id="GitHubOIDCProvider", + url="https://token.actions.githubusercontent.com", + client_ids=["sts.amazonaws.com"], + thumbprints=[ + # https://github.blog/changelog/2023-06-27-github-actions-update-on-oidc-integration-with-aws/ + "6938fd4d98bab03faadb97b34396831e3780aea1", + "1c58a3a8518e8759bf075b76b750d4f2df264fcd", + ], + ) + oidc_provider_arn = github_provider.oidc_provider_arn + + # Create the IAM Role for GitHub Actions OIDC + github_oidc_role = iam.Role( + self, + "GitHubActionsOIDCRole", + assumed_by=iam.FederatedPrincipal( + # federated="arn:aws:iam::" + self.account + ":oidc-provider/token.actions.githubusercontent.com", + federated=oidc_provider_arn, + conditions={ + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com", + }, + "StringLike": { + "token.actions.githubusercontent.com:sub": f"repo:{github_repo}:*", + }, + }, + assume_role_action="sts:AssumeRoleWithWebIdentity", + ), + role_name="GitHubActionsOIDCRole", + description="IAM Role for GitHub Actions to assume via OIDC for deploying CDK stack.", + ) + + # Attach necessary policies to the role (e.g., AdministratorAccess, PowerUserAccess, or custom policies) + github_oidc_role.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name("PowerUserAccess")) + + # Output the ARN of the IAM Role + cdk.CfnOutput( + self, + "GitHubActionsOIDCRoleARNOutput", + value=github_oidc_role.role_arn, + description="ARN of the IAM Role for GitHub Actions OIDC", + ) + + +############### +# --- App --- # +############### + +# CDK App +app = cdk.App() + +cdk.Tags.of(app).add("x-project", "files-api") + + +GitHubActionsOIDCRoleStack( + app, + construct_id="GitHubActionsOIDCRoleStack", + # If you don't specify 'env', this stack will be environment-agnostic. + # Account/Region-dependent features and context lookups will not work, + # but a single synthesized template can be deployed anywhere. + # Uncomment the next line to specialize this stack for the AWS Account + # and Region that are implied by the current CLI configuration. + env=cdk.Environment( + account=os.getenv("CDK_DEFAULT_ACCOUNT"), + region=os.getenv("CDK_DEFAULT_REGION"), + ), + # Uncomment the next line if you know exactly what Account and Region you + # want to deploy the stack to. */ + # env=cdk.Environment(account='123456789012', region='us-east-1'), + # For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html +) + +app.synth() diff --git a/infra.py b/infra.py index 79222c2..22a51d5 100644 --- a/infra.py +++ b/infra.py @@ -1,4 +1,12 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "aws-cdk-lib>=2.233.0", +# "constructs>=10.4.4", +# ] +# /// import hashlib +import json import os from pathlib import Path @@ -7,6 +15,7 @@ Stack, aws_apigateway as apigw, aws_lambda as _lambda, + aws_logs as logs, aws_s3 as s3, aws_ssm as ssm, ) @@ -120,6 +129,17 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # ^^^I found the layer ARN here from the AWS docs: # https://docs.aws.amazon.com/systems-manager/latest/userguide/ps-integration-lambda-extensions.html#ps-integration-lambda-extensions-add + # Log group for Lambda function + # Lambda by default creates a log group of format /aws/lambda/ + # But here we are explicitly creating it to set retention and removal policy + files_api_lambda_log_group = logs.LogGroup( + self, + id="FilesApiLambdaLogGroup", + log_group_name="/aws/lambda/files-api", + retention=logs.RetentionDays.ONE_MONTH, + removal_policy=cdk.RemovalPolicy.DESTROY, + ) + files_api_lambda = _lambda.Function( self, id="FilesApiLambda", @@ -136,6 +156,8 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: ), # Add Lambda Layers for dependencies and AWS Secrets Manager extension layers=[files_api_lambda_layer, secrets_manager_lambda_extension_layer], + # Specify the log group for the Lambda function + log_group=files_api_lambda_log_group, # Enable X-Ray Tracing for the Lambda function tracing=_lambda.Tracing.ACTIVE, environment={ @@ -166,11 +188,23 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # Grant the Lambda function permissions to read the OpenAI API Key secret from SSM Parameter Store ssm_openai_api_secret_key.grant_read(files_api_lambda) + # Grant the Lambda function permissions to write logs to CloudWatch Logs + files_api_lambda_log_group.grant_write(files_api_lambda) + # Setup API Gateway with resources and methods - # The LambdaRestApi construct by default creates a `test-invoke-stage` stage for the API + # Log group for API Gateway access logs + api_gw_access_log_group_prod = logs.LogGroup( + self, + id="FilesApiGwAccessLogGroup", + log_group_name="/aws/apigateway/access-logs/files-api/prod", + retention=logs.RetentionDays.ONE_MONTH, + removal_policy=cdk.RemovalPolicy.DESTROY, + ) + + # The LambdaRestApi L2 construct by default creates a `test-invoke-stage` stage for the API - # I disabled the proxy integration(proxy=False) because on the root path it defined "ANY" method by default which we do not want. + # I disabled the proxy integration(proxy=False) because on the root path it defined "ANY" method by default which we do not want. # For the root path we only want to allow "GET" method to access the OpenAPI docs page. files_api_gw = apigw.LambdaRestApi( self, @@ -184,7 +218,21 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: deploy_options=apigw.StageOptions( stage_name="prod", tracing_enabled=True, + metrics_enabled=True, + # API Gateway Execution Logs: it is recommended to turn it off in production + # Execution logs capture detailed information about API request processing lifecycle. It should be + # enabled only for debugging purposes as it can quite verbose and incur additional cloudwatch costs and may expose sensitive information. + # AWS auto creates a log group for execution logs with format: API-Gateway-Execution-Logs/{rest-api-id}/{stage-name} + logging_level=None, # apigw.MethodLoggingLevel.INFO, # Set to INFO or ERROR to enable execution logging + # Access Logs: Access logs capture traffic-related information about the requests coming into API Gateway. + access_log_destination=apigw.LogGroupLogDestination(log_group=api_gw_access_log_group_prod), + # access_log_format=apigw.AccessLogFormat.clf(), # Common Log Format for access logs + # access_log_format=apigw.AccessLogFormat.json_with_standard_fields(...) # Pre-defined JSON format with standard fields + access_log_format=apigw_custom_access_log_format(), # Custom JSON format for access logs ), + # Setting cloudWatchRole to true ensures CDK creates the necessary IAM role for logging + cloud_watch_role=True, + cloud_watch_role_removal_policy=cdk.RemovalPolicy.DESTROY, # endpoint_configuration=apigw.EndpointConfiguration( # types=[apigw.EndpointType.REGIONAL], # Use REGIONAL endpoint type for cost-effectiveness # ), @@ -220,20 +268,61 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: # self, # id="FilesApiGatewayUrl", # value=files_api_gw.url, - # description="Files-API API Gateway URL", + # description="Files-API API Gateway Invoke URL", # ) # Print an output saying manually create the SSM SecureString parameter cdk.CfnOutput( self, id="ManualSSMParameterCreationNotice", - value="Please remember to manually create the SSM Parameter Store SecureString parameter" - " '/files-api/openai-api-key' with your OpenAI API Key before deploying the stack." - f" You can create it here: https://{self.region}.console.aws.amazon.com/systems-manager/parameters/", + value="Please remember to manually create a SecureString parameter in AWS SSM Parameter Store with name" + " '/files-api/openai-api-key' with your OpenAI API Key after the first deployment of this stack.\n" + f"You can create it here: https://{self.region}.console.aws.amazon.com/systems-manager/parameters/", description="Manual SSM Parameter Creation Notice", ) +def apigw_custom_access_log_format() -> apigw.AccessLogFormat: + """ + Custom API Gateway Access Log Format based on Alex DeBrie's blog post. + + Ref: + - Alex DeBrie's blog post: https://www.alexdebrie.com/posts/api-gateway-access-logs/#access-logging-fields + - My article: https://ericriddoch.notion.site/Deep-Dive-Log-Correlation-Setting-up-Access-and-Execution-Logs-in-our-API-19a29335f6d880149ec2e1875e8b8761?pvs=143 + """ + # ref: Refer to Alex DeBrie's blog post for custom access log format: + return apigw.AccessLogFormat.custom( + json.dumps( + { # Request Information + "requestTime": apigw.AccessLogField.context_request_time(), + "requestId": apigw.AccessLogField.context_request_id(), + # There is slight difference in requestId & extendedRequestId: Clients can override the requestID + # but not the extendedRequestId, which may be helpful for troubleshooting & debugging purposes + "extendedRequestId": apigw.AccessLogField.context_extended_request_id(), + "httpMethod": apigw.AccessLogField.context_http_method(), + "path": apigw.AccessLogField.context_path(), + "resourcePath": apigw.AccessLogField.context_resource_path(), + "status": apigw.AccessLogField.context_status(), + "responseLatency": apigw.AccessLogField.context_response_latency(), # in milliseconds + "xrayTraceId": apigw.AccessLogField.context_xray_trace_id(), + # Integration Information + # AWS Endpoint Request ID: The requestID generated by Lambda function invocation + # "integrationRequestId": apigw.AccessLogField.context_integration_request_id, + "integrationRequestId": "$context.integration.requestId", + # Integration Response Status Code: Status code returned by the AWS Lambda function + "functionResponseStatus": apigw.AccessLogField.context_integration_status(), + # Latency of the integration, like Lambda function, in milliseconds + "integrationLatency": apigw.AccessLogField.context_integration_latency(), + # Status code returned by the AWS Lambda Service and not the backend Lambda function code + "integrationServiceStatus": apigw.AccessLogField.context_integration_status(), + # User Identity Information + "ip": apigw.AccessLogField.context_identity_source_ip(), + "userAgent": apigw.AccessLogField.context_identity_user_agent(), + } + ), + ) + + ############### # --- App --- # ############### diff --git a/pyproject.toml b/pyproject.toml index 64e4e70..835383a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "files-api" -version = "0.1.5" +version = "0.2.0" description = "Files API to upload and download files from S3 using FastAPI." readme = "README.md" authors = [ diff --git a/run b/run index d5305f7..c444312 100755 --- a/run +++ b/run @@ -2,12 +2,23 @@ set -euo pipefail -export AWS_PROFILE=sandbox +# Only set AWS_PROFILE when not running in GitHub Actions +if [[ -z "${GITHUB_ACTIONS:-}" ]]; then + export AWS_PROFILE=sandbox +fi + export AWS_REGION=us-west-2 export AWS_DEFAULT_REGION=${AWS_REGION} # aws account id/hash used to make a unique, but consistent bucket name -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text || echo "") +if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + # running in GitHub Actions + # retrieve AWS Account ID from environment variable set by GitHub Actions OIDC token + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text || echo "") +else + # not running in GitHub Actions + AWS_ACCOUNT_ID=$(aws sts get-caller-identity --profile $AWS_PROFILE --query "Account" --output text) +fi # Only create bucket name if AWS_ACCOUNT_ID is available # Suppress errors if profile doesn't exist (e.g., in CI/CD) @@ -15,7 +26,7 @@ if [ -n "$AWS_ACCOUNT_ID" ]; then AWS_ACCOUNT_ID_HASH=$(echo -n "${AWS_ACCOUNT_ID}" | sha256sum | cut -c5-8) export S3_BUCKET_NAME="python-aws-cloud-course-bucket-${AWS_ACCOUNT_ID_HASH}" # python-aws-cloud-course-bucket-6721 else - echo "WARNING: Could not retrieve AWS Account ID. Profile '$AWS_PROFILE' may not exist." + echo "WARNING: Could not retrieve AWS Account ID. Profile '${AWS_PROFILE:-}' may not exist." echo "S3_BUCKET_NAME environment variable will not be set." fi @@ -34,6 +45,18 @@ cdk-bootstrap () { cdk bootstrap "aws://${AWS_ACCOUNT_ID}/${AWS_REGION}" } + +cdk-synth () { + # Check if variable is set and not empty + if [ -n "${GITHUB_ACTIONS:-}" ]; then + # running in GitHub Actions + cdk synth --app 'uv run --script infra.py' --region $AWS_REGION + else + cdk synth --app 'uv run --script infra.py' --profile $AWS_PROFILE --region $AWS_REGION + fi +} + + cdk-deploy () { # Check if Docker is running if ! docker info >/dev/null 2>&1; then @@ -47,9 +70,38 @@ cdk-deploy () { exit 1 fi - cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION + if [ -n "${GITHUB_ACTIONS:-}" ]; then + # running in GitHub Actions + cdk deploy --app 'uv run --script infra.py' '*' --region $AWS_REGION + else + cdk deploy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION + fi +} + + +cdk-deploy:github_oidc_stack () { + set -x + local github_repo=${1} # avr2002/files-api + local create_oidc_provider=${2:-false} # boolean to indicate whether to create a new OIDC provider + + if [ -z "$github_repo" ]; then + echo "ERROR: GitHub repository not specified. Usage: cdk-deploy:github_oidc_stack . E.g., avr2002/files-api" + exit 1 + fi + + # Check if create_oidc_provider is a boolean + if [[ "$create_oidc_provider" != "true" && "$create_oidc_provider" != "false" ]]; then + echo "ERROR: create_oidc_provider must be 'true' or 'false'." + exit 1 + fi + + cdk deploy --app 'uv run --script github_oidc_infra.py' '*' \ + --context github_repo="$github_repo" \ + --context create_oidc_provider="$create_oidc_provider" \ + --profile $AWS_PROFILE --region $AWS_REGION } + cdk-destroy () { cdk destroy --app 'uv run --script infra.py' '*' --profile $AWS_PROFILE --region $AWS_REGION } @@ -257,10 +309,13 @@ test:ci() { # (example) ./run.sh test tests/test_states_info.py::test__slow_add run-tests() { + set -x PYTEST_EXIT_STATUS=0 - # Unset AWS credentials - unset AWS_PROFILE + # Unset AWS credentials (only if set) + if [[ -n "${AWS_PROFILE:-}" ]]; then + unset AWS_PROFILE + fi # Disable AWS X-Ray and AWS EMF export AWS_XRAY_SDK_ENABLED="false" diff --git a/uv.lock b/uv.lock index ac36dc6..66ccfe8 100644 --- a/uv.lock +++ b/uv.lock @@ -930,7 +930,7 @@ wheels = [ [[package]] name = "files-api" -version = "0.1.5" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "aws-embedded-metrics" },