diff --git a/.github/workflows/deploy-to-ecr.yml b/.github/workflows/deploy-to-ecr.yml index d9dbf8e..d596c63 100644 --- a/.github/workflows/deploy-to-ecr.yml +++ b/.github/workflows/deploy-to-ecr.yml @@ -6,21 +6,17 @@ on: - master jobs: - build: + build-and-deploy: if: github.repository == 'abdenlab/cfdb' - # Available versions: - # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on runs-on: ubuntu-24.04 permissions: id-token: write contents: read - # Steps represent a sequence of tasks that will be executed as part of the job steps: - uses: actions/checkout@v4 - name: Configure AWS Credentials - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} uses: aws-actions/configure-aws-credentials@v3 with: role-to-assume: ${{ secrets.AWS_IAM_ROLE }} @@ -30,11 +26,24 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Push to ECR + - name: Build and push to ECR env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} REPOSITORY: cfdb - IMAGE_TAG: latest run: | - docker build --file Dockerfile.api -t $REGISTRY/$REPOSITORY:$IMAGE_TAG . - docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG \ No newline at end of file + SHA_TAG=${GITHUB_SHA::8} + docker build --file Dockerfile.api \ + -t $REGISTRY/$REPOSITORY:latest \ + -t $REGISTRY/$REPOSITORY:$SHA_TAG . + docker push $REGISTRY/$REPOSITORY:latest + docker push $REGISTRY/$REPOSITORY:$SHA_TAG + + - name: Deploy to ECS + env: + ECS_CLUSTER: ${{ secrets.ECS_CLUSTER }} + ECS_SERVICE: ${{ secrets.ECS_SERVICE }} + run: | + aws ecs update-service \ + --cluster $ECS_CLUSTER \ + --service $ECS_SERVICE \ + --force-new-deployment diff --git a/Dockerfile.api b/Dockerfile.api index 05ede65..20d1f75 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -10,6 +10,12 @@ EXPOSE 8000 ENV DATABASE_URL="mongodb://cvh-backend:27017" WORKDIR /app +# Install curl (for ECS health checks) and download AWS DocumentDB CA bundle +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /etc/cfdb/certs \ + && curl --fail -sS -o /etc/cfdb/certs/global-bundle.pem https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \ + && chmod 644 /etc/cfdb/certs/global-bundle.pem + # Install the materializer binary COPY --from=builder /build/target/release/materialize /usr/local/bin/materialize diff --git a/Dockerfile.mongodb b/Dockerfile.mongodb index fc16aa1..2dbe282 100644 --- a/Dockerfile.mongodb +++ b/Dockerfile.mongodb @@ -3,87 +3,37 @@ FROM mongo:latest # Copy database dump and scripts COPY database/ /data/database/ COPY scripts/create-indexes.js /scripts/create-indexes.js -COPY scripts/create-x509-users.js /scripts/create-x509-users.js -# Copy TLS configuration -COPY docker/mongodb/mongod-tls.conf /etc/mongodb/mongod-tls.conf - -# Create startup script with conditional TLS support +# Create startup script (development mode only) COPY <<'EOF' /startup.sh #!/bin/bash set -e -# Check if TLS certificates are mounted (production mode) -if [ -f /etc/mongodb/certs/server-bundle.pem ] && [ -f /etc/mongodb/certs/ca.pem ]; then - echo "=== TLS certificates found - starting in PRODUCTION mode ===" - - # Phase 1: Start MongoDB with TLS but WITHOUT auth (for initial setup) - echo "Phase 1: Starting MongoDB with TLS (no auth for setup)..." - mongod --bind_ip_all \ - --tlsMode requireTLS \ - --tlsCertificateKeyFile /etc/mongodb/certs/server-bundle.pem \ - --tlsCAFile /etc/mongodb/certs/ca.pem \ - --tlsAllowConnectionsWithoutCertificates & - MONGOD_PID=$! - - # Wait for MongoDB to be ready - echo "Waiting for MongoDB to start..." - until mongosh --tls --tlsAllowInvalidCertificates \ - --eval "db.adminCommand('ping')" >/dev/null 2>&1; do - sleep 1 - done - echo "MongoDB started with TLS (no auth)." - - # Restore database - echo "Restoring database..." - mongorestore --gzip /data/database \ - --ssl --sslAllowInvalidCertificates - - # Create indexes - echo "Creating indexes..." - mongosh --tls --tlsAllowInvalidCertificates \ - cfdb /scripts/create-indexes.js - - # Create X.509 users - echo "Creating X.509 users..." - mongosh --tls --tlsAllowInvalidCertificates \ - admin /scripts/create-x509-users.js - - echo "Phase 1 complete. Restarting MongoDB with auth enabled..." - - # Phase 2: Shutdown and restart with full security - kill $MONGOD_PID - wait $MONGOD_PID 2>/dev/null - - echo "Phase 2: Starting MongoDB with TLS and X.509 authentication..." - exec mongod --config /etc/mongodb/mongod-tls.conf --setParameter authenticationMechanisms=MONGODB-X509 -else - echo "=== No TLS certificates found - starting in DEVELOPMENT mode ===" +echo "=== Starting MongoDB in DEVELOPMENT mode ===" - # Start MongoDB without TLS (original behavior) - mongod --bind_ip_all & - MONGOD_PID=$! +# Start MongoDB without TLS +mongod --bind_ip_all & +MONGOD_PID=$! - # Wait for MongoDB to be ready - echo "Waiting for MongoDB to start..." - until mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; do - sleep 1 - done - echo "MongoDB started." +# Wait for MongoDB to be ready +echo "Waiting for MongoDB to start..." +until mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; do + sleep 1 +done +echo "MongoDB started." - # Restore database - echo "Restoring database..." - mongorestore --gzip /data/database +# Restore database +echo "Restoring database..." +mongorestore --gzip /data/database - # Create indexes - echo "Creating indexes..." - mongosh cfdb /scripts/create-indexes.js +# Create indexes +echo "Creating indexes..." +mongosh cfdb /scripts/create-indexes.js - echo "=== Development initialization complete ===" +echo "=== Development initialization complete ===" - # Keep MongoDB running - wait $MONGOD_PID -fi +# Keep MongoDB running +wait $MONGOD_PID EOF RUN chmod +x /startup.sh diff --git a/Makefile b/Makefile index dad2f29..e26653e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -# Certificate directory (customize for production) -CERT_DIR ?= $(PWD)/certs - network: @echo "Checking if Docker network 'cvh-backend-network' exists..." @if ! docker network inspect cvh-backend-network >/dev/null 2>&1; then \ @@ -10,40 +7,16 @@ network: echo "Network cvh-backend-network already exists."; \ fi -# Generate certificates for TLS/X.509 authentication -certs: - @echo "Generating TLS certificates..." - ./certs/generate-certs.sh - @echo "Certificates generated in $(CERT_DIR)" - -# Development mode (no authentication) mongodb: make network @docker stop mongodb 2>/dev/null || true @docker rm mongodb 2>/dev/null || true @echo "Building MongoDB image..." docker build -t cfdb-mongodb -f Dockerfile.mongodb . - @echo "Starting MongoDB container in DEVELOPMENT mode (no TLS)..." + @echo "Starting MongoDB container..." docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 cfdb-mongodb @echo "MongoDB container starting on port 27017. Check logs with: docker logs -f mongodb" -# Production mode (TLS/X.509 authentication) -mongodb-prod: - make network - @docker stop mongodb 2>/dev/null || true - @docker rm mongodb 2>/dev/null || true - @echo "Building MongoDB image..." - docker build -t cfdb-mongodb -f Dockerfile.mongodb . - @echo "Starting MongoDB container in PRODUCTION mode (TLS/X.509)..." - docker run -d --name mongodb \ - --network cvh-backend-network \ - --network-alias cvh-backend \ - -p 27017:27017 \ - -v $(CERT_DIR)/ca/ca.pem:/etc/mongodb/certs/ca.pem:ro \ - -v $(CERT_DIR)/server/cvh-backend-bundle.pem:/etc/mongodb/certs/server-bundle.pem:ro \ - cfdb-mongodb - @echo "MongoDB container starting on port 27017 with TLS. Check logs with: docker logs -f mongodb" - build-materialize: @echo "Building materializer..." cd materialize && cargo build --release @@ -54,64 +27,22 @@ install-materialize: build-materialize sudo cp materialize/target/release/materialize /usr/local/bin/ @echo "Materializer installed." -# Development mode (no authentication) materialize-files: build-materialize - @echo "Materializing 'files' collection (dev mode)..." + @echo "Materializing 'files' collection..." ./materialize/target/release/materialize @echo "Files collection created successfully." materialize-dcc: build-materialize - @echo "Materializing file metadata for $(DCC) (dev mode)..." - ./materialize/target/release/materialize --submission $(DCC) - @echo "Done." - -# Production mode (TLS/X.509 authentication) -materialize-files-prod: build-materialize - @echo "Materializing 'files' collection (TLS/X.509)..." - MONGODB_TLS_ENABLED=true \ - MONGODB_CERT_PATH=$(CERT_DIR)/clients/cfdb-materializer-bundle.pem \ - MONGODB_CA_PATH=$(CERT_DIR)/ca/ca.pem \ - DATABASE_URL=mongodb://cvh-backend:27017 \ - ./materialize/target/release/materialize - @echo "Files collection created successfully." - -materialize-dcc-prod: build-materialize - @echo "Materializing file metadata for $(DCC) (TLS/X.509)..." - MONGODB_TLS_ENABLED=true \ - MONGODB_CERT_PATH=$(CERT_DIR)/clients/cfdb-materializer-bundle.pem \ - MONGODB_CA_PATH=$(CERT_DIR)/ca/ca.pem \ - DATABASE_URL=mongodb://cvh-backend:27017 \ + @echo "Materializing file metadata for $(DCC)..." ./materialize/target/release/materialize --submission $(DCC) @echo "Done." -# Development mode (no authentication) api: make network @docker stop api 2>/dev/null || true @docker rm api 2>/dev/null || true @echo "Building the API Docker image..." docker build -t api -f Dockerfile.api . - @echo "Starting the API container in DEVELOPMENT mode (no TLS)..." + @echo "Starting the API container..." docker run -d --name api --network cvh-backend-network --network-alias cvh-backend -p 8000:8000 -e SYNC_API_KEY=dev-sync-key -e SYNC_DATA_DIR=/tmp/sync-data api @echo "API container is up and running on port 8000 (http://0.0.0.0:8000/metadata)." - -# Production mode (TLS/X.509 authentication) -api-prod: - make network - @docker stop api 2>/dev/null || true - @docker rm api 2>/dev/null || true - @echo "Building the API Docker image..." - docker build -t api -f Dockerfile.api . - @echo "Starting the API container in PRODUCTION mode (TLS/X.509)..." - docker run -d --name api \ - --network cvh-backend-network \ - --network-alias cvh-backend \ - -p 8000:8000 \ - -e MONGODB_TLS_ENABLED=true \ - -e DATABASE_URL=mongodb://cvh-backend:27017 \ - -e SYNC_API_KEY=$(SYNC_API_KEY) \ - -e SYNC_DATA_DIR=/tmp/sync-data \ - -v $(CERT_DIR)/ca/ca.pem:/etc/cfdb/certs/ca.pem:ro \ - -v $(CERT_DIR)/clients/cfdb-api-bundle.pem:/etc/cfdb/certs/client-bundle.pem:ro \ - api - @echo "API container is up with TLS on port 8000 (http://0.0.0.0:8000/metadata)." diff --git a/certs/.gitignore b/certs/.gitignore deleted file mode 100644 index a9c17eb..0000000 --- a/certs/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -# Exclude all generated certificates and keys -# These contain sensitive cryptographic material - -# CA files -ca/ - -# Server certificates -server/ - -# Client certificates -clients/ - -# Keep the generation script -!generate-certs.sh -!.gitignore diff --git a/certs/generate-certs.sh b/certs/generate-certs.sh deleted file mode 100755 index 2afbfc8..0000000 --- a/certs/generate-certs.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/bin/bash -# Certificate generation script for MongoDB X.509 authentication - -set -e - -usage() { - cat << EOF -Usage: $(basename "$0") [OPTIONS] [HOSTNAME] [IP_ADDRESS] - -Generate TLS certificates for MongoDB X.509 authentication. - -Arguments: - HOSTNAME MongoDB server hostname (default: cvh-backend) - IP_ADDRESS MongoDB server IP address (default: 127.0.0.1) - -Options: - -h, --help Show this help message and exit - -Environment variables (used if arguments not provided): - MONGODB_HOSTNAME MongoDB server hostname - MONGODB_IP MongoDB server IP address - -Configuration precedence: - 1. Command-line arguments (highest) - 2. Environment variables - 3. Defaults (lowest) - -Examples: - $(basename "$0") # Use defaults (local dev) - $(basename "$0") mongodb.example.com 10.0.1.50 # Production with args - MONGODB_HOSTNAME=db.example.com $(basename "$0") # Production with env var - -Output: - certs/ca/ca.pem - CA certificate (deploy everywhere) - certs/server/mongodb-server-bundle.pem - Server certificate bundle - certs/clients/cfdb-api-bundle.pem - API client certificate - certs/clients/cfdb-materializer-bundle.pem - Materializer client certificate -EOF - exit 0 -} - -# Parse options -case "${1:-}" in - -h|--help) - usage - ;; -esac - -CERT_DIR="$(cd "$(dirname "$0")" && pwd)" -DAYS_CA=3650 # 10 years -DAYS_CERT=365 # 1 year - -# Organization details (customize as needed) -ORG="Abdenlab" -COUNTRY="US" - -# MongoDB server hostname: arg1 > env var > default -MONGODB_HOSTNAME="${1:-${MONGODB_HOSTNAME:-cvh-backend}}" - -# MongoDB server IP: arg2 > env var > default -MONGODB_IP="${2:-${MONGODB_IP:-127.0.0.1}}" - -echo "=== CFDB Certificate Generation ===" -echo "Output directory: ${CERT_DIR}" -echo "MongoDB hostname: ${MONGODB_HOSTNAME}" -echo "MongoDB IP: ${MONGODB_IP}" -echo "" - -# Create directories -mkdir -p "${CERT_DIR}/ca" "${CERT_DIR}/server" "${CERT_DIR}/clients" - -# ============================================================================= -# Generate Root CA -# ============================================================================= -echo "=== Generating Root CA ===" -openssl genrsa -out "${CERT_DIR}/ca/ca.key" 4096 -openssl req -new -x509 -days ${DAYS_CA} \ - -key "${CERT_DIR}/ca/ca.key" \ - -out "${CERT_DIR}/ca/ca.pem" \ - -subj "/CN=CFDB Root CA/O=${ORG}/C=${COUNTRY}" - -echo " Created: ca/ca.key (private key - keep secure!)" -echo " Created: ca/ca.pem (certificate - deploy to all containers)" - -# ============================================================================= -# Generate MongoDB Server Certificate -# ============================================================================= -echo "" -echo "=== Generating MongoDB Server Certificate ===" - -# Generate key and CSR -openssl genrsa -out "${CERT_DIR}/server/mongodb-server.key" 2048 -openssl req -new \ - -key "${CERT_DIR}/server/mongodb-server.key" \ - -out "${CERT_DIR}/server/mongodb-server.csr" \ - -subj "/CN=${MONGODB_HOSTNAME}/O=${ORG}/C=${COUNTRY}" - -# Create SAN extension config with configured hostname and IP -cat > "${CERT_DIR}/server/san.cnf" << SANEOF -[req] -distinguished_name = req_distinguished_name -req_extensions = v3_req -prompt = no - -[req_distinguished_name] -CN = ${MONGODB_HOSTNAME} - -[v3_req] -subjectAltName = @alt_names - -[alt_names] -DNS.1 = ${MONGODB_HOSTNAME} -DNS.2 = localhost -DNS.3 = mongodb -DNS.4 = cvh-backend -IP.1 = ${MONGODB_IP} -IP.2 = 127.0.0.1 -SANEOF - -# Sign with CA including SAN -openssl x509 -req -days ${DAYS_CERT} \ - -in "${CERT_DIR}/server/mongodb-server.csr" \ - -CA "${CERT_DIR}/ca/ca.pem" \ - -CAkey "${CERT_DIR}/ca/ca.key" \ - -CAcreateserial \ - -out "${CERT_DIR}/server/mongodb-server.pem" \ - -sha256 \ - -extfile "${CERT_DIR}/server/san.cnf" \ - -extensions v3_req - -# Create server bundle (MongoDB requires key + cert in one file) -cat "${CERT_DIR}/server/mongodb-server.key" \ - "${CERT_DIR}/server/mongodb-server.pem" > \ - "${CERT_DIR}/server/mongodb-server-bundle.pem" - -# Also create symlink with old name for backward compatibility -ln -sf mongodb-server-bundle.pem "${CERT_DIR}/server/cvh-backend-bundle.pem" - -echo " Created: server/mongodb-server-bundle.pem (server key+cert bundle)" -echo " Created: server/cvh-backend-bundle.pem -> mongodb-server-bundle.pem (symlink)" - -# ============================================================================= -# Generate API Client Certificate -# ============================================================================= -echo "" -echo "=== Generating API Client Certificate ===" -# Note: Client certificates must use a DIFFERENT Organization than server cert -# to avoid MongoDB thinking they're cluster members - -openssl genrsa -out "${CERT_DIR}/clients/cfdb-api.key" 2048 -openssl req -new \ - -key "${CERT_DIR}/clients/cfdb-api.key" \ - -out "${CERT_DIR}/clients/cfdb-api.csr" \ - -subj "/CN=cfdb-api/OU=Clients/O=${ORG}-Clients/C=${COUNTRY}" - -openssl x509 -req -days ${DAYS_CERT} \ - -in "${CERT_DIR}/clients/cfdb-api.csr" \ - -CA "${CERT_DIR}/ca/ca.pem" \ - -CAkey "${CERT_DIR}/ca/ca.key" \ - -CAcreateserial \ - -out "${CERT_DIR}/clients/cfdb-api.pem" \ - -sha256 - -# Create client bundle -cat "${CERT_DIR}/clients/cfdb-api.key" \ - "${CERT_DIR}/clients/cfdb-api.pem" > \ - "${CERT_DIR}/clients/cfdb-api-bundle.pem" - -echo " Created: clients/cfdb-api-bundle.pem" - -# ============================================================================= -# Generate Materializer Client Certificate -# ============================================================================= -echo "" -echo "=== Generating Materializer Client Certificate ===" - -openssl genrsa -out "${CERT_DIR}/clients/cfdb-materializer.key" 2048 -openssl req -new \ - -key "${CERT_DIR}/clients/cfdb-materializer.key" \ - -out "${CERT_DIR}/clients/cfdb-materializer.csr" \ - -subj "/CN=cfdb-materializer/OU=Clients/O=${ORG}-Clients/C=${COUNTRY}" - -openssl x509 -req -days ${DAYS_CERT} \ - -in "${CERT_DIR}/clients/cfdb-materializer.csr" \ - -CA "${CERT_DIR}/ca/ca.pem" \ - -CAkey "${CERT_DIR}/ca/ca.key" \ - -CAcreateserial \ - -out "${CERT_DIR}/clients/cfdb-materializer.pem" \ - -sha256 - -# Create client bundle -cat "${CERT_DIR}/clients/cfdb-materializer.key" \ - "${CERT_DIR}/clients/cfdb-materializer.pem" > \ - "${CERT_DIR}/clients/cfdb-materializer-bundle.pem" - -echo " Created: clients/cfdb-materializer-bundle.pem" - -# ============================================================================= -# Set Permissions -# ============================================================================= -echo "" -echo "=== Setting Permissions ===" -chmod 400 "${CERT_DIR}/ca/ca.key" -chmod 400 "${CERT_DIR}/server/"*.key 2>/dev/null || true -chmod 400 "${CERT_DIR}/clients/"*.key -chmod 444 "${CERT_DIR}/ca/ca.pem" -chmod 444 "${CERT_DIR}/server/"*.pem 2>/dev/null || true -chmod 444 "${CERT_DIR}/clients/"*.pem -echo " Private keys: 400 (owner read only)" -echo " Certificates: 444 (read only)" - -# ============================================================================= -# Cleanup temporary files -# ============================================================================= -rm -f "${CERT_DIR}/server/"*.csr "${CERT_DIR}/clients/"*.csr "${CERT_DIR}/server/san.cnf" - -# ============================================================================= -# Summary -# ============================================================================= -echo "" -echo "=== Certificate Generation Complete ===" -echo "" -echo "Configuration used:" -echo " MongoDB hostname: ${MONGODB_HOSTNAME}" -echo " MongoDB IP: ${MONGODB_IP}" -echo "" -echo "Files created:" -echo " ${CERT_DIR}/ca/ca.pem - CA certificate" -echo " ${CERT_DIR}/server/mongodb-server-bundle.pem - MongoDB server bundle" -echo " ${CERT_DIR}/clients/cfdb-api-bundle.pem - API client bundle" -echo " ${CERT_DIR}/clients/cfdb-materializer-bundle.pem - Materializer client bundle" -echo "" -echo "Server certificate SANs:" -echo " DNS: ${MONGODB_HOSTNAME}, localhost, mongodb, cvh-backend" -echo " IP: ${MONGODB_IP}, 127.0.0.1" -echo "" -echo "MongoDB X.509 usernames (Subject DNs - RFC 2253 order):" -echo " API: C=${COUNTRY},O=${ORG}-Clients,OU=Clients,CN=cfdb-api" -echo " Materializer: C=${COUNTRY},O=${ORG}-Clients,OU=Clients,CN=cfdb-materializer" -echo "" -echo "Usage examples:" -echo " Local dev: ./generate-certs.sh" -echo " With args: ./generate-certs.sh mongodb.example.com 10.0.1.50" -echo " With env: MONGODB_HOSTNAME=db.example.com MONGODB_IP=10.0.1.50 ./generate-certs.sh" -echo "" -echo "IMPORTANT: Keep ca/ca.key secure and never commit certificates to git!" diff --git a/cloudformation/backend.yml b/cloudformation/backend.yml new file mode 100644 index 0000000..18566a2 --- /dev/null +++ b/cloudformation/backend.yml @@ -0,0 +1,285 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + CFDB backend — ECS Fargate service behind an ALB with HTTPS, + Route 53 DNS, and Secrets Manager integration for DocumentDB. + +Parameters: + NetworkStackName: + Type: String + Description: Name of the network stack + DatabaseStackName: + Type: String + Description: Name of the database stack + ImageURI: + Type: String + Description: ECR image URI for the CFDB API + DesiredCount: + Type: Number + Default: 1 + Description: Number of ECS tasks + HostedZoneName: + Type: String + Default: vis-api.link + HostedZoneId: + Type: String + Default: Z09477406JQAR0KB7G87 + DomainName: + Type: String + Default: cfdb.vis-api.link + +Resources: + # ---------- Secrets ---------- + SyncApiKeySecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub "${AWS::StackName}-sync-api-key" + Description: API key for the CFDB /sync endpoint + GenerateSecretString: + PasswordLength: 48 + ExcludePunctuation: true + + # ---------- ACM certificate ---------- + Certificate: + Type: AWS::CertificateManager::Certificate + Properties: + DomainName: !Ref DomainName + ValidationMethod: DNS + DomainValidationOptions: + - DomainName: !Ref DomainName + HostedZoneId: !Ref HostedZoneId + + # ---------- ALB ---------- + LoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Scheme: internet-facing + Type: application + SecurityGroups: + - Fn::ImportValue: !Sub "${NetworkStackName}-ALBSecurityGroupId" + Subnets: !Split + - "," + - Fn::ImportValue: !Sub "${NetworkStackName}-PublicSubnetIds" + LoadBalancerAttributes: + - Key: idle_timeout.timeout_seconds + Value: "30" + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-alb" + + TargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Name: !Sub "cfdb-tg" + Port: 8000 + Protocol: HTTP + TargetType: ip + VpcId: + Fn::ImportValue: !Sub "${NetworkStackName}-VpcId" + HealthCheckPath: /health + HealthCheckIntervalSeconds: 30 + HealthCheckTimeoutSeconds: 5 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 3 + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-tg" + + HTTPSListener: + Type: AWS::ElasticLoadBalancingV2::Listener + DependsOn: LoadBalancer + Properties: + LoadBalancerArn: !Ref LoadBalancer + Port: 443 + Protocol: HTTPS + SslPolicy: ELBSecurityPolicy-TLS13-1-2-2021-06 + Certificates: + - CertificateArn: !Ref Certificate + DefaultActions: + - Type: forward + TargetGroupArn: !Ref TargetGroup + + HTTPRedirectListener: + Type: AWS::ElasticLoadBalancingV2::Listener + DependsOn: LoadBalancer + Properties: + LoadBalancerArn: !Ref LoadBalancer + Port: 80 + Protocol: HTTP + DefaultActions: + - Type: redirect + RedirectConfig: + Protocol: HTTPS + Port: "443" + StatusCode: HTTP_301 + + # ---------- Route 53 ---------- + DNSRecord: + Type: AWS::Route53::RecordSetGroup + Properties: + HostedZoneName: !Sub "${HostedZoneName}." + RecordSets: + - Name: !Ref DomainName + Type: A + AliasTarget: + DNSName: !GetAtt LoadBalancer.DNSName + HostedZoneId: !GetAtt LoadBalancer.CanonicalHostedZoneID + EvaluateTargetHealth: false + + # ---------- ECS cluster ---------- + ECSCluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Sub "${AWS::StackName}-cluster" + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-cluster" + + # ---------- CloudWatch log group ---------- + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/ecs/${AWS::StackName}" + RetentionInDays: 30 + + # ---------- Task definition ---------- + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub "${AWS::StackName}-task" + Cpu: "1024" + Memory: "2048" + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: !GetAtt ExecutionRole.Arn + TaskRoleArn: !GetAtt TaskRole.Arn + ContainerDefinitions: + - Name: cfdb-api + Image: !Ref ImageURI + Essential: true + PortMappings: + - ContainerPort: 8000 + Protocol: tcp + Environment: + - Name: MONGODB_TLS_ENABLED + Value: "true" + - Name: MONGODB_RETRY_WRITES + Value: "false" + - Name: SYNC_DATA_DIR + Value: /tmp/sync-data + Secrets: + - Name: DATABASE_URL + ValueFrom: + Fn::ImportValue: !Sub "${DatabaseStackName}-ConnectionURLSecretArn" + - Name: SYNC_API_KEY + ValueFrom: !Ref SyncApiKeySecret + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref LogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: ecs + HealthCheck: + Command: + - CMD-SHELL + - curl -f http://127.0.0.1:8000/health || exit 1 + Interval: 30 + Timeout: 5 + Retries: 3 + + # ---------- ECS service ---------- + ECSService: + Type: AWS::ECS::Service + DependsOn: + - HTTPSListener + - HTTPRedirectListener + Properties: + Cluster: !Ref ECSCluster + TaskDefinition: !Ref TaskDefinition + DesiredCount: !Ref DesiredCount + LaunchType: FARGATE + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 100 + HealthCheckGracePeriodSeconds: 60 + EnableECSManagedTags: true + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + SecurityGroups: + - Fn::ImportValue: !Sub "${NetworkStackName}-ECSSecurityGroupId" + Subnets: !Split + - "," + - Fn::ImportValue: !Sub "${NetworkStackName}-PublicSubnetIds" + LoadBalancers: + - ContainerName: cfdb-api + ContainerPort: 8000 + TargetGroupArn: !Ref TargetGroup + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-service" + + # ---------- IAM: execution role ---------- + ExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: execution + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ecr:GetAuthorizationToken + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + Resource: "*" + - Effect: Allow + Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: !GetAtt LogGroup.Arn + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - Fn::ImportValue: !Sub "${DatabaseStackName}-ConnectionURLSecretArn" + - !Ref SyncApiKeySecret + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-execution-role" + + # ---------- IAM: task role ---------- + TaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-task-role" + +Outputs: + LoadBalancerDNSName: + Value: !GetAtt LoadBalancer.DNSName + ECSClusterName: + Value: !Ref ECSCluster + Export: + Name: !Sub "${AWS::StackName}-ECSClusterName" + ECSServiceName: + Value: !GetAtt ECSService.Name + Export: + Name: !Sub "${AWS::StackName}-ECSServiceName" diff --git a/cloudformation/database.yml b/cloudformation/database.yml new file mode 100644 index 0000000..996f85f --- /dev/null +++ b/cloudformation/database.yml @@ -0,0 +1,123 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + CFDB DocumentDB cluster with Secrets Manager for credential and + connection URL management. + +Parameters: + NetworkStackName: + Type: String + Description: Name of the network stack (for cross-stack imports) + DBMasterUsername: + Type: String + Description: Master username for the DocumentDB cluster + NoEcho: true + InstanceClass: + Type: String + Default: db.t3.medium + Description: DocumentDB instance class + InstanceCount: + Type: Number + Default: 1 + MinValue: 1 + MaxValue: 1 + Description: Number of DocumentDB instances + +Resources: + # ---------- Master credentials ---------- + DBMasterSecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub "${AWS::StackName}-master-credentials" + Description: DocumentDB master credentials + GenerateSecretString: + SecretStringTemplate: !Sub '{"username": "${DBMasterUsername}"}' + GenerateStringKey: password + PasswordLength: 32 + ExcludePunctuation: true + + # ---------- DocumentDB cluster ---------- + DBClusterParameterGroup: + Type: AWS::DocDB::DBClusterParameterGroup + Properties: + Description: CFDB DocumentDB 5.0 parameters + Family: docdb5.0 + Parameters: + tls: enabled + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-params" + + DBCluster: + Type: AWS::DocDB::DBCluster + DeletionPolicy: Snapshot + Properties: + DBClusterIdentifier: !Sub "${AWS::StackName}-cluster" + DBSubnetGroupName: + Fn::ImportValue: !Sub "${NetworkStackName}-DocumentDBSubnetGroupName" + VpcSecurityGroupIds: + - Fn::ImportValue: !Sub "${NetworkStackName}-DocumentDBSecurityGroupId" + DBClusterParameterGroupName: !Ref DBClusterParameterGroup + EngineVersion: "5.0.0" + MasterUsername: !Sub "{{resolve:secretsmanager:${DBMasterSecret}:SecretString:username}}" + MasterUserPassword: !Sub "{{resolve:secretsmanager:${DBMasterSecret}:SecretString:password}}" + StorageEncrypted: true + BackupRetentionPeriod: 7 + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-cluster" + + DBInstance1: + Type: AWS::DocDB::DBInstance + Properties: + DBClusterIdentifier: !Ref DBCluster + DBInstanceClass: !Ref InstanceClass + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-instance-1" + + DBInstance2: + Type: AWS::DocDB::DBInstance + Condition: MultiInstance + Properties: + DBClusterIdentifier: !Ref DBCluster + DBInstanceClass: !Ref InstanceClass + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-instance-2" + + DBInstance3: + Type: AWS::DocDB::DBInstance + Condition: ThreeInstances + Properties: + DBClusterIdentifier: !Ref DBCluster + DBInstanceClass: !Ref InstanceClass + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-instance-3" + + # ---------- Connection URL secret ---------- + ConnectionURLSecret: + Type: AWS::SecretsManager::Secret + DependsOn: DBCluster + Properties: + Name: !Sub "${AWS::StackName}-connection-url" + Description: Full DATABASE_URL for the CFDB API + SecretString: !Sub + - "mongodb://{{resolve:secretsmanager:${Secret}:SecretString:username}}:{{resolve:secretsmanager:${Secret}:SecretString:password}}@${Endpoint}:${Port}/cfdb?tls=true&tlsCAFile=/etc/cfdb/certs/global-bundle.pem&replicaSet=rs0&readPreference=secondaryPreferred&retryWrites=false" + - Secret: !Ref DBMasterSecret + Endpoint: !GetAtt DBCluster.Endpoint + Port: !GetAtt DBCluster.Port + +Conditions: + MultiInstance: !Not [!Equals [!Ref InstanceCount, 1]] + ThreeInstances: !Equals [!Ref InstanceCount, 3] + +Outputs: + ClusterEndpoint: + Value: !GetAtt DBCluster.Endpoint + Export: + Name: !Sub "${AWS::StackName}-ClusterEndpoint" + ConnectionURLSecretArn: + Value: !Ref ConnectionURLSecret + Export: + Name: !Sub "${AWS::StackName}-ConnectionURLSecretArn" diff --git a/cloudformation/network.yml b/cloudformation/network.yml new file mode 100644 index 0000000..c0a8950 --- /dev/null +++ b/cloudformation/network.yml @@ -0,0 +1,453 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: > + CFDB network infrastructure — VPC with public subnets (ALB + ECS) and + private subnets (DocumentDB) across two Availability Zones. + +Parameters: + CidrBlock: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.2.0.0/21 + Description: VPC CIDR block + Type: String + CidrPublicSubnetA: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.2.0.0/24 + Type: String + CidrPublicSubnetB: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.2.1.0/24 + Type: String + CidrPrivateSubnetA: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.2.2.0/24 + Type: String + CidrPrivateSubnetB: + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/\d{1,2}' + Default: 10.2.3.0/24 + Type: String + +Mappings: + # Deterministic AZ-ID-to-name mapping (account-independent) + RegionMap: + us-east-1: + ZoneId1: use1-az6 + ZoneId2: use1-az4 + us-east-2: + ZoneId1: use2-az2 + ZoneId2: use2-az3 + us-west-1: + ZoneId1: usw1-az1 + ZoneId2: usw1-az3 + us-west-2: + ZoneId1: usw2-az1 + ZoneId2: usw2-az2 + eu-central-1: + ZoneId1: euc1-az3 + ZoneId2: euc1-az2 + eu-west-1: + ZoneId1: euw1-az1 + ZoneId2: euw1-az2 + eu-west-2: + ZoneId1: euw2-az2 + ZoneId2: euw2-az3 + eu-west-3: + ZoneId1: euw3-az1 + ZoneId2: euw3-az2 + eu-north-1: + ZoneId1: eun1-az2 + ZoneId2: eun1-az1 + ca-central-1: + ZoneId1: cac1-az2 + ZoneId2: cac1-az1 + eu-south-1: + ZoneId1: eus1-az2 + ZoneId2: eus1-az1 + ap-east-1: + ZoneId1: ape1-az3 + ZoneId2: ape1-az2 + ap-northeast-1: + ZoneId1: apne1-az4 + ZoneId2: apne1-az1 + ap-northeast-2: + ZoneId1: apne2-az1 + ZoneId2: apne2-az3 + ap-south-1: + ZoneId1: aps1-az2 + ZoneId2: aps1-az3 + ap-southeast-1: + ZoneId1: apse1-az1 + ZoneId2: apse1-az2 + ap-southeast-2: + ZoneId1: apse2-az3 + ZoneId2: apse2-az1 + us-gov-west-1: + ZoneId1: usgw1-az1 + ZoneId2: usgw1-az2 + ap-northeast-3: + ZoneId1: apne3-az3 + ZoneId2: apne3-az2 + sa-east-1: + ZoneId1: sae1-az3 + ZoneId2: sae1-az2 + af-south-1: + ZoneId1: afs1-az3 + ZoneId2: afs1-az2 + ap-south-2: + ZoneId1: aps2-az3 + ZoneId2: aps2-az2 + ap-southeast-3: + ZoneId1: apse3-az3 + ZoneId2: apse3-az2 + ap-southeast-4: + ZoneId1: apse4-az3 + ZoneId2: apse4-az2 + ca-west-1: + ZoneId1: caw1-az3 + ZoneId2: caw1-az2 + eu-central-2: + ZoneId1: euc2-az3 + ZoneId2: euc2-az2 + eu-south-2: + ZoneId1: eus2-az3 + ZoneId2: eus2-az2 + il-central-1: + ZoneId1: ilc1-az3 + ZoneId2: ilc1-az2 + me-central-1: + ZoneId1: mec1-az3 + ZoneId2: mec1-az2 + +Resources: + # ---------- VPC ---------- + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: !Ref CidrBlock + EnableDnsHostnames: true + EnableDnsSupport: true + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-vpc" + + VPCFlowLog: + Type: AWS::EC2::FlowLog + Properties: + ResourceId: !Ref VPC + ResourceType: VPC + TrafficType: ALL + LogDestinationType: cloud-watch-logs + LogGroupName: !Sub "${AWS::StackName}-VPCFlowLogs" + DeliverLogsPermissionArn: !GetAtt FlowLogRole.Arn + + FlowLogRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: vpc-flow-logs.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: FlowLogPolicy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DescribeLogGroups + - logs:DescribeLogStreams + Resource: !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${AWS::StackName}-VPCFlowLogs:*" + + InternetGateway: + Type: AWS::EC2::InternetGateway + + AttachGateway: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: !Ref VPC + InternetGatewayId: !Ref InternetGateway + + # ---------- AZ resolution ---------- + AvailabilityZone1: + Type: Custom::AvailabilityZone + DependsOn: LogGroupGetAZLambdaFunction + Properties: + ServiceToken: !GetAtt GetAZLambdaFunction.Arn + ZoneId: !FindInMap [RegionMap, !Ref "AWS::Region", ZoneId1] + + AvailabilityZone2: + Type: Custom::AvailabilityZone + DependsOn: LogGroupGetAZLambdaFunction + Properties: + ServiceToken: !GetAtt GetAZLambdaFunction.Arn + ZoneId: !FindInMap [RegionMap, !Ref "AWS::Region", ZoneId2] + + LogGroupGetAZLambdaFunction: + Type: AWS::Logs::LogGroup + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + LogGroupName: !Sub /aws/lambda/${GetAZLambdaFunction} + RetentionInDays: 7 + + GetAZLambdaFunction: + Type: AWS::Lambda::Function + Properties: + Description: Resolve AZ ID to AZ name + Timeout: 60 + Runtime: python3.9 + Handler: index.handler + Role: !GetAtt GetAZLambdaRole.Arn + Code: + ZipFile: | + import cfnresponse + from json import dumps + from boto3 import client + EC2 = client('ec2') + def handler(event, context): + if event['RequestType'] in ('Create', 'Update'): + print(dumps(event, default=str)) + data = {} + try: + response = EC2.describe_availability_zones( + Filters=[{'Name': 'zone-id', 'Values': [event['ResourceProperties']['ZoneId']]}] + ) + print(dumps(response, default=str)) + data['ZoneName'] = response['AvailabilityZones'][0]['ZoneName'] + except Exception as error: + cfnresponse.send(event, context, cfnresponse.FAILED, {}, reason=str(error)) + return + cfnresponse.send(event, context, cfnresponse.SUCCESS, data) + else: + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}) + + GetAZLambdaRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: sts:AssumeRole + Principal: + Service: !Sub "lambda.${AWS::URLSuffix}" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + Policies: + - PolicyName: DescribeAZs + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: ec2:DescribeAvailabilityZones + Resource: "*" + + # ---------- Public subnets (ALB + ECS) ---------- + PublicSubnetA: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: !Ref CidrPublicSubnetA + AvailabilityZone: !GetAtt AvailabilityZone1.ZoneName + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-public-a" + + PublicSubnetB: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: !Ref CidrPublicSubnetB + AvailabilityZone: !GetAtt AvailabilityZone2.ZoneName + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-public-b" + + PublicRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-public-rt" + + PublicRoute: + Type: AWS::EC2::Route + Properties: + RouteTableId: !Ref PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: !Ref InternetGateway + + PublicSubnetARouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref PublicSubnetA + RouteTableId: !Ref PublicRouteTable + + PublicSubnetBRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref PublicSubnetB + RouteTableId: !Ref PublicRouteTable + + # ---------- Private subnets (DocumentDB) ---------- + PrivateSubnetA: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: !Ref CidrPrivateSubnetA + AvailabilityZone: !GetAtt AvailabilityZone1.ZoneName + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-private-a" + + PrivateSubnetB: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + CidrBlock: !Ref CidrPrivateSubnetB + AvailabilityZone: !GetAtt AvailabilityZone2.ZoneName + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-private-b" + + PrivateRouteTableA: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-private-rt-a" + + PrivateRouteTableB: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VPC + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-private-rt-b" + + PrivateSubnetARouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref PrivateSubnetA + RouteTableId: !Ref PrivateRouteTableA + + PrivateSubnetBRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref PrivateSubnetB + RouteTableId: !Ref PrivateRouteTableB + + # ---------- Security groups ---------- + ALBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: ALB - allow HTTP/HTTPS from internet + VpcId: !Ref VPC + SecurityGroupIngress: + - CidrIp: 0.0.0.0/0 + IpProtocol: tcp + FromPort: 80 + ToPort: 80 + - CidrIp: 0.0.0.0/0 + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-alb-sg" + + ECSSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: ECS tasks - allow 8000 from ALB, all egress + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 8000 + ToPort: 8000 + SourceSecurityGroupId: !Ref ALBSecurityGroup + SecurityGroupEgress: + - CidrIp: 0.0.0.0/0 + IpProtocol: "-1" + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-ecs-sg" + + DocumentDBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: DocumentDB - allow 27017 from ECS only + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 27017 + ToPort: 27017 + SourceSecurityGroupId: !Ref ECSSecurityGroup + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-docdb-sg" + + # ---------- DocumentDB subnet group ---------- + DocumentDBSubnetGroup: + Type: AWS::DocDB::DBSubnetGroup + Properties: + DBSubnetGroupDescription: Private subnets for DocumentDB + SubnetIds: + - !Ref PrivateSubnetA + - !Ref PrivateSubnetB + Tags: + - Key: Name + Value: !Sub "${AWS::StackName}-docdb-subnet-group" + + # ---------- S3 VPC endpoint ---------- + S3Endpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + VpcEndpointType: Gateway + ServiceName: !Sub "com.amazonaws.${AWS::Region}.s3" + VpcId: !Ref VPC + RouteTableIds: + - !Ref PublicRouteTable + - !Ref PrivateRouteTableA + - !Ref PrivateRouteTableB + +Outputs: + VpcId: + Value: !Ref VPC + Export: + Name: !Sub "${AWS::StackName}-VpcId" + PublicSubnetIds: + Value: !Join [",", [!Ref PublicSubnetA, !Ref PublicSubnetB]] + Export: + Name: !Sub "${AWS::StackName}-PublicSubnetIds" + PrivateSubnetIds: + Value: !Join [",", [!Ref PrivateSubnetA, !Ref PrivateSubnetB]] + Export: + Name: !Sub "${AWS::StackName}-PrivateSubnetIds" + ALBSecurityGroupId: + Value: !GetAtt ALBSecurityGroup.GroupId + Export: + Name: !Sub "${AWS::StackName}-ALBSecurityGroupId" + ECSSecurityGroupId: + Value: !GetAtt ECSSecurityGroup.GroupId + Export: + Name: !Sub "${AWS::StackName}-ECSSecurityGroupId" + DocumentDBSecurityGroupId: + Value: !GetAtt DocumentDBSecurityGroup.GroupId + Export: + Name: !Sub "${AWS::StackName}-DocumentDBSecurityGroupId" + DocumentDBSubnetGroupName: + Value: !Ref DocumentDBSubnetGroup + Export: + Name: !Sub "${AWS::StackName}-DocumentDBSubnetGroupName" diff --git a/docker/mongodb/mongod-tls.conf b/docker/mongodb/mongod-tls.conf deleted file mode 100644 index cb22669..0000000 --- a/docker/mongodb/mongod-tls.conf +++ /dev/null @@ -1,17 +0,0 @@ -# MongoDB configuration for TLS/X.509 authentication -# Used in production mode when certificates are mounted - -net: - port: 27017 - bindIp: 0.0.0.0 - tls: - mode: requireTLS - certificateKeyFile: /etc/mongodb/certs/server-bundle.pem - CAFile: /etc/mongodb/certs/ca.pem - allowConnectionsWithoutCertificates: true - -security: - authorization: enabled - -setParameter: - authenticationMechanisms: MONGODB-X509 diff --git a/materialize/Cargo.toml b/materialize/Cargo.toml index 94bb624..0586c53 100644 --- a/materialize/Cargo.toml +++ b/materialize/Cargo.toml @@ -12,6 +12,7 @@ rayon = "1" indicatif = "0.17" anyhow = "1" clap = { version = "4", features = ["derive"] } +regex = "1" [profile.release] lto = true diff --git a/materialize/src/main.rs b/materialize/src/main.rs index d365adc..bd97c7d 100644 --- a/materialize/src/main.rs +++ b/materialize/src/main.rs @@ -2,7 +2,7 @@ use anyhow::Result; use bson::{doc, Document}; use clap::Parser; use indicatif::{ProgressBar, ProgressStyle}; -use mongodb::options::{AuthMechanism, ClientOptions, Credential, TlsOptions}; +use mongodb::options::{ClientOptions, TlsOptions}; use mongodb::sync::{Client, Collection}; use rayon::prelude::*; use std::collections::HashMap; @@ -54,40 +54,43 @@ struct LookupTables { subject_in_collection: MultiMap, } -/// Create MongoDB client with optional TLS/X.509 authentication. +/// Create MongoDB client with optional TLS authentication. +/// When TLS is enabled, SCRAM credentials are parsed from the URI automatically. fn create_mongodb_client() -> Result { let uri = env::var("DATABASE_URL").unwrap_or_else(|_| "mongodb://localhost:27017".to_string()); let tls_enabled = env::var("MONGODB_TLS_ENABLED") .map(|v| v.to_lowercase() == "true") .unwrap_or(false); + let retry_writes = env::var("MONGODB_RETRY_WRITES") + .map(|v| v.to_lowercase() == "true") + .unwrap_or(true); + + let mut options = ClientOptions::parse(&uri).run()?; + + if !retry_writes { + options.retry_writes = Some(false); + } if tls_enabled { - let cert_path = env::var("MONGODB_CERT_PATH") - .unwrap_or_else(|_| "/etc/cfdb/certs/client-bundle.pem".to_string()); let ca_path = env::var("MONGODB_CA_PATH") - .unwrap_or_else(|_| "/etc/cfdb/certs/ca.pem".to_string()); + .unwrap_or_else(|_| "/etc/cfdb/certs/global-bundle.pem".to_string()); - println!("Connecting to MongoDB at {} with X.509 authentication", uri); - - let mut options = ClientOptions::parse(&uri).run()?; + // Redact password from URI for logging + let redacted = regex::Regex::new(r"://([^:]+):([^@]+)@") + .unwrap() + .replace(&uri, "://$1:***@"); + println!("Connecting to MongoDB at {} with TLS", redacted); let tls_options = TlsOptions::builder() .ca_file_path(Some(PathBuf::from(ca_path))) - .cert_key_file_path(Some(PathBuf::from(cert_path))) .build(); options.tls = Some(mongodb::options::Tls::Enabled(tls_options)); - options.credential = Some( - Credential::builder() - .mechanism(AuthMechanism::MongoDbX509) - .source(Some("$external".to_string())) - .build(), - ); Ok(Client::with_options(options)?) } else { println!("Connecting to MongoDB at {} (no authentication)", uri); - Ok(Client::with_uri_str(&uri)?) + Ok(Client::with_options(options)?) } } diff --git a/scripts/create-x509-users.js b/scripts/create-x509-users.js deleted file mode 100644 index dddee62..0000000 --- a/scripts/create-x509-users.js +++ /dev/null @@ -1,29 +0,0 @@ -// Create X.509 authenticated users for MongoDB -// Run this script after MongoDB starts with TLS enabled - -// Switch to $external database (required for X.509 authentication) -db = db.getSiblingDB('$external'); - -// Create API client user -// Subject DN must match exactly as MongoDB reads it from the certificate -// MongoDB reads the DN in RFC 2253 order (reversed): C, O, OU, CN -db.createUser({ - user: "C=US,O=Abdenlab-Clients,OU=Clients,CN=cfdb-api", - roles: [ - { role: "readWrite", db: "cfdb" } - ] -}); -print("Created X.509 user for API client"); - -// Create Materializer client user -// Needs additional dbAdmin role for creating indexes -db.createUser({ - user: "C=US,O=Abdenlab-Clients,OU=Clients,CN=cfdb-materializer", - roles: [ - { role: "readWrite", db: "cfdb" }, - { role: "dbAdmin", db: "cfdb" } - ] -}); -print("Created X.509 user for Materializer client"); - -print("X.509 user setup complete"); diff --git a/src/cfdb/api/__init__.py b/src/cfdb/api/__init__.py index 478915f..34c04e5 100644 --- a/src/cfdb/api/__init__.py +++ b/src/cfdb/api/__init__.py @@ -7,12 +7,12 @@ DATABASE_NAME: Final = os.getenv("DATABASE_NAME", "cfdb") PAGE_SIZE: Final = 25 -# TLS/X.509 authentication configuration (production) +# TLS authentication configuration (production) MONGODB_TLS_ENABLED: Final = os.getenv("MONGODB_TLS_ENABLED", "false").lower() == "true" -MONGODB_CERT_PATH: Final = os.getenv( - "MONGODB_CERT_PATH", "/etc/cfdb/certs/client-bundle.pem" +MONGODB_CA_PATH: Final = os.getenv( + "MONGODB_CA_PATH", "/etc/cfdb/certs/global-bundle.pem" ) -MONGODB_CA_PATH: Final = os.getenv("MONGODB_CA_PATH", "/etc/cfdb/certs/ca.pem") +MONGODB_RETRY_WRITES: Final = os.getenv("MONGODB_RETRY_WRITES", "true").lower() == "true" # Sync API authentication SYNC_API_KEY: Final = os.getenv("SYNC_API_KEY", "") diff --git a/src/cfdb/api/main.py b/src/cfdb/api/main.py index d902ab9..c3fe656 100644 --- a/src/cfdb/api/main.py +++ b/src/cfdb/api/main.py @@ -1,7 +1,9 @@ import logging +import re from contextlib import asynccontextmanager from fastapi import FastAPI +from fastapi.responses import JSONResponse from motor.motor_asyncio import AsyncIOMotorClient from strawberry.fastapi import GraphQLRouter @@ -14,20 +16,28 @@ logging.basicConfig(level=logging.INFO) +def redact_url(url: str) -> str: + """Redact password from a MongoDB connection string for safe logging.""" + return re.sub(r"://([^:]+):([^@]+)@", r"://\1:***@", url) + + def create_mongodb_client() -> AsyncIOMotorClient: - """Create MongoDB client with optional TLS/X.509 authentication.""" + """Create MongoDB client with optional TLS authentication.""" + kwargs: dict = {} + + if not api.MONGODB_RETRY_WRITES: + kwargs["retryWrites"] = False + if api.MONGODB_TLS_ENABLED: - print(f"Connecting to MongoDB at {api.DATABASE_URL} with X.509 authentication") + print(f"Connecting to MongoDB at {redact_url(api.DATABASE_URL)} with TLS") return AsyncIOMotorClient( api.DATABASE_URL, - authMechanism="MONGODB-X509", tls=True, - tlsCertificateKeyFile=api.MONGODB_CERT_PATH, tlsCAFile=api.MONGODB_CA_PATH, - authSource="$external", + **kwargs, ) print(f"Connecting to MongoDB at {api.DATABASE_URL} (no authentication)") - return AsyncIOMotorClient(api.DATABASE_URL) + return AsyncIOMotorClient(api.DATABASE_URL, **kwargs) @asynccontextmanager @@ -48,3 +58,8 @@ async def lifespan(_: FastAPI): app.include_router(data_router) app.include_router(index_router) app.include_router(sync_router) + + +@app.get("/health") +async def health(): + return JSONResponse({"status": "ok"})