From b721a9a6036336f66215051fde2881c27da1907f Mon Sep 17 00:00:00 2001 From: samchungy Date: Wed, 2 Oct 2024 18:02:53 +1000 Subject: [PATCH] further alignment --- template/lambda-sqs-worker-cdk/README.md | 145 ++++++++++++++++++ .../infra/__snapshots__/appStack.test.ts.snap | 134 ---------------- .../lambda-sqs-worker-cdk/infra/appStack.ts | 19 ++- template/lambda-sqs-worker/serverless.yml | 2 +- 4 files changed, 157 insertions(+), 143 deletions(-) create mode 100644 template/lambda-sqs-worker-cdk/README.md diff --git a/template/lambda-sqs-worker-cdk/README.md b/template/lambda-sqs-worker-cdk/README.md new file mode 100644 index 000000000..5db50567e --- /dev/null +++ b/template/lambda-sqs-worker-cdk/README.md @@ -0,0 +1,145 @@ +# <%- repoName %> + +[![Powered by skuba](https://img.shields.io/badge/🤿%20skuba-powered-009DC4)](https://github.com/seek-oss/skuba) + +Next steps: + +1. [ ] Finish templating if this was skipped earlier: + + ```shell + pnpm exec skuba configure + ``` + +2. [ ] Create a new repository in the appropriate GitHub organisation. +3. [ ] Add the repository to BuildAgency; + see our internal [Buildkite Docs] for more information. +4. [ ] Add Datadog extension, deployment bucket configuration and data classification tags to [infra/config.ts](infra/config.ts). +5. [ ] Push local commits to the upstream GitHub branch. +6. [ ] Configure [GitHub repository settings]. +7. [ ] Delete this checklist 😌. + +[Buildkite Docs]: https://backstage.myseek.xyz/docs/default/component/buildkite-docs +[GitHub repository settings]: https://github.com/<%-orgName%>/<%-repoName%>/settings + +## Design + +<%-repoName %> is a Node.js [Lambda] application built in line with our [Technical Guidelines]. +It is backed by a typical SQS message + dead letter queue configuration and uses common SEEK packages. +Workers enable fault-tolerant asynchronous processing of events. + +The `lambda-sqs-worker-cdk` template is modelled after a hypothetical enricher that scores job advertisements. +It's stubbed out with in-memory [scoring service](src/services/jobScorer.ts). +This would be replaced with internal logic or an external service in production. + +This project is deployed with [AWS CDK]. +The Lambda runtime provisions a single Node.js process per container. +The supplied [infra/appStack.ts](infra/appStack.ts) starts out with a minimal `memorySize` which may require tuning based on workload. +Under load, we autoscale horizontally in terms of container count up to `reservedConcurrency`. + +[@seek/aws-codedeploy-hooks] configures [CodeDeploy] for a blue-green deployment approach. +A smoke test is run against the new version before traffic is switched over, +providing an opportunity to test access and connectivity to online dependencies. +This defaults to an invocation with an empty object `{}`. + +## Development + +### Test + +```shell +# Run Jest tests locally +pnpm test + +# Authenticate to dev account +awsauth + +# Run smoke test against deployed application +ENVIRONMENT=dev pnpm smoke +``` + +### Lint + +```shell +# Fix issues +pnpm format + +# Check for issues +pnpm lint +``` + +### Start + +```shell +# Start a local HTTP server +pnpm start + +# Start with Node.js Inspector enabled +pnpm start:debug +``` + +This serves the Lambda application over HTTP. +For example, to invoke the handler with an empty object `{}` for smoke testing: + +```shell +curl --data '[{}, {"awsRequestId": "local"}]' --include localhost:<%- port %> +``` + +### Deploy + +This project is deployed through a [Buildkite pipeline](.buildkite/pipeline.yml). + +- Commits to a feature branch can be deployed to the dev environment by unblocking a step in the Buildkite UI +- Commits to the default branch are automatically deployed to the dev and prod environments in sequence + +To deploy locally: + +```shell +# Authenticate to dev account +awsauth + +ENVIRONMENT=dev pnpm run deploy +``` + +A hotswap deploy enables faster deployment but come with caveats such as requiring a Lambda to be rebuilt with every build. + +To deploy a [hotswap]: + +```shell +# Authenticate to dev account +awsauth + +ENVIRONMENT=dev pnpm run deploy:hotswap +``` + +To rapidly roll back a change, +retry an individual deployment step from the previous build in Buildkite. +Note that this will introduce drift between the head of the default Git branch and the live environment; +use with caution and always follow up with a proper revert or fix in Git history. + +## Support + +### Dev + +TODO: add support links for the dev environment. + + + +### Prod + +TODO: add support links for the prod environment. + + + +[@seek/aws-codedeploy-hooks]: https://github.com/seek-oss/aws-codedeploy-hooks +[AWS CDK]: https://docs.aws.amazon.com/cdk/v2/guide/home.html +[CodeDeploy]: https://docs.aws.amazon.com/codedeploy +[Hotswap]: https://docs.aws.amazon.com/cdk/v2/guide/ref-cli-cmd-deploy.html#ref-cli-cmd-deploy-options +[Lambda]: https://docs.aws.amazon.com/lambda +[Technical Guidelines]: https://myseek.atlassian.net/wiki/spaces/AA/pages/2358346017/ diff --git a/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap b/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap index d04490990..5d784a5fa 100644 --- a/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap +++ b/template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap @@ -139,17 +139,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = ` }, "Resource": "*", }, - { - "Action": [ - "kms:Decrypt", - "kms:GenerateDataKey", - ], - "Effect": "Allow", - "Principal": { - "Service": "sns.amazonaws.com", - }, - "Resource": "*", - }, ], "Version": "2012-10-17", }, @@ -169,9 +158,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = ` }, "Type": "AWS::KMS::Alias", }, - "sourcetopic7C3DC892": { - "Type": "AWS::SNS::Topic", - }, "worker28EA3E30": { "DependsOn": [ "workerServiceRoleDefaultPolicyBA498553", @@ -662,59 +648,6 @@ exports[`returns expected CloudFormation stack for dev 1`] = ` "Type": "AWS::SQS::Queue", "UpdateReplacePolicy": "Delete", }, - "workerqueuePolicy97054CB4": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "sqs:SendMessage", - "Condition": { - "ArnEquals": { - "aws:SourceArn": { - "Ref": "sourcetopic7C3DC892", - }, - }, - }, - "Effect": "Allow", - "Principal": { - "Service": "sns.amazonaws.com", - }, - "Resource": { - "Fn::GetAtt": [ - "workerqueueA05CE5C6", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "Queues": [ - { - "Ref": "workerqueueA05CE5C6", - }, - ], - }, - "Type": "AWS::SQS::QueuePolicy", - }, - "workerqueueappStacksourcetopic613C6BDBD2F224F5": { - "DependsOn": [ - "workerqueuePolicy97054CB4", - ], - "Properties": { - "Endpoint": { - "Fn::GetAtt": [ - "workerqueueA05CE5C6", - "Arn", - ], - }, - "Protocol": "sqs", - "TopicArn": { - "Ref": "sourcetopic7C3DC892", - }, - }, - "Type": "AWS::SNS::Subscription", - }, "workerqueuedeadletters83F3505C": { "DeletionPolicy": "Delete", "Properties": { @@ -899,17 +832,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = ` }, "Resource": "*", }, - { - "Action": [ - "kms:Decrypt", - "kms:GenerateDataKey", - ], - "Effect": "Allow", - "Principal": { - "Service": "sns.amazonaws.com", - }, - "Resource": "*", - }, ], "Version": "2012-10-17", }, @@ -929,9 +851,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = ` }, "Type": "AWS::KMS::Alias", }, - "sourcetopic7C3DC892": { - "Type": "AWS::SNS::Topic", - }, "worker28EA3E30": { "DependsOn": [ "workerServiceRoleDefaultPolicyBA498553", @@ -1422,59 +1341,6 @@ exports[`returns expected CloudFormation stack for prod 1`] = ` "Type": "AWS::SQS::Queue", "UpdateReplacePolicy": "Delete", }, - "workerqueuePolicy97054CB4": { - "Properties": { - "PolicyDocument": { - "Statement": [ - { - "Action": "sqs:SendMessage", - "Condition": { - "ArnEquals": { - "aws:SourceArn": { - "Ref": "sourcetopic7C3DC892", - }, - }, - }, - "Effect": "Allow", - "Principal": { - "Service": "sns.amazonaws.com", - }, - "Resource": { - "Fn::GetAtt": [ - "workerqueueA05CE5C6", - "Arn", - ], - }, - }, - ], - "Version": "2012-10-17", - }, - "Queues": [ - { - "Ref": "workerqueueA05CE5C6", - }, - ], - }, - "Type": "AWS::SQS::QueuePolicy", - }, - "workerqueueappStacksourcetopic613C6BDBD2F224F5": { - "DependsOn": [ - "workerqueuePolicy97054CB4", - ], - "Properties": { - "Endpoint": { - "Fn::GetAtt": [ - "workerqueueA05CE5C6", - "Arn", - ], - }, - "Protocol": "sqs", - "TopicArn": { - "Ref": "sourcetopic7C3DC892", - }, - }, - "Type": "AWS::SNS::Subscription", - }, "workerqueuedeadletters83F3505C": { "DeletionPolicy": "Delete", "Properties": { diff --git a/template/lambda-sqs-worker-cdk/infra/appStack.ts b/template/lambda-sqs-worker-cdk/infra/appStack.ts index dac55101a..ab55d9c1c 100644 --- a/template/lambda-sqs-worker-cdk/infra/appStack.ts +++ b/template/lambda-sqs-worker-cdk/infra/appStack.ts @@ -10,7 +10,6 @@ import { aws_lambda_nodejs, aws_secretsmanager, aws_sns, - aws_sns_subscriptions, aws_sqs, } from 'aws-cdk-lib'; import type { Construct } from 'constructs'; @@ -51,13 +50,17 @@ export class AppStack extends Stack { encryptionMasterKey: kmsKey, }); - const topic = aws_sns.Topic.fromTopicArn( - this, - 'source-topic', - config.sourceSnsTopicArn, - ); - - topic.addSubscription(new aws_sns_subscriptions.SqsSubscription(queue)); + // const topic = aws_sns.Topic.fromTopicArn( + // this, + // 'source-topic', + // config.sourceSnsTopicArn, + // ); + + // topic.addSubscription( + // new aws_sns_subscriptions.SqsSubscription(queue, { + // rawMessageDelivery: true, // Remove this property if you require end to end datadog tracing + // }), + // ); const snsKey = aws_kms.Alias.fromAliasName( this, diff --git a/template/lambda-sqs-worker/serverless.yml b/template/lambda-sqs-worker/serverless.yml index 0b57fc07c..4e9020217 100644 --- a/template/lambda-sqs-worker/serverless.yml +++ b/template/lambda-sqs-worker/serverless.yml @@ -173,7 +173,7 @@ resources: # Properties: # Endpoint: !GetAtt MessageQueue.Arn # Protocol: sqs - # RawMessageDelivery: true + # RawMessageDelivery: true # Remove this property if you require end to end datadog tracing # TopicArn: 'TODO: sourceSnsTopicArn' DestinationTopic: