Skip to content

Commit

Permalink
Add alarm on missing specific types of notifications
Browse files Browse the repository at this point in the history
  • Loading branch information
@waisingyiu
waisingyiu committed Oct 9, 2024
1 parent 000e4f5 commit de0f307
Show file tree
Hide file tree
Showing 2 changed files with 283 additions and 6 deletions.
228 changes: 228 additions & 0 deletions notificationworkerlambda/cdk/lib/__snapshots__/senderworker.test.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,63 @@ exports[`The Sender Worker stack matches the snapshot 1`] = `
},
"Type": "AWS::CloudWatch::Alarm",
},
"androidSenderTooFewBreakingNewsAlarmEE8C7099": {
"Properties": {
"ActionsEnabled": true,
"AlarmActions": [
{
"Ref": "AlarmTopicArn",
},
],
"AlarmDescription": "Triggers if the android sender lambda is not frequently invoked for breaking news notification in PROD.",
"ComparisonOperator": "LessThanOrEqualToThreshold",
"Dimensions": [
{
"Name": "platform",
"Value": "android",
},
{
"Name": "type",
"Value": "breakingNews",
},
],
"EvaluationPeriods": 24,
"MetricName": "worker.notificationProcessingTime",
"Namespace": "Notifications/PROD/workers",
"OKActions": [
{
"Ref": "AlarmTopicArn",
},
],
"Period": 900,
"Statistic": "SampleCount",
"Tags": [
{
"Key": "App",
"Value": "android",
},
{
"Key": "gu:cdk:version",
"Value": "59.2.3",
},
{
"Key": "gu:repo",
"Value": "guardian/mobile-n10n",
},
{
"Key": "Stack",
"Value": "mobile-notifications-workers",
},
{
"Key": "Stage",
"Value": "PROD",
},
],
"Threshold": 0,
"TreatMissingData": "breaching",
},
"Type": "AWS::CloudWatch::Alarm",
},
"androidSenderTooFewInvocationsAlarmC306ECD5": {
"Properties": {
"ActionsEnabled": true,
Expand Down Expand Up @@ -609,6 +666,63 @@ exports[`The Sender Worker stack matches the snapshot 1`] = `
},
"Type": "AWS::CloudWatch::Alarm",
},
"androidSenderTooFewNonBreakingAlarm35FD6EBB": {
"Properties": {
"ActionsEnabled": true,
"AlarmActions": [
{
"Ref": "AlarmTopicArn",
},
],
"AlarmDescription": "Triggers if the android sender lambda is not frequently invoked for non-breaking news notification in PROD.",
"ComparisonOperator": "LessThanOrEqualToThreshold",
"Dimensions": [
{
"Name": "platform",
"Value": "android",
},
{
"Name": "type",
"Value": "other",
},
],
"EvaluationPeriods": 4,
"MetricName": "worker.notificationProcessingTime",
"Namespace": "Notifications/PROD/workers",
"OKActions": [
{
"Ref": "AlarmTopicArn",
},
],
"Period": 900,
"Statistic": "SampleCount",
"Tags": [
{
"Key": "App",
"Value": "android",
},
{
"Key": "gu:cdk:version",
"Value": "59.2.3",
},
{
"Key": "gu:repo",
"Value": "guardian/mobile-n10n",
},
{
"Key": "Stack",
"Value": "mobile-notifications-workers",
},
{
"Key": "Stage",
"Value": "PROD",
},
],
"Threshold": 0,
"TreatMissingData": "breaching",
},
"Type": "AWS::CloudWatch::Alarm",
},
"androidbetaExecutionRoleFA113395": {
"Properties": {
"AssumeRolePolicyDocument": {
Expand Down Expand Up @@ -2162,6 +2276,63 @@ exports[`The Sender Worker stack matches the snapshot 1`] = `
},
"Type": "AWS::CloudWatch::Alarm",
},
"iosSenderTooFewBreakingNewsAlarm870D69CB": {
"Properties": {
"ActionsEnabled": true,
"AlarmActions": [
{
"Ref": "AlarmTopicArn",
},
],
"AlarmDescription": "Triggers if the ios sender lambda is not frequently invoked for breaking news notification in PROD.",
"ComparisonOperator": "LessThanOrEqualToThreshold",
"Dimensions": [
{
"Name": "platform",
"Value": "ios",
},
{
"Name": "type",
"Value": "breakingNews",
},
],
"EvaluationPeriods": 24,
"MetricName": "worker.notificationProcessingTime",
"Namespace": "Notifications/PROD/workers",
"OKActions": [
{
"Ref": "AlarmTopicArn",
},
],
"Period": 900,
"Statistic": "SampleCount",
"Tags": [
{
"Key": "App",
"Value": "ios",
},
{
"Key": "gu:cdk:version",
"Value": "59.2.3",
},
{
"Key": "gu:repo",
"Value": "guardian/mobile-n10n",
},
{
"Key": "Stack",
"Value": "mobile-notifications-workers",
},
{
"Key": "Stage",
"Value": "PROD",
},
],
"Threshold": 0,
"TreatMissingData": "breaching",
},
"Type": "AWS::CloudWatch::Alarm",
},
"iosSenderTooFewInvocationsAlarm14117FE9": {
"Properties": {
"ActionsEnabled": true,
Expand Down Expand Up @@ -2217,6 +2388,63 @@ exports[`The Sender Worker stack matches the snapshot 1`] = `
},
"Type": "AWS::CloudWatch::Alarm",
},
"iosSenderTooFewNonBreakingAlarmC2075137": {
"Properties": {
"ActionsEnabled": true,
"AlarmActions": [
{
"Ref": "AlarmTopicArn",
},
],
"AlarmDescription": "Triggers if the ios sender lambda is not frequently invoked for non-breaking news notification in PROD.",
"ComparisonOperator": "LessThanOrEqualToThreshold",
"Dimensions": [
{
"Name": "platform",
"Value": "ios",
},
{
"Name": "type",
"Value": "other",
},
],
"EvaluationPeriods": 4,
"MetricName": "worker.notificationProcessingTime",
"Namespace": "Notifications/PROD/workers",
"OKActions": [
{
"Ref": "AlarmTopicArn",
},
],
"Period": 900,
"Statistic": "SampleCount",
"Tags": [
{
"Key": "App",
"Value": "ios",
},
{
"Key": "gu:cdk:version",
"Value": "59.2.3",
},
{
"Key": "gu:repo",
"Value": "guardian/mobile-n10n",
},
{
"Key": "Stack",
"Value": "mobile-notifications-workers",
},
{
"Key": "Stage",
"Value": "PROD",
},
],
"Threshold": 0,
"TreatMissingData": "breaching",
},
"Type": "AWS::CloudWatch::Alarm",
},
"ioseditionExecutionRole61247F06": {
"Properties": {
"AssumeRolePolicyDocument": {
Expand Down
61 changes: 55 additions & 6 deletions notificationworkerlambda/cdk/lib/senderworker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import * as cdkcore from 'constructs'

import {SnsAction} from 'aws-cdk-lib/aws-cloudwatch-actions'
import { GuStack } from "@guardian/cdk/lib/constructs/core"
import type { App } from "aws-cdk-lib"
import { Duration, type App } from "aws-cdk-lib"
import type { GuStackProps } from "@guardian/cdk/lib/constructs/core"
import { Metric } from 'aws-cdk-lib/aws-cloudwatch'

interface SenderWorkerOpts {
handler: string,
Expand All @@ -25,7 +26,8 @@ interface SenderWorkerOpts {
platform: string,
paramPrefix: string,
isBatchingSqsMessages: boolean,
dailyAlarmPeriod: boolean
dailyAlarmPeriod: boolean,
tooFewNotificationByTypeAlarms: boolean,
}

class SenderWorker extends cdkcore.Construct {
Expand Down Expand Up @@ -161,6 +163,52 @@ class SenderWorker extends cdkcore.Construct {
senderTooFewInvocationsAlarm.addAlarmAction(snsTopicAction)
senderTooFewInvocationsAlarm.addOkAction(snsTopicAction)

if (props.tooFewNotificationByTypeAlarms) {
const nonBreakingCountMetric = new Metric({
namespace: `Notifications/${scope.stage}/workers`,
metricName: "worker.notificationProcessingTime",
period: Duration.minutes(15),
statistic: "SampleCount",
dimensionsMap: {
platform: id,
type: "other",
},
});
const senderTooFewNonBreakingAlarm = new cloudwatch.Alarm(this, 'SenderTooFewNonBreakingAlarm', {
alarmDescription: `Triggers if the ${id} sender lambda is not frequently invoked for non-breaking news notification in ${scope.stage}.`,
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
evaluationPeriods: 4,
threshold: 0,
metric: nonBreakingCountMetric,
treatMissingData: cloudwatch.TreatMissingData.BREACHING,
actionsEnabled: (scope.stage === 'PROD'),
})
senderTooFewNonBreakingAlarm.addAlarmAction(snsTopicAction)
senderTooFewNonBreakingAlarm.addOkAction(snsTopicAction)

const breakingNewsCountMetric = new Metric({
namespace: `Notifications/${scope.stage}/workers`,
metricName: "worker.notificationProcessingTime",
period: Duration.minutes(15),
statistic: "SampleCount",
dimensionsMap: {
platform: id,
type: "breakingNews",
},
});
const senderTooFewBreakingNewsAlarm = new cloudwatch.Alarm(this, 'SenderTooFewBreakingNewsAlarm', {
alarmDescription: `Triggers if the ${id} sender lambda is not frequently invoked for breaking news notification in ${scope.stage}.`,
comparisonOperator: cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD,
evaluationPeriods: 24,
threshold: 0,
metric: breakingNewsCountMetric,
treatMissingData: cloudwatch.TreatMissingData.BREACHING,
actionsEnabled: (scope.stage === 'PROD'),
})
senderTooFewBreakingNewsAlarm.addAlarmAction(snsTopicAction)
senderTooFewBreakingNewsAlarm.addOkAction(snsTopicAction)
}

// this advertises the name of the sender queue to the harvester app
new ssm.StringParameter(this, 'SenderQueueSSMParameter', {
parameterName: `/notifications/${scope.stage}/workers/harvester/${props.paramPrefix}SqsCdkUrl`,
Expand Down Expand Up @@ -219,15 +267,16 @@ export class SenderWorkerStack extends GuStack {

let workerQueueArns: string[] = []

const addWorker = (workerName: string, paramPrefix: string, handler: string, isBatchingSqsMessages: boolean = false, dailyAlarmPeriod: boolean = false) => {
const addWorker = (workerName: string, paramPrefix: string, handler: string, isBatchingSqsMessages: boolean = false, dailyAlarmPeriod: boolean = false, tooFewNotificationByTypeAlarms: boolean = false) => {
let worker = new SenderWorker(this, workerName, {
...props,
platform: workerName,
paramPrefix: paramPrefix,
handler: handler,
isBatchingSqsMessages,
...sharedOpts,
dailyAlarmPeriod: dailyAlarmPeriod
dailyAlarmPeriod: dailyAlarmPeriod,
tooFewNotificationByTypeAlarms: tooFewNotificationByTypeAlarms,
})
workerQueueArns.push(worker.senderSqs.queueArn)
}
Expand All @@ -237,8 +286,8 @@ export class SenderWorkerStack extends GuStack {
* platform or app by talking to a different lambda handler function
*/

addWorker("ios", "iosLive", "com.gu.notifications.worker.IOSSender::handleChunkTokens")
addWorker("android", "androidLive", "com.gu.notifications.worker.AndroidSender::handleChunkTokens", true)
addWorker("ios", "iosLive", "com.gu.notifications.worker.IOSSender::handleChunkTokens", false, false, true)
addWorker("android", "androidLive", "com.gu.notifications.worker.AndroidSender::handleChunkTokens", true, false, true)
// edition apps only send one notification a day in order to get content for that day
addWorker("ios-edition", "iosEdition", "com.gu.notifications.worker.IOSSender::handleChunkTokens", false, true)
addWorker("android-edition", "androidEdition", "com.gu.notifications.worker.AndroidSender::handleChunkTokens", false, true)
Expand Down

0 comments on commit de0f307

Please sign in to comment.