Create quality checker lambda

Bump up versions

Begin adding quality checker values and updating variables

Bump up requirements

Improve variable names and use terraform jsonencode

Update more terraform

Add serverless cron

Add more dos searching

Add base level reporting functions

Reduce code duplication and log on z code incorrectly profiled

Start unit tests

Increase unit test coverage

Add schedule and deployment changes

Add job complete and incomplete alarms

Fix incorrect alarm threshold

.coveragerc

@@ -5,4 +5,5 @@ omit =
   tests/*
   **/tests/*
   application/dos_db_handler/*.py
+  **/conftest.py
 branch = True

.github/workflows/dependency-review.yml

@@ -1,4 +1,4 @@
-name: 'Dependency Review'
+name: "Dependency Review"
 on: [pull_request]
 
 permissions:
@@ -8,7 +8,7 @@ jobs:
   dependency-review:
     runs-on: ubuntu-latest
     steps:
-      - name: 'Checkout Repository'
+      - name: "Checkout Repository"
         uses: actions/checkout@v4
-      - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v2
+      - name: "Dependency Review"
+        uses: actions/dependency-review-action@v3

.github/workflows/pull-request.yml

@@ -4,9 +4,9 @@ on:
   pull_request:
     types: [opened, edited, synchronize, reopened]
     branches-ignore:
-      - 'dependabot/**'
-      - 'release/**'
-      - 'main'
+      - "dependabot/**"
+      - "release/**"
+      - "main"
 
 permissions:
   contents: read

.github/workflows/update-pull-request-description.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened]
     branches-ignore:
-      - 'dependabot/**'
+      - "dependabot/**"
 
 permissions:
   contents: read
@@ -38,6 +38,6 @@ jobs:
       - uses: tzkhan/pr-update-action@v2
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
-          head-branch-regex: '${{ env.BRANCH_DESIGNATOR }}/.*'
+          head-branch-regex: "${{ env.BRANCH_DESIGNATOR }}/.*"
           body-template: ${{ steps.template.outputs.result }}
-          body-update-action: 'replace'
+          body-update-action: "replace"

Makefile

@@ -106,13 +106,15 @@ UNIT_TEST_ARGS=" \
 		-e POWERTOOLS_LOG_DEDUPLICATION_DISABLED="1" \
 		--volume $(APPLICATION_DIR)/common:/tmp/.packages/common \
 		--volume $(APPLICATION_DIR)/change_event_dlq_handler:/tmp/.packages/change_event_dlq_handler \
+		--volume $(APPLICATION_DIR)/dos_db_handler:/tmp/.packages/dos_db_handler \
 		--volume $(APPLICATION_DIR)/dos_db_update_dlq_handler:/tmp/.packages/dos_db_update_dlq_handler \
 		--volume $(APPLICATION_DIR)/event_replay:/tmp/.packages/event_replay \
 		--volume $(APPLICATION_DIR)/ingest_change_event:/tmp/.packages/ingest_change_event \
 		--volume $(APPLICATION_DIR)/send_email:/tmp/.packages/send_email \
 		--volume $(APPLICATION_DIR)/service_matcher:/tmp/.packages/service_matcher \
 		--volume $(APPLICATION_DIR)/service_sync:/tmp/.packages/service_sync \
 		--volume $(APPLICATION_DIR)/slack_messenger:/tmp/.packages/slack_messenger \
+		--volume $(APPLICATION_DIR)/quality_checker:/tmp/.packages/quality_checker \
 		"
 
 integration-test-autoflags-no-logs: # End to end test DI project - mandatory: PROFILE; optional: ENVIRONMENT, PARALLEL_TEST_COUNT
@@ -169,58 +171,64 @@ remove-development-environments: # Removes development environments - mandatory:
 	done
 
 # ==============================================================================
-# Service Sync
+# Change Event Dead Letter Queue Handler (change-event-dlq-handler)
 
-service-sync-build-and-deploy: ### Build and deploy service sync lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=service-sync
+change-event-dlq-handler-build-and-deploy: ### Build and deploy change event dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=change-event-dlq-handler
 
 # ==============================================================================
-# Slack Messenger
+# DoS DB Update Dead Letter Queue Handler (dos-db-update-dlq-handler)
 
-slack-messenger-build-and-deploy: ### Build and deploy slack messenger lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=slack-messenger
+dos-db-update-dlq-handler-build-and-deploy: ### Build and deploy dos db update dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=dos-db-update-dlq-handler
 
 # ==============================================================================
-# Service Matcher
+# DoS DB Checker Handler (dos-db-handler)
 
-service-matcher-build-and-deploy: ### Build and deploy service matcher lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=service-matcher
+dos-db-handler-build-and-deploy: ### Build and deploy test db checker handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=dos-db-handler
 
 # ==============================================================================
-# Change Event Dead Letter Queue Handler (change-event-dlq-handler)
+# Event Replay lambda (event-replay)
 
-change-event-dlq-handler-build-and-deploy: ### Build and deploy change event dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=change-event-dlq-handler
+event-replay-build-and-deploy: ### Build and deploy event replay lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=event-replay
 
 # ==============================================================================
-# DoS DB Update Dead Letter Queue Handler (dos-db-update-dlq-handler) Nonprod only
+# Ingest Change Event
 
-dos-db-update-dlq-handler-build-and-deploy: ### Build and deploy dos db update dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=dos-db-update-dlq-handler
+ingest-change-event-build-and-deploy: ### Build and deploy ingest change event lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=ingest-change-event
 
 # ==============================================================================
-# Event Replay lambda (event-replay)
+# Send Email
 
-event-replay-build-and-deploy: ### Build and deploy event replay lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=event-replay
+send-email-build-and-deploy: ### Build and deploy send email lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=send-email
 
 # ==============================================================================
-# DoS DB Checker Handler (dos-db-handler)
+# Service Matcher
 
-dos-db-handler-build-and-deploy: ### Build and deploy test db checker handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=dos-db-handler
+service-matcher-build-and-deploy: ### Build and deploy service matcher lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=service-matcher
 
 # ==============================================================================
-# Send Email
+# Service Sync
 
-send-email-build-and-deploy: ### Build and deploy send email lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=send-email
+service-sync-build-and-deploy: ### Build and deploy service sync lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=service-sync
 
 # ==============================================================================
-# Ingest Change Event
+# Slack Messenger
 
-ingest-change-event-build-and-deploy: ### Build and deploy ingest change event lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=ingest-change-event
+slack-messenger-build-and-deploy: ### Build and deploy slack messenger lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=slack-messenger
+
+# ==============================================================================
+# Quality Checker
+
+quality-checker-build-and-deploy: ### Build and deploy quality checker lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=quality-checker
 
 # ==============================================================================
 # Deployments
@@ -250,15 +258,6 @@ push-images: # Use VERSION=[] to push a perticular version otherwise with defaul
 push-tester-image:
 	make docker-push NAME=tester
 
-# ==============================================================================
-# SES (Simple Email Service)
-
-deploy-email: # Deploys SES resources - mandatory: PROFILE=[live/test]
-	make terraform-apply-auto-approve STACKS=email ENVIRONMENT=$(AWS_ACCOUNT_NAME)
-
-undeploy-email: # Deploys SES resources - mandatory: PROFILE=[live/test]
-	make terraform-destroy-auto-approve STACKS=email ENVIRONMENT=$(AWS_ACCOUNT_NAME)
-
 # ==============================================================================
 # Development Tools
 
@@ -399,29 +398,6 @@ performance-test-clean: # Clean up performance test results
 	rm -f $(TMP_DIR)/*.zip
 	rm -rf $(PROJECT_DIR)/test/performance/results/*.csv
 
-stress-test-in-pipeline: # An all in one stress test make target
-	START_TIME=$$(date +%Y-%m-%d_%H-%M-%S)
-	AWS_START_TIME=$$(date +%FT%TZ)
-	CODE_VERSION=$$($(AWSCLI) lambda get-function --function-name $(TF_VAR_service_matcher_lambda_name) | jq --raw-output '.Configuration.Environment.Variables.CODE_VERSION')
-	make stress-test START_TIME=$$START_TIME PIPELINE=true
-	sleep 4.5h
-	END_TIME=$$(date +%Y-%m-%d_%H-%M-%S)
-	AWS_END_TIME=$$(date +%FT%TZ)
-	make send-performance-dashboard-slack-message START_DATE_TIME=$$AWS_START_TIME END_DATE_TIME=$$AWS_END_TIME
-
-load-test-in-pipeline: # An all in one load test make target
-	START_TIME=$$(date +%Y-%m-%d_%H-%M-%S)
-	AWS_START_TIME=$$(date +%FT%TZ)
-	CODE_VERSION=$$($(AWSCLI) lambda get-function --function-name $(TF_VAR_service_matcher_lambda_name) | jq --raw-output '.Configuration.Environment.Variables.CODE_VERSION')
-	make load-test START_TIME=$$START_TIME
-	sleep 10m
-	END_TIME=$$(date +%Y-%m-%d_%H-%M-%S)
-	AWS_END_TIME=$$(date +%FT%TZ)
-	make send-performance-dashboard-slack-message START_DATE_TIME=$$AWS_START_TIME END_DATE_TIME=$$AWS_END_TIME
-
-send-performance-dashboard-slack-message:
-	make slack-codebuild-notification PROFILE=$(PROFILE) ENVIRONMENT=$(ENVIRONMENT) PIPELINE_NAME="$(PERF_TEST_TITLE) Tests Codebuild Stage" CODEBUILD_PROJECT_NAME=$(CB_PROJECT_NAME) CODEBUILD_BUILD_ID=$(CODEBUILD_BUILD_ID) SLACK_MESSAGE="Performance Dashboard Here - https://$(AWS_REGION).console.aws.amazon.com/cloudwatch/home?region=$(AWS_REGION)#dashboards:name=$(TF_VAR_cloudwatch_monitoring_dashboard_name);start=$(START_DATE_TIME);end=$(END_DATE_TIME)"
-
 # -----------------------------
 # Other
 

application/change_event_dlq_handler/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.2.0
+aws-lambda-powertools[tracer] ~= 2.24.0

application/common/service_type.py

@@ -9,8 +9,13 @@
     DOS_CONTRACEPTION_SYMPTOM_DISCRIMINATOR,
     DOS_CONTRACEPTION_SYMPTOM_GROUP,
     DOS_CONTRACEPTION_TYPE_ID,
+    DOS_PALLIATIVE_CARE_SGSDID,
+    DOS_PALLIATIVE_CARE_SYMPTOM_DISCRIMINATOR,
+    DOS_PALLIATIVE_CARE_SYMPTOM_GROUP,
+    DOS_PALLIATIVE_CARE_TYPE_ID,
     NHS_UK_BLOOD_PRESSURE_SERVICE_CODE,
     NHS_UK_CONTRACEPTION_SERVICE_CODE,
+    NHS_UK_PALLIATIVE_CARE_SERVICE_CODE,
 )
 
 
@@ -25,6 +30,29 @@ class ServiceType:
     DOS_SYMPTOM_DISCRIMINATOR: int
     DOS_SG_SD_ID: str
 
+    def __eq__(self, __value: object) -> bool:
+        """Check if the ServiceType is equal to another ServiceType."""
+        try:
+            return (
+                self.TYPE_NAME == __value.TYPE_NAME
+                and self.NHS_UK_SERVICE_CODE == __value.NHS_UK_SERVICE_CODE
+                and self.DOS_TYPE_ID == __value.DOS_TYPE_ID
+                and self.DOS_SYMPTOM_GROUP == __value.DOS_SYMPTOM_GROUP
+                and self.DOS_SYMPTOM_DISCRIMINATOR == __value.DOS_SYMPTOM_DISCRIMINATOR
+                and self.DOS_SG_SD_ID == __value.DOS_SG_SD_ID
+            )
+        except AttributeError:
+            return False
+
+
+PALLIATIVE_CARE = ServiceType(
+    TYPE_NAME="Palliative Care",
+    NHS_UK_SERVICE_CODE=NHS_UK_PALLIATIVE_CARE_SERVICE_CODE,
+    DOS_TYPE_ID=DOS_PALLIATIVE_CARE_TYPE_ID,
+    DOS_SYMPTOM_GROUP=DOS_PALLIATIVE_CARE_SYMPTOM_GROUP,
+    DOS_SYMPTOM_DISCRIMINATOR=DOS_PALLIATIVE_CARE_SYMPTOM_DISCRIMINATOR,
+    DOS_SG_SD_ID=DOS_PALLIATIVE_CARE_SGSDID,
+)
 
 BLOOD_PRESSURE = ServiceType(
     TYPE_NAME="Blood Pressure",

application/common/tests/test_service_type.py

@@ -11,6 +11,11 @@
 from application.common.service_type import BLOOD_PRESSURE, CONTRACEPTION
 
 
+def test_service_type__eq__():
+    assert BLOOD_PRESSURE == BLOOD_PRESSURE  # noqa: PLR0124
+    assert BLOOD_PRESSURE != CONTRACEPTION
+
+
 def test_blood_pressure_service_type():
     assert BLOOD_PRESSURE.TYPE_NAME == "Blood Pressure"
     assert BLOOD_PRESSURE.NHS_UK_SERVICE_CODE == "SRV0560"

application/dos_db_handler/requirements.txt

@@ -1,3 +1,3 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.2.0
+aws-lambda-powertools[tracer] ~= 2.24.0
 psycopg[binary]

application/dos_db_update_dlq_handler/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.2.0
+aws-lambda-powertools[tracer] ~= 2.24.0

application/event_replay/requirements.txt

@@ -1,3 +1,3 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.2.0
+aws-lambda-powertools[tracer] ~= 2.24.0
 simplejson

application/ingest_change_event/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer, validation] ~= 2.2.0
+aws-lambda-powertools[tracer, validation] ~= 2.24.0

application/pyproject.toml

@@ -1,9 +1,7 @@
 [tool.vulture]
 make_whitelist = true
-paths=["."]
-exclude = [
-  "/tests",
-  ]
+paths = ["."]
+exclude = ["/tests"]
 sort_by_size = true
 min_confidence = 100
 ignore_names = [
@@ -16,4 +14,4 @@ ignore_names = [
   "metadata",
   "recipient_email_address",
   "recipient_id",
-  ]
+]

application/quality_checker/check_dos.py

@@ -0,0 +1,61 @@
+from aws_lambda_powertools.logging import Logger
+from psycopg import Connection
+
+from .reporting import log_to_quality_check_report
+from .search_dos import (
+    search_for_incorrectly_profiled_z_code,
+    search_for_matching_services,
+    search_for_pharmacy_ods_codes,
+)
+from common.dos import DoSService
+from common.service_type import BLOOD_PRESSURE, CONTRACEPTION, ServiceType
+
+logger = Logger(child=True)
+
+
+def check_pharmacy_profiling(connection: Connection) -> None:
+    """Check the pharmacy profiling data quality of the dos database.
+
+    Args:
+        connection (Connection): Connection to the DoS DB.
+    """
+    odscodes = search_for_pharmacy_ods_codes(connection)
+    for odscode in odscodes:
+        matched_services = search_for_matching_services(connection, odscode)
+        check_for_multiple_of_service_type(matched_services, BLOOD_PRESSURE)
+        check_for_multiple_of_service_type(matched_services, CONTRACEPTION)
+
+
+def check_incorrect_zcode_profiling(connection: Connection, service_type: ServiceType) -> None:
+    """Check the palliative care profiling data quality of the dos database.
+
+    Args:
+        connection (Connection): Connection to the DoS DB.
+        service_type (ServiceType): Service type to check for.
+    """
+    if incorrectly_profiled_services := search_for_incorrectly_profiled_z_code(connection, service_type):
+        logger.info(
+            f"Found {len(incorrectly_profiled_services)} incorrectly "
+            f"profiled {service_type.TYPE_NAME.lower()} services.",
+            services=incorrectly_profiled_services,
+        )
+        log_to_quality_check_report(
+            incorrectly_profiled_services,
+            f"{service_type.TYPE_NAME} ZCode is on invalid service type",
+            service_type.DOS_SG_SD_ID,
+        )
+
+
+def check_for_multiple_of_service_type(matched_services: list[DoSService], service_type: ServiceType) -> None:
+    """Check for multiple of service type.
+
+    Args:
+        matched_services (list[DoSService]): List of matched services.
+        service_type (ServiceType): Service type to check for.
+    """
+    matched_service_types = [service for service in matched_services if service.typeid == service_type.DOS_TYPE_ID]
+    if len(matched_service_types) > 1:
+        log_to_quality_check_report(
+            matched_service_types,
+            f"Multiple 'Pharmacy' type services found (type {service_type.DOS_TYPE_ID})",
+        )