Quality Checker Lambda

Bump up versions

Begin adding quality checker values and updating variables

Bump up requirements

Improve variable names and use terraform jsonencode

Update more terraform

Add serverless cron

Add more dos searching

Add base level reporting functions

Reduce code duplication and log on z code incorrectly profiled

Start unit tests

Increase unit test coverage

Add schedule and deployment changes

Add job complete and incomplete alarms

Fix incorrect alarm threshold

Sort issues after rebase

Fix dependency conflict

Fix rebase error

Fix more rebase errors

Increase unit test coverage

Add additional quality checker metric

Add adr document

Add missing functionality

Fix unit tests

.coveragerc

@@ -5,4 +5,5 @@ omit =
   tests/*
   **/tests/*
   application/dos_db_handler/*.py
+  **/conftest.py
 branch = True

.github/workflows/dependency-review.yml

@@ -1,4 +1,4 @@
-name: 'Dependency Review'
+name: "Dependency Review"
 on: [pull_request]
 
 permissions:
@@ -8,7 +8,7 @@ jobs:
   dependency-review:
     runs-on: ubuntu-latest
     steps:
-      - name: 'Checkout Repository'
+      - name: "Checkout Repository"
         uses: actions/checkout@v4
-      - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v2
+      - name: "Dependency Review"
+        uses: actions/dependency-review-action@v3

.github/workflows/pull-request.yml

@@ -4,9 +4,9 @@ on:
   pull_request:
     types: [opened, edited, synchronize, reopened]
     branches-ignore:
-      - 'dependabot/**'
-      - 'release/**'
-      - 'main'
+      - "dependabot/**"
+      - "release/**"
+      - "main"
 
 permissions:
   contents: read

.github/workflows/update-pull-request-description.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     types: [opened]
     branches-ignore:
-      - 'dependabot/**'
+      - "dependabot/**"
 
 permissions:
   contents: read
@@ -38,6 +38,6 @@ jobs:
       - uses: tzkhan/pr-update-action@v2
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
-          head-branch-regex: '${{ env.BRANCH_DESIGNATOR }}/.*'
+          head-branch-regex: "${{ env.BRANCH_DESIGNATOR }}/.*"
           body-template: ${{ steps.template.outputs.result }}
-          body-update-action: 'replace'
+          body-update-action: "replace"

Makefile

@@ -97,13 +97,15 @@ UNIT_TEST_ARGS=" \
 		-e POWERTOOLS_LOG_DEDUPLICATION_DISABLED="1" \
 		--volume $(APPLICATION_DIR)/common:/tmp/.packages/common \
 		--volume $(APPLICATION_DIR)/change_event_dlq_handler:/tmp/.packages/change_event_dlq_handler \
+		--volume $(APPLICATION_DIR)/dos_db_handler:/tmp/.packages/dos_db_handler \
 		--volume $(APPLICATION_DIR)/dos_db_update_dlq_handler:/tmp/.packages/dos_db_update_dlq_handler \
 		--volume $(APPLICATION_DIR)/event_replay:/tmp/.packages/event_replay \
 		--volume $(APPLICATION_DIR)/ingest_change_event:/tmp/.packages/ingest_change_event \
 		--volume $(APPLICATION_DIR)/send_email:/tmp/.packages/send_email \
 		--volume $(APPLICATION_DIR)/service_matcher:/tmp/.packages/service_matcher \
 		--volume $(APPLICATION_DIR)/service_sync:/tmp/.packages/service_sync \
 		--volume $(APPLICATION_DIR)/slack_messenger:/tmp/.packages/slack_messenger \
+		--volume $(APPLICATION_DIR)/quality_checker:/tmp/.packages/quality_checker \
 		"
 
 integration-test: # End to end test DI project - mandatory: PROFILE, TAG=[complete|dev]; optional: ENVIRONMENT, PARALLEL_TEST_COUNT
@@ -152,58 +154,64 @@ remove-development-environments: # Removes development environments - mandatory:
 	done
 
 # ==============================================================================
-# Service Sync
+# Change Event Dead Letter Queue Handler (change-event-dlq-handler)
 
-service-sync-build-and-deploy: ### Build and deploy service sync lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=service-sync
+change-event-dlq-handler-build-and-deploy: ### Build and deploy change event dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=change-event-dlq-handler
 
 # ==============================================================================
-# Slack Messenger
+# DoS DB Update Dead Letter Queue Handler (dos-db-update-dlq-handler)
 
-slack-messenger-build-and-deploy: ### Build and deploy slack messenger lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=slack-messenger
+dos-db-update-dlq-handler-build-and-deploy: ### Build and deploy dos db update dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=dos-db-update-dlq-handler
 
 # ==============================================================================
-# Service Matcher
+# DoS DB Checker Handler (dos-db-handler)
 
-service-matcher-build-and-deploy: ### Build and deploy service matcher lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=service-matcher
+dos-db-handler-build-and-deploy: ### Build and deploy test db checker handler lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=dos-db-handler
 
 # ==============================================================================
-# Change Event Dead Letter Queue Handler (change-event-dlq-handler)
+# Event Replay lambda (event-replay)
 
-change-event-dlq-handler-build-and-deploy: ### Build and deploy change event dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=change-event-dlq-handler
+event-replay-build-and-deploy: ### Build and deploy event replay lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=event-replay
 
 # ==============================================================================
-# DoS DB Update Dead Letter Queue Handler (dos-db-update-dlq-handler) Nonprod only
+# Ingest Change Event
 
-dos-db-update-dlq-handler-build-and-deploy: ### Build and deploy dos db update dlq handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=dos-db-update-dlq-handler
+ingest-change-event-build-and-deploy: ### Build and deploy ingest change event lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=ingest-change-event
 
 # ==============================================================================
-# Event Replay lambda (event-replay)
+# Send Email
 
-event-replay-build-and-deploy: ### Build and deploy event replay lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=event-replay
+send-email-build-and-deploy: ### Build and deploy send email lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=send-email
 
 # ==============================================================================
-# DoS DB Checker Handler (dos-db-handler)
+# Service Matcher
 
-dos-db-handler-build-and-deploy: ### Build and deploy test db checker handler lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=dos-db-handler
+service-matcher-build-and-deploy: ### Build and deploy service matcher lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=service-matcher
 
 # ==============================================================================
-# Send Email
+# Service Sync
 
-send-email-build-and-deploy: ### Build and deploy send email lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=send-email
+service-sync-build-and-deploy: ### Build and deploy service sync lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=service-sync
 
 # ==============================================================================
-# Ingest Change Event
+# Slack Messenger
 
-ingest-change-event-build-and-deploy: ### Build and deploy ingest change event lambda docker image - mandatory: PROFILE, ENVIRONMENT, FUNCTION_NAME
-	make build-and-deploy-single-function FUNCTION_NAME=ingest-change-event
+slack-messenger-build-and-deploy: ### Build and deploy slack messenger lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=slack-messenger
+
+# ==============================================================================
+# Quality Checker
+
+quality-checker-build-and-deploy: ### Build and deploy quality checker lambda docker image - mandatory: PROFILE, ENVIRONMENT
+	make build-and-deploy-single-function FUNCTION_NAME=quality-checker
 
 # ==============================================================================
 # Deployments
@@ -233,15 +241,6 @@ push-images: # Use VERSION=[] to push a perticular version otherwise with defaul
 push-tester-image:
 	make docker-push NAME=tester
 
-# ==============================================================================
-# SES (Simple Email Service)
-
-deploy-email: # Deploys SES resources - mandatory: PROFILE=[live/test]
-	make terraform-apply-auto-approve STACKS=email ENVIRONMENT=$(AWS_ACCOUNT_NAME)
-
-undeploy-email: # Deploys SES resources - mandatory: PROFILE=[live/test]
-	make terraform-destroy-auto-approve STACKS=email ENVIRONMENT=$(AWS_ACCOUNT_NAME)
-
 # ==============================================================================
 # Development Tools
 

application/change_event_dlq_handler/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.25.0
+aws-lambda-powertools[tracer] ~= 2.26.0

application/dos_db_handler/requirements.txt

@@ -1,3 +1,3 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.25.0
+aws-lambda-powertools[tracer] ~= 2.26.0
 psycopg[binary]

application/dos_db_update_dlq_handler/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.25.0
+aws-lambda-powertools[tracer] ~= 2.26.0

application/event_replay/requirements.txt

@@ -1,3 +1,3 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer] ~= 2.25.0
+aws-lambda-powertools[tracer] ~= 2.26.0
 simplejson

application/ingest_change_event/requirements.txt

@@ -1,2 +1,2 @@
 aws-embedded-metrics
-aws-lambda-powertools[tracer, validation] ~= 2.25.0
+aws-lambda-powertools[tracer, validation] ~= 2.26.0

application/pyproject.toml

@@ -1,9 +1,7 @@
 [tool.vulture]
 make_whitelist = true
-paths=["."]
-exclude = [
-  "/tests",
-  ]
+paths = ["."]
+exclude = ["/tests"]
 sort_by_size = true
 min_confidence = 100
 ignore_names = [
@@ -16,4 +14,4 @@ ignore_names = [
   "metadata",
   "recipient_email_address",
   "recipient_id",
-  ]
+]

application/quality_checker/check_dos.py

@@ -0,0 +1,93 @@
+from aws_lambda_powertools.logging import Logger
+from psycopg import Connection
+
+from .reporting import log_to_quality_check_report
+from .search_dos import (
+    search_for_incorrectly_profiled_z_code_on_correct_type,
+    search_for_incorrectly_profiled_z_code_on_incorrect_type,
+    search_for_matching_services,
+    search_for_pharmacy_ods_codes,
+)
+from common.commissioned_service_type import BLOOD_PRESSURE, CONTRACEPTION, PALLIATIVE_CARE, CommissionedServiceType
+from common.dos import DoSService
+
+logger = Logger(child=True)
+
+
+def check_pharmacy_profiling(connection: Connection) -> None:
+    """Check the pharmacy profiling data quality of the dos database.
+
+    Args:
+        connection (Connection): Connection to the DoS DB.
+    """
+    odscodes = search_for_pharmacy_ods_codes(connection)
+    for odscode in odscodes:
+        matched_services = search_for_matching_services(connection, odscode)
+        check_for_multiple_of_service_type(matched_services, BLOOD_PRESSURE)
+        check_for_multiple_of_service_type(matched_services, CONTRACEPTION)
+
+
+def check_for_zcode_profiling_on_incorrect_type(connection: Connection, service_type: CommissionedServiceType) -> None:
+    """Check the zcode profiling data quality of the dos database.
+
+    Args:
+        connection (Connection): Connection to the DoS DB.
+        service_type (CommissionedServiceType): Service type to check for.
+    """
+    if incorrectly_profiled_services := search_for_incorrectly_profiled_z_code_on_incorrect_type(
+        connection,
+        service_type,
+    ):
+        logger.info(
+            f"Found {len(incorrectly_profiled_services)} incorrectly "
+            f"profiled {service_type.TYPE_NAME.lower()} services.",
+            services=incorrectly_profiled_services,
+        )
+        log_to_quality_check_report(
+            incorrectly_profiled_services,
+            f"{service_type.TYPE_NAME} ZCode is on invalid service type",
+            service_type.DOS_SG_SD_ID,
+        )
+
+
+def check_for_palliative_care_profiling(connection: Connection) -> None:
+    """Check the zcode profiling data quality of the dos database.
+
+    Args:
+        connection (Connection): Connection to the DoS DB.
+        service_type (CommissionedServiceType): Service type to check for.
+    """
+    check_for_zcode_profiling_on_incorrect_type(connection, PALLIATIVE_CARE)
+    if incorrectly_profiled_services := search_for_incorrectly_profiled_z_code_on_correct_type(
+        connection,
+        PALLIATIVE_CARE,
+    ):
+        logger.info(
+            f"Found {len(incorrectly_profiled_services)} incorrectly "
+            f"profiled {PALLIATIVE_CARE.TYPE_NAME.lower()} services.",
+            services=incorrectly_profiled_services,
+        )
+        log_to_quality_check_report(
+            incorrectly_profiled_services,
+            f"{PALLIATIVE_CARE.TYPE_NAME} ZCode is on the correct service type, "
+            "but the service is incorrectly profiled",
+            PALLIATIVE_CARE.DOS_SG_SD_ID,
+        )
+
+
+def check_for_multiple_of_service_type(
+    matched_services: list[DoSService],
+    service_type: CommissionedServiceType,
+) -> None:
+    """Check for multiple of service type.
+
+    Args:
+        matched_services (list[DoSService]): List of matched services.
+        service_type (CommissionedServiceType): Service type to check for.
+    """
+    matched_service_types = [service for service in matched_services if service.typeid == service_type.DOS_TYPE_ID]
+    if len(matched_service_types) > 1:
+        log_to_quality_check_report(
+            matched_service_types,
+            f"Multiple 'Pharmacy' type services found (type {service_type.DOS_TYPE_ID})",
+        )

application/quality_checker/quality_checker.py

@@ -0,0 +1,76 @@
+from os import environ
+from typing import Any
+
+from aws_embedded_metrics import metric_scope
+from aws_lambda_powertools.logging import Logger
+from aws_lambda_powertools.tracing import Tracer
+from aws_lambda_powertools.utilities.data_classes import EventBridgeEvent, event_source
+from aws_lambda_powertools.utilities.typing.lambda_context import LambdaContext
+
+from .check_dos import (
+    check_for_palliative_care_profiling,
+    check_for_zcode_profiling_on_incorrect_type,
+    check_pharmacy_profiling,
+)
+from common.commissioned_service_type import BLOOD_PRESSURE, CONTRACEPTION
+from common.dos_db_connection import connect_to_db_reader
+from common.middlewares import unhandled_exception_logging
+
+logger = Logger()
+tracer = Tracer()
+
+
+@tracer.capture_lambda_handler()
+@logger.inject_lambda_context(clear_state=True)
+@unhandled_exception_logging
+@event_source(data_class=EventBridgeEvent)
+def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> None:  # noqa: ARG001
+    """Lambda handler for quality checker."""
+    try:
+        logger.debug("Quality checker started.")
+        check_dos_data_quality()
+        logger.debug("Quality checker finished.")
+        send_finished_metric()
+    except Exception:
+        send_errored_metric()
+        raise
+
+
+def check_dos_data_quality() -> None:
+    """Check the data quality of the dos database."""
+    with connect_to_db_reader() as db_connection:
+        # Checks matched odscode services for pharmacy profiling
+        check_pharmacy_profiling(db_connection)
+
+        # Checks matched odscode services for incorrectly profiled palliative care
+        check_for_palliative_care_profiling(db_connection)
+
+        # Checks matched odscode services for incorrectly profiled blood pressure
+        check_for_zcode_profiling_on_incorrect_type(db_connection, BLOOD_PRESSURE)
+
+        # Checks matched odscode services for incorrectly profiled contraception
+        check_for_zcode_profiling_on_incorrect_type(db_connection, CONTRACEPTION)
+
+
+@metric_scope
+def send_finished_metric(metrics: Any) -> None:  # noqa: ANN401
+    """Send a metric to indicate that the quality checker has finished.
+
+    Args:
+        metrics (Metrics): CloudWatch embedded metrics object
+    """
+    metrics.set_namespace("UEC-DOS-INT")
+    metrics.set_dimensions({"ENV": environ["ENV"]})
+    metrics.put_metric("QualityCheckerFinished", 1, "Count")
+
+
+@metric_scope
+def send_errored_metric(metrics: Any) -> None:  # noqa: ANN401
+    """Send a metric to indicate that the quality checker has errored.
+
+    Args:
+        metrics (Metrics): CloudWatch embedded metrics object
+    """
+    metrics.set_namespace("UEC-DOS-INT")
+    metrics.set_dimensions({"ENV": environ["ENV"]})
+    metrics.put_metric("QualityCheckerErrored", 1, "Count")