DPTS-1096 Add stt tests (#64)

* DPTS-1096 Add int tests for stt

* DPTS-1096 Correct sql script

* DPTS-1096 Adjust assertions post test

* DPTS-1096 Fix broken tests

Makefile

@@ -313,6 +313,7 @@ build-tester: # Builds image used for testing - mandatory: PROFILE=[name]
 	mkdir $(DOCKER_DIR)/tester/assets/integration/data-files
 	cp $(APPLICATION_TEST_DIR)/integration/data-files/* $(DOCKER_DIR)/tester/assets/integration/data-files
 	make docker-image NAME=tester
+	rm -rf $(DOCKER_DIR)/tester/assets/integration
 
 push-tester: # Pushes image used for testing - mandatory: PROFILE=[name]
 	make docker-push NAME=tester

build/jenkins/Jenkinsfile.hk-integration

@@ -193,6 +193,18 @@ pipeline {
                 }
             }
         }
+        stage('Run stt tests') {
+            steps {
+                echo 'Running tests for stt scenario bundle upload'
+                catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                    build job: "${SINGLE_HK_TEST_JOB_NAME}/${BRANCH_NAME}",\
+                    parameters: [string(name:'Task' , value:"stt"),\
+                    string(name:'Bucket' , value:"${BUCKET}"),\
+                    string(name:'Profile' , value:"${PROFILE}")
+                    ]
+                }
+            }
+        }
         stage('Check and remove older versions') {
             steps {
                 script { echo 'Check and remove' }

build/jenkins/Jenkinsfile.hk-integration-test

@@ -27,6 +27,7 @@ pipeline {
         BUCKET = "${params.Bucket}"
         PROFILE = "${params.Profile}"
         FILENAME = "int_${TASK}.csv"
+        STT_FILENAME = '36.2.0_stt.zip'
         MAX_ATTEMPTS = 10
     }
     triggers { pollSCM("* * * * *") }
@@ -36,7 +37,30 @@ pipeline {
                 script { sh 'make show-configuration' }
             }
         }
+        stage('Load stt scenario zipfile to s3') {
+            when { expression { params.Task == "stt" } }
+            steps {
+                script {
+                    echo "Loading test file ${STT_FILENAME} to s3 bucket ${BUCKET}"
+                    catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                        sh "make load-single-integration-test-file-to-s3 FILENAME=${STT_FILENAME} BUCKET=${BUCKET}"
+                    }
+                }
+            }
+        }
+        stage('Check archive folder for stt zipfile file') {
+            when { expression { params.Task == "stt" } }
+            steps {
+                script {
+                    echo "Checking archive folder in s3 bucket ${BUCKET} for test file ${STT_FILENAME}"
+                    catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                        sh "make check-single-integration-test-file MAX_ATTEMPTS=${MAX_ATTEMPTS} FILENAME=${STT_FILENAME} BUCKET=${BUCKET}/archive"
+                    }
+                }
+            }
+        }
         stage('Load test file to s3') {
+            when { expression { params.Task != "stt" } }
             steps {
                 script {
                     echo "Loading test file ${FILENAME} to s3 bucket ${BUCKET}"
@@ -47,6 +71,7 @@ pipeline {
             }
         }
         stage('Check archive folder for test file') {
+            when { expression { params.Task != "stt" } }
             steps {
                 script {
                     echo "Checking archive folder in s3 bucket ${BUCKET} for test file ${FILENAME}"

test/integration/data-files/test-data.sql

@@ -40,3 +40,11 @@ values (1121,4017);
 -- create record to symptomdiscriminatorsynonyms job to delete later
 insert into pathwaysdos.symptomdiscriminatorsynonyms (name, symptomdiscriminatorid)
 values ('Integration test delete','11009');
+--
+--for stt integration testing
+-- remove all existing stt scenarios
+delete from pathwaysdos.scenarios s
+where s.scenarioid is not null;
+-- remove all existing stt bundles
+delete from pathwaysdos.scenariobundles sb
+where sb.id is not null;

test/integration/handler.py

@@ -2,11 +2,11 @@
 import os
 import json
 
-from models import symptomgroup, referralrole, servicetype, symptomdiscriminator, symptomgroupsymptomdiscriminator, symptomdiscriminatorsynonyms
+from models import symptomgroup, referralrole, servicetype, symptomdiscriminator, symptomgroupsymptomdiscriminator, symptomdiscriminatorsynonyms, stt
 
 data_sql_scripts = ("./data-files/test-data.sql",)
 # List of tasks handled by this code DO NOT change the order add any new task to end
-valid_tasks = ("data", "symptomgroups", "referralroles", "servicetypes", "symptomdiscriminators", "symptomgroupdiscriminators", "symptomdiscriminatorsynonyms")
+valid_tasks = ("data", "symptomgroups", "referralroles", "servicetypes", "symptomdiscriminators", "symptomgroupdiscriminators", "symptomdiscriminatorsynonyms", "stt")
 
 
 def request(event, context):
@@ -52,6 +52,8 @@ def run_data_checks_for_hk_task(env, task, db_connection):
         checks_pass = symptomgroupsymptomdiscriminator.check_symptom_group_symptom_discriminators_data(env, db_connection)
     if task.lower() == valid_tasks[6].lower():
         checks_pass = symptomdiscriminatorsynonyms.check_symptom_discriminator_synonyms_data(env, db_connection)
+    if task.lower() == valid_tasks[7].lower():
+        checks_pass = stt.check_stt_scenario_data(env, db_connection)
     # if no code to handle task default to fail
     if checks_pass is None:
         logger.log_for_audit(env, "No function to handle task {}".format(task))

test/integration/models/stt.py

@@ -0,0 +1,69 @@
+from utilities import logger, database
+
+#  Note
+#  db tables are scenarios and scenariobundles
+#  See test-data.sql (clears down all bundles/scenarios0 and test/integration/test-files/36.2.0_stt.zip
+#  Will test
+#  That there is only one record in scenarionbundlees table
+
+expected_stt_bundle_name = '36.2.0'
+
+expected_disposition_id = 1
+expected_disposition_group_id = 18
+expected_scenario_id = 893
+expected_symptomgroup_id = 1014
+expected_symptom_discriminator_id = 4499
+expected_age_id = 1
+expected_gender_id = 2
+
+
+def get_stt_scenario_data(env, db_connection):
+    """Returns expected data on all stt scenario bundles and scenarios within bundles"""
+    result_set = {}
+    try:
+        query = create_stt_scenarios_query()
+        result_set = database.execute_resultset_query(env, db_connection, query, ())
+    except Exception as e:
+        logger.log_for_error(
+            env,
+            "Error checking results for {0} => {1}".format("referral roles", str(e)),
+        )
+    return result_set
+
+def create_stt_scenarios_query():
+    query = ("""select sb.name, s.scenarioid, s.symptomgroupid, s.dispositionid, s.dispositiongroupid, s.symptomdiscriminatorid,
+    s.ageid, s.genderid
+    from scenariobundles sb
+    join scenarios s
+    on s.scenariobundleid = sb.id"""
+    )
+    return query
+
+def check_scenario_data_item(env, scenario, column_name, expected_value):
+    ok = True
+    if scenario[column_name] != expected_value:
+        ok = False
+        logger.log_for_audit(
+            env,
+            "Column name {0} set to {1} and not expected value of {2}".format(column_name,scenario[column_name],expected_value),
+        )
+    return ok
+
+
+def check_stt_scenario_data(env, db_connection):
+    """Returns True if single record in resultset and all checks pass ; otherwise returns False"""
+    flags = []
+    result_set = get_stt_scenario_data(env, db_connection)
+    for scenario in result_set:
+        flags.append(check_scenario_data_item(env, scenario, "name", expected_stt_bundle_name))
+        flags.append(check_scenario_data_item(env, scenario, "scenarioid", expected_scenario_id))
+        flags.append(check_scenario_data_item(env, scenario, "dispositionid", expected_disposition_id))
+        flags.append(check_scenario_data_item(env, scenario, "dispositiongroupid", expected_disposition_group_id))
+        flags.append(check_scenario_data_item(env, scenario, "symptomgroupid", expected_symptomgroup_id))
+        flags.append(check_scenario_data_item(env, scenario, "symptomdiscriminatorid", expected_symptom_discriminator_id))
+        flags.append(check_scenario_data_item(env, scenario, "ageid", expected_age_id))
+        flags.append(check_scenario_data_item(env, scenario, "genderid", expected_gender_id))
+    if len(flags)==0:
+        flags.append(False)
+    return all(flags)
+

test/unit/hk/integration/test_handler.py

@@ -8,7 +8,7 @@
 
 file_path = "application.hk.integration.handler"
 env = 'integration'
-valid_tasks = ("data","symptomgroups","referralroles", "servicetypes", "symptomdiscriminators", "symptomgroupdiscriminators", "symptomdiscriminatorsynonyms")
+valid_tasks = ("data","symptomgroups","referralroles", "servicetypes", "symptomdiscriminators", "symptomgroupdiscriminators", "symptomdiscriminatorsynonyms", "stt")
 
 @patch(f"{file_path}.logger.log_for_audit")
 def test_is_valid_task_invalid_lc(mock_logger):
@@ -36,7 +36,7 @@ def test_valid_task_list():
 # This may be hard to maintain as nothing in this list can be in its actual position
 # add any new task as penultimate in list
 def test_invalid_task_list():
-    temp_valid_tasks = ("symptomgroups","referralroles","servicetypes","symptomdiscriminators","symptomgroupdiscriminators","symptomdiscriminatorsynonyms","data")
+    temp_valid_tasks = ("symptomgroups","referralroles","servicetypes","symptomdiscriminators","symptomgroupdiscriminators","symptomdiscriminatorsynonyms","stt", "data")
     assert len(handler.valid_tasks) == len(temp_valid_tasks)
     for i in range(len(handler.valid_tasks)):
         assert handler.valid_tasks[i] != temp_valid_tasks[i]

test/unit/hk/integration/test_stt.py

@@ -0,0 +1,76 @@
+from unittest.mock import Mock, patch
+import pytest
+import string
+import psycopg2
+
+from models import stt
+
+file_path = "application.models.stt"
+env = 'unittest'
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_expected(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == True
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.3.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_name(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":2,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_age(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":1,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_gender(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":892,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_scenarioid(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":1,"scenarioid":893,"symptomgroupid":1015, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_symptomgroupid(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":2, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_dispositionid(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":19, "symptomdiscriminatorid":4499}])
+def test_check_stt_wrong_dispositiongroupid(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4500}])
+def test_check_stt_wrong_symptomdiscriminatorid(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [])
+def test_check_stt_empty_resultset(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.1',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499},{"name":'36.2.0',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499}])
+def test_two_records_first_wrong(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query", return_value = [{"name":'36.2.0',"ageid":1,"genderid":1,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4499},{"name":'36.2.3',"ageid":1,"genderid":2,"scenarioid":893,"symptomgroupid":1014, "dispositionid":1, "dispositiongroupid":18, "symptomdiscriminatorid":4500}])
+def test_two_records_second_wrong(mock_db, mock_resultset):
+    assert stt.check_stt_scenario_data(env, mock_db) == False
+
+@patch("psycopg2.connect")
+@patch(f"{file_path}.database.execute_resultset_query",side_effect=Exception)
+def test_get_stt_scenario_data(mock_resultset, mock_db):
+    result_set = stt.get_stt_scenario_data(env,mock_db)
+    assert result_set == {}