Changed file based tests

.github/workflows/cicd_unittest_deploy.yml

@@ -13,8 +13,62 @@ on:
       - "*"
 
 jobs:
+  files-changed:
+    name: Detect what files changed
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+
+    # Map a step output to a job output
+    outputs:
+      common: ${{ steps.changes.outputs.common }}
+      assignments: ${{ steps.changes.outputs.assignments }}
+      docs: ${{ steps.changes.outputs.docs }}
+      dq: ${{ steps.changes.outputs.dq }}
+      media_files: ${{ steps.changes.outputs.media_files }}
+      timezones: ${{ steps.changes.outputs.timezones }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          token: ${{ github.token }}
+          filters: |
+            # Common changes that require all tests to run
+            common:
+              - 'app/utils/**'
+              - 'app/blueprints/forms/**'
+              - 'app/blueprints/module_questionnaire/**'
+              - 'app/blueprints/module_selection/**'
+              - 'app/blueprints/notifications/**'
+              - 'app/blueprints/auth/**'
+              - 'app/blueprints/healthcheck/**'
+              - 'app/blueprints/profile/**'
+              - 'app/blueprints/surveys/**'
+              - 'app/blueprints/user_management/**'
+              - 'app/blueprints/roles/**'
+            # Grouped module wise changes
+            assignments:
+              - 'app/blueprints/assignments/**'
+              - 'app/blueprints/enumerators/**'
+              - 'app/blueprints/locations/**'
+              - 'app/blueprints/mapping/**'
+              - 'app/blueprints/emails/**'
+              - 'app/blueprints/target_status_mapping/**'
+              - 'app/blueprints/targets/**'
+            docs:
+              - 'app/blueprints/docs/**'
+            dq:
+              - 'app/blueprints/dq/**'
+            media_files:
+              - 'app/blueprints/media_files/**'
+            timezones:
+              - 'app/blueprints/timezones/**'
   UnitTest:
     runs-on: ubuntu-latest
+    needs: files-changed
 
     permissions:
       id-token: write
@@ -47,8 +101,35 @@ jobs:
       - name: Check DB Migration
         run: docker compose -f docker-compose/docker-compose.db-check.yml -f docker-compose/docker-compose.override-unit-test.yml run --rm api ;
 
+      - name: Select test files based on changes
+        run: |
+          if [ "${{ needs.files-changed.outputs.common }}" = 'true' ]; then
+            TEST_PATHS="tests"
+          else
+            TEST_PATHS=''
+            [ "${{ needs.files-changed.outputs.assignments }}" = 'true' ] && TEST_PATHS="$TEST_PATHS tests/test_assignments.py tests/test_enumerators.py tests/test_locations.py tests/test_mapping.py tests/test_emails.py tests/test_target_status_mapping.py tests/test_targets.py"
+            [ "${{ needs.files-changed.outputs.docs }}" = 'true' ] && TEST_PATHS="$TEST_PATHS tests/test_docs.py"
+            [ "${{ needs.files-changed.outputs.dq }}" = 'true' ] && TEST_PATHS="$TEST_PATHS tests/test_dq.py"
+            [ "${{ needs.files-changed.outputs.media_files }}" = 'true' ] && TEST_PATHS="$TEST_PATHS tests/test_media_files.py"
+            [ "${{ needs.files-changed.outputs.timezones }}" = 'true' ] && TEST_PATHS="$TEST_PATHS tests/test_timezones.py"
+          fi
+          # Trim leading/trailing whitespace
+          TEST_PATHS="${TEST_PATHS## }"
+          TEST_PATHS="${TEST_PATHS%% }"
+          echo "Selected TEST_PATHS: '$TEST_PATHS'"
+          # Persist for subsequent steps (preserve spaces)
+          {
+          echo "TEST_PATHS<<__EOF__"
+          echo "$TEST_PATHS"
+          echo "__EOF__"
+            } >> "$GITHUB_ENV"
+            echo "Tests according to file changes: ${TEST_PATHS}"
+
       - name: Run unit tests
-        run: docker compose -f docker-compose/docker-compose.unit-test.yml -f docker-compose/docker-compose.override-unit-test.yml run --rm api ;
+        shell: bash
+        run: |
+          echo "Workflow TEST_PATHS: '${TEST_PATHS}'"
+          docker compose -f docker-compose/docker-compose.unit-test.yml -f docker-compose/docker-compose.override-unit-test.yml run --rm -e TEST_PATHS api ;
 
       - name: Send coverage report to Coveralls
         uses: coverallsapp/github-action@v2

app/blueprints/docs/surveystream.yml

@@ -11696,8 +11696,8 @@ paths:
                   description: Indicates if the DQ check applies to all questions in the form
                   example: true
                 question_name:
-                  type: string
-                  description: Name of the question the DQ check applies to. Value is None if check applies to all questions
+                  type: string or list
+                  description: List of questions or Name of the question the DQ check applies to. Value is None if check applies to all questions
                   example: "district_name"
                 module_name:
                   type: string
@@ -11910,6 +11910,7 @@ paths:
                     type: string
                     example: "An error occurred while deleting the DQ check"
 
+
   /dq/checks/{dq_check_uid}:
     put:
       summary: Update DQ check for a form

app/blueprints/dq/controllers.py

@@ -28,6 +28,7 @@
 )
 from .utils import validate_dq_check
 from .validators import (
+    BulkDQCheckValidator,
     DQChecksQueryParamValidator,
     DQCheckValidator,
     DQConfigQueryParamValidator,
@@ -543,9 +544,9 @@ def get_dq_checks(validated_query_params):
                     not logic_check_question_found and question != check.question_name
                 ):  # Check if the question is not the same as the main question
                     check_dict["active"] = False
-                    check_dict[
-                        "note"
-                    ] = "Logic check question not found in form definition"
+                    check_dict["note"] = (
+                        "Logic check question not found in form definition"
+                    )
 
                 logic_check_questions_list.append(question_dict)
             check_dict["check_components"][
@@ -632,7 +633,7 @@ def get_dq_checks(validated_query_params):
 
 @dq_bp.route("/checks", methods=["POST"])
 @logged_in_active_user_required
-@validate_payload(DQCheckValidator)
+@validate_payload(BulkDQCheckValidator)
 @custom_permissions_required("WRITE Data Quality", "body", "form_uid")
 def add_dq_check(validated_payload):
     """
@@ -641,13 +642,17 @@ def add_dq_check(validated_payload):
     """
     form_uid = validated_payload.form_uid.data
     type_id = validated_payload.type_id.data
+    if isinstance(validated_payload.question_name.data, str):
+        question_list = [validated_payload.question_name.data]
+    else:
+        question_list = validated_payload.question_name.data
 
     try:
         validate_dq_check(
             form_uid,
             type_id,
             validated_payload.all_questions.data,
-            validated_payload.question_name.data,
+            question_list,
             validated_payload.dq_scto_form_uid.data,
             validated_payload.check_components.data,
             validated_payload.filters.data,
@@ -677,68 +682,77 @@ def add_dq_check(validated_payload):
     logic_check_questions = check_components.pop("logic_check_questions", None)
     logic_check_assertions = check_components.pop("logic_check_assertions", None)
 
-    dq_check = DQCheck(
-        form_uid=form_uid,
-        type_id=validated_payload.type_id.data,
-        all_questions=validated_payload.all_questions.data,
-        question_name=validated_payload.question_name.data,
-        dq_scto_form_uid=validated_payload.dq_scto_form_uid.data,
-        module_name=validated_payload.module_name.data,
-        flag_description=validated_payload.flag_description.data,
-        check_components=check_components,
-        active=validated_payload.active.data,
-    )
-
-    try:
-        db.session.add(dq_check)
-        db.session.flush()
-
-        dq_check_uid = dq_check.dq_check_uid
-
-        # Add filters for the check
-        max_filter_group_id = 0
-
-        for filter_group in validated_payload.filters.data:
-            max_filter_group_id += 1
-
-            for filter in filter_group.get("filter_group"):
-                dq_check_filter = DQCheckFilters(
-                    dq_check_uid=dq_check_uid,
-                    filter_group_id=max_filter_group_id,
-                    question_name=filter["question_name"],
-                    filter_operator=filter["filter_operator"],
-                    filter_value=filter["filter_value"],
-                )
-                db.session.add(dq_check_filter)
-        db.session.flush()
+    def create_dq_check(question_name=None):
+        """Helper function to create DQ check and related records"""
+        dq_check = DQCheck(
+            form_uid=form_uid,
+            type_id=validated_payload.type_id.data,
+            all_questions=validated_payload.all_questions.data,
+            question_name=question_name,
+            dq_scto_form_uid=validated_payload.dq_scto_form_uid.data,
+            module_name=validated_payload.module_name.data,
+            flag_description=validated_payload.flag_description.data,
+            check_components=check_components,
+            active=validated_payload.active.data,
+        )
 
-        # Add logic check questions and assertions
-        if validated_payload.type_id.data == 1:
-            for question in logic_check_questions:
-                logic_check_question = DQLogicCheckQuestions(
-                    dq_check_uid=dq_check_uid,
-                    question_name=question["question_name"],
-                    alias=question["alias"],
-                )
-                db.session.add(logic_check_question)
+        try:
+            db.session.add(dq_check)
             db.session.flush()
 
-            max_assert_group_id = 0
-            for assert_group in logic_check_assertions:
-                max_assert_group_id += 1
-
-                for assertion in assert_group.get("assert_group"):
-                    logic_check_assertion = DQLogicCheckAssertions(
-                        dq_check_uid=dq_check_uid,
-                        assert_group_id=max_assert_group_id,
-                        assertion=assertion["assertion"],
+            # Add filters
+            for filter_group_id, filter_group in enumerate(
+                validated_payload.filters.data, 1
+            ):
+                for filter in filter_group.get("filter_group"):
+                    db.session.add(
+                        DQCheckFilters(
+                            dq_check_uid=dq_check.dq_check_uid,
+                            filter_group_id=filter_group_id,
+                            question_name=filter["question_name"],
+                            filter_operator=filter["filter_operator"],
+                            filter_value=filter["filter_value"],
+                        )
                     )
-                    db.session.add(logic_check_assertion)
             db.session.flush()
 
-    except Exception as e:
-        db.session.rollback()
-        return jsonify({"message": str(e), "success": False}), 500
+            # Add logic check questions and assertions if type is 1
+            if validated_payload.type_id.data == 1:
+                # Add questions
+                for question in logic_check_questions:
+                    db.session.add(
+                        DQLogicCheckQuestions(
+                            dq_check_uid=dq_check.dq_check_uid,
+                            question_name=question["question_name"],
+                            alias=question["alias"],
+                        )
+                    )
+                db.session.flush()
+
+                # Add assertions
+                for assert_group_id, assert_group in enumerate(
+                    logic_check_assertions, 1
+                ):
+                    for assertion in assert_group.get("assert_group"):
+                        db.session.add(
+                            DQLogicCheckAssertions(
+                                dq_check_uid=dq_check.dq_check_uid,
+                                assert_group_id=assert_group_id,
+                                assertion=assertion["assertion"],
+                            )
+                        )
+                db.session.flush()
+
+        except Exception as e:
+            db.session.rollback()
+            return jsonify({"message": str(e), "success": False}), 500
+
+    # Create checks based on whether we have specific questions or all questions
+    if validated_payload.all_questions.data:
+        create_dq_check()
+    else:
+        for question_name in question_list:
+            create_dq_check(question_name)
 
     try:
         db.session.commit()

app/blueprints/dq/utils.py

@@ -16,6 +16,10 @@ def validate_dq_check(
 
     """
 
+    # Check if question name is list or string
+    if isinstance(question_name, str):
+        question_name = [question_name]
+
     # Raise error if both all_questions and question_name are not provided
     if not all_questions and not question_name:
         raise Exception(
@@ -34,18 +38,33 @@ def validate_dq_check(
             "Question name cannot be provided if all questions is selected."
         )
 
+    # if type_id is 1, 9, 10 question name should contain only one question
+    if type_id == 1 and len(question_name) > 1:
+        raise Exception(
+            "For logic checks, question name should contain only one question."
+        )
+    if type_id == 9 and len(question_name) > 1:
+        raise Exception(
+            "For spotcheck, question name should contain only one question."
+        )
+    if type_id == 10 and len(question_name) > 1:
+        raise Exception(
+            "For GPS checks, question name should contain only one question."
+        )
+
     # Check if the question name is valid, when check is active
     # For protocol (8) and spotcheck (9) checks, question name is from DQ form which is checked later
     if question_name and active is True and type_id not in [8, 9]:
-        scto_question = SCTOQuestion.query.filter(
-            SCTOQuestion.form_uid == form_uid,
-            SCTOQuestion.question_name == question_name,
-        ).first()
+        for name in question_name:
+            scto_question = SCTOQuestion.query.filter(
+                SCTOQuestion.form_uid == form_uid,
+                SCTOQuestion.question_name == name,
+            ).first()
 
-        if scto_question is None:
-            raise Exception(
-                f"Question name '{question_name}' not found in form definition. Active checks must have a valid question name."
-            )
+            if scto_question is None:
+                raise Exception(
+                    f"Question name '{name}' not found in form definition. Active checks must have a valid question name."
+                )
 
     # Check if the filter question names are valid, when check is active
     if filters and active is True and type_id not in [7, 8, 9]:
@@ -82,15 +101,16 @@ def validate_dq_check(
 
     # for mismatch (7), protocol (8) and spotcheck (9), check if question name is present in dq form
     if type_id in [7, 8, 9] and active is True:
-        dq_scto_question = SCTOQuestion.query.filter(
-            SCTOQuestion.form_uid == dq_scto_form_uid,
-            SCTOQuestion.question_name == question_name,
-        ).first()
+        for name in question_name:
+            dq_scto_question = SCTOQuestion.query.filter(
+                SCTOQuestion.form_uid == dq_scto_form_uid,
+                SCTOQuestion.question_name == name,
+            ).first()
 
-        if dq_scto_question is None:
-            raise Exception(
-                f"Question name '{question_name}' not found in DQ form definition. Active checks must have a valid question name."
-            )
+            if dq_scto_question is None:
+                raise Exception(
+                    f"Question name '{name}' not found in DQ form definition. Active checks must have a valid question name."
+                )
 
     # for mismatch (7), protocol (8) and spotcheck (9), check if question name used in filters is present in dq form
     if type_id in [7, 8, 9] and filters and active is True: