Changes to have all answer options to appear in the report (#24)

* adding table to store available option in analytic

* etl changes to load available option table

* fixing linting

* fixing linting

analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py

@@ -0,0 +1,41 @@
+"""adding_available_response_options
+
+Revision ID: e7fdf769e8ff
+Revises: 3a705c422892
+Create Date: 2023-11-21 12:45:34.871602
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e7fdf769e8ff'
+down_revision = '3a705c422892'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('available_response_option',
+    sa.Column('created_date', sa.DateTime(), nullable=True),
+    sa.Column('updated_date', sa.DateTime(), nullable=True),
+    sa.Column('is_active', sa.Boolean(), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('participant_id', sa.Integer(), nullable=True),
+    sa.Column('request_key', sa.String(length=100), nullable=False),
+    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('request_id', sa.String(length=20), nullable=True),
+    sa.Column('survey_id', sa.Integer(), nullable=False),
+    sa.Column('runcycle_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['survey_id'], ['survey.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', 'request_key')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('available_response_option')
+    # ### end Alembic commands ###

analytics-api/src/analytics_api/models/available_response_option.py

@@ -0,0 +1,12 @@
+"""available_response_option model class.
+
+Manages the Available Options for a option type questions on a survey
+"""
+from .base_model import BaseModel
+from .response_mixin import ResponseMixin
+
+
+class AvailableResponseOption(BaseModel, ResponseMixin):  # pylint: disable=too-few-public-methods
+    """Definition of the Available Response Options entity."""
+
+    __tablename__ = 'available_response_option'

analytics-api/src/analytics_api/models/request_type_option.py

@@ -4,6 +4,7 @@
 """
 from sqlalchemy import and_, func, or_
 from sqlalchemy.sql.expression import true
+from analytics_api.models.available_response_option import AvailableResponseOption as AvailableResponseOptionModel
 from analytics_api.models.survey import Survey as SurveyModel
 from analytics_api.models.response_type_option import ResponseTypeOption as ResponseTypeOptionModel
 from .base_model import BaseModel
@@ -50,6 +51,12 @@ def get_survey_result(
                                .order_by(RequestTypeOption.position)
                                .subquery())
 
+        # Get all the available responses for each question within the survey.
+        available_response = (db.session.query(AvailableResponseOptionModel.request_key,
+                                               AvailableResponseOptionModel.value)
+                              .filter(and_(AvailableResponseOptionModel.survey_id.in_(
+                                  analytics_survey_id), AvailableResponseOptionModel.is_active == true()))
+                              .subquery())
         # Get all the survey responses with the counts for each response specific to a survey id which
         # are in active status.
         survey_response = (db.session.query(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value,
@@ -59,17 +66,25 @@ def get_survey_result(
                            .group_by(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value)
                            .subquery())
 
-        # Combine the data fetched above such that the result has a format as below
-        # - position: is a unique value for each question which helps to get the order of question on the survey
-        # - label: is the the survey question
-        # - value: user selected response for each question
-        # - count: number of time the same value is selected as a response to each question
-        survey_result = (db.session.query((survey_question.c.position).label('position'),
-                                          (survey_question.c.label).label('question'),
-                                          func.json_agg(func.json_build_object('value', survey_response.c.value,
-                                                                               'count', survey_response.c.response))
-                                          .label('result'))
-                         .join(survey_response, survey_response.c.request_key == survey_question.c.key)
-                         .group_by(survey_question.c.position, survey_question.c.label))
+        # Check if there are records in survey_response before executing the final query
+        if db.session.query(survey_response.c.request_key).first():
+            # Combine the data fetched above such that the result has a format as below
+            # - position: is a unique value for each question which helps to get the order of question on the survey
+            # - label: is the the survey question
+            # - value: user selected response for each question
+            # - count: number of time the same value is selected as a response to each question
+            survey_result = (db.session.query((survey_question.c.position).label('position'),
+                                              (survey_question.c.label).label('question'),
+                                              func.json_agg(func.json_build_object(
+                                                  'value', available_response.c.value,
+                                                  'count', func.coalesce(survey_response.c.response, 0)))
+                             .label('result'))
+                             .outerjoin(available_response, survey_question.c.key == available_response.c.request_key)
+                             .outerjoin(survey_response,
+                                        (available_response.c.value == survey_response.c.value) &
+                                        (available_response.c.request_key == survey_response.c.request_key))
+                             .group_by(survey_question.c.position, survey_question.c.label))
 
-        return survey_result.all()
+            return survey_result.all()
+
+        return None  # Return None indicating no records

met-etl/src/etl_project/services/ops/submission_etl_service.py

@@ -286,7 +286,7 @@ def _save_survey(met_etl_session, context, answer_key, component, survey, partic
             # id and the key
             radio_response = EtlResponseTypeOptionModel(
                 survey_id=survey.id,
-                request_key=component['key'],
+                request_key=component['key']+'-'+key,
                 value=answer_label,
                 request_id=component['id']+'-'+key,
                 participant_id=getattr(participant, 'id', None),

met-etl/src/etl_project/services/ops/survey_etl_service.py

@@ -1,3 +1,4 @@
+from analytics_api.models.available_response_option import AvailableResponseOption as EtlAvailableResponseOption
 from analytics_api.models.etlruncycle import EtlRunCycle as EtlRunCycleModel
 from analytics_api.models.request_type_option import RequestTypeOption as EtlRequestTypeOption
 from analytics_api.models.response_type_option import ResponseTypeOption as EtlResponseTypeOptionModel
@@ -121,7 +122,7 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f
         context.log.info('Survey Found without any component in form_json: %s. Skipping it', survey.id)
         return position
 
-    _inactivate_old_questions(session, survey.id)
+    _refresh_questions_and_available_option_status(session, survey.id)
 
     for component in form_components:
         position = position + 1
@@ -144,12 +145,14 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f
             for survey_id in etl_survey:
                 position = _do_etl_survey_inputs(session, survey_id, component, component_type,
                                                  survey_new_runcycleid, position)
+                _load_available_response_option(context, session, survey_id, component, component_type,
+                                                survey_new_runcycleid)
 
     return position
 
 
 # inactivate if record is existing in analytics database
-def _inactivate_old_questions(session, source_survey_id):
+def _refresh_questions_and_available_option_status(session, source_survey_id):
     etl_survey_model = session.query(EtlSurveyModel.id).filter(EtlSurveyModel.source_survey_id == source_survey_id,
                                                                EtlSurveyModel.is_active == False)
     if not etl_survey_model:
@@ -159,6 +162,8 @@ def _inactivate_old_questions(session, source_survey_id):
 
     for survey_id in etl_survey_model:
         session.query(EtlRequestTypeOption).filter(EtlRequestTypeOption.survey_id == survey_id).update(deactive_flag)
+        session.query(EtlAvailableResponseOption).filter(
+            EtlAvailableResponseOption.survey_id == survey_id).update(deactive_flag)
 
 
 def _do_etl_survey_data(session, survey, survey_new_runcycleid):
@@ -215,6 +220,66 @@ def _do_etl_survey_inputs(session, survey_id, component, component_type, survey_
     return position
 
 
+# load data to table available response option
+def _load_available_response_option(context, session, survey_id, component, component_type, survey_new_runcycleid):
+
+    if component_type == FormIoComponentType.SURVEY.value:
+        _load_survey_available_response(session, component, survey_id, survey_new_runcycleid)
+    elif component_type == FormIoComponentType.SELECTLIST.value:
+        _load_selectlist_available_response(session, component, survey_id, survey_new_runcycleid)
+    else:
+        _load_default_available_response(session, component, survey_id, survey_new_runcycleid)
+
+
+def _load_survey_available_response(session, component, survey_id, survey_new_runcycleid):
+    values = component.get('values', None)
+    if not values:
+        return
+
+    questions = component.get('questions', None)
+    if not questions:
+        return
+
+    for question in questions:
+        request_key = component['key'] + '-' + question['value']
+        _do_etl_available_response_data(session, component, survey_id, values,
+                                        request_key, survey_new_runcycleid)
+
+def _load_selectlist_available_response(session, component, survey_id, survey_new_runcycleid):
+    data = component.get('data', None)
+    values = data.get('values', None)
+
+    if not values:
+        return
+
+    request_key = component['key']
+    _do_etl_available_response_data(session, component, survey_id, values,
+                                    request_key, survey_new_runcycleid)
+
+def _load_default_available_response(session, component, survey_id, survey_new_runcycleid):
+    values = component.get('values', None)
+    if not values:
+        return
+
+    request_key = component['key']
+    _do_etl_available_response_data(session, component, survey_id, values,
+                                    request_key, survey_new_runcycleid)
+
+
+def _do_etl_available_response_data(session, component, survey_id, values, request_key, survey_new_runcycleid):
+    for value in values:
+        model_name = EtlAvailableResponseOption(survey_id=survey_id,
+                                                request_key=request_key,
+                                                value=value['label'],
+                                                request_id=component['id'],
+                                                is_active=True,
+                                                runcycle_id=survey_new_runcycleid)
+
+        session.add(model_name)
+
+    session.commit()
+
+
 def _validate_form_type(context, component_type):
     component_type = component_type.lower()