From 9e93cb27ce931f45eee9d62285f0103ff9a55889 Mon Sep 17 00:00:00 2001 From: VineetBala-AOT <90332175+VineetBala-AOT@users.noreply.github.com> Date: Thu, 23 Nov 2023 15:08:36 -0800 Subject: [PATCH] Changes to have all answer options to appear in the report (#24) * adding table to store available option in analytic * etl changes to load available option table * fixing linting * fixing linting --- ...9e8ff_adding_available_response_options.py | 41 +++++++++++ .../models/available_response_option.py | 12 ++++ .../models/request_type_option.py | 41 +++++++---- .../services/ops/submission_etl_service.py | 2 +- .../services/ops/survey_etl_service.py | 69 ++++++++++++++++++- 5 files changed, 149 insertions(+), 16 deletions(-) create mode 100644 analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py create mode 100644 analytics-api/src/analytics_api/models/available_response_option.py diff --git a/analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py b/analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py new file mode 100644 index 000000000..679276848 --- /dev/null +++ b/analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py @@ -0,0 +1,41 @@ +"""adding_available_response_options + +Revision ID: e7fdf769e8ff +Revises: 3a705c422892 +Create Date: 2023-11-21 12:45:34.871602 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'e7fdf769e8ff' +down_revision = '3a705c422892' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('available_response_option', + sa.Column('created_date', sa.DateTime(), nullable=True), + sa.Column('updated_date', sa.DateTime(), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('participant_id', sa.Integer(), nullable=True), + sa.Column('request_key', sa.String(length=100), nullable=False), + sa.Column('value', sa.Text(), nullable=True), + sa.Column('request_id', sa.String(length=20), nullable=True), + sa.Column('survey_id', sa.Integer(), nullable=False), + sa.Column('runcycle_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['survey_id'], ['survey.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', 'request_key') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('available_response_option') + # ### end Alembic commands ### diff --git a/analytics-api/src/analytics_api/models/available_response_option.py b/analytics-api/src/analytics_api/models/available_response_option.py new file mode 100644 index 000000000..0e406a4ed --- /dev/null +++ b/analytics-api/src/analytics_api/models/available_response_option.py @@ -0,0 +1,12 @@ +"""available_response_option model class. + +Manages the Available Options for a option type questions on a survey +""" +from .base_model import BaseModel +from .response_mixin import ResponseMixin + + +class AvailableResponseOption(BaseModel, ResponseMixin): # pylint: disable=too-few-public-methods + """Definition of the Available Response Options entity.""" + + __tablename__ = 'available_response_option' diff --git a/analytics-api/src/analytics_api/models/request_type_option.py b/analytics-api/src/analytics_api/models/request_type_option.py index e59e86c7d..362f64b0c 100644 --- a/analytics-api/src/analytics_api/models/request_type_option.py +++ b/analytics-api/src/analytics_api/models/request_type_option.py @@ -4,6 +4,7 @@ """ from sqlalchemy import and_, func, or_ from sqlalchemy.sql.expression import true +from analytics_api.models.available_response_option import AvailableResponseOption as AvailableResponseOptionModel from analytics_api.models.survey import Survey as SurveyModel from analytics_api.models.response_type_option import ResponseTypeOption as ResponseTypeOptionModel from .base_model import BaseModel @@ -50,6 +51,12 @@ def get_survey_result( .order_by(RequestTypeOption.position) .subquery()) + # Get all the available responses for each question within the survey. + available_response = (db.session.query(AvailableResponseOptionModel.request_key, + AvailableResponseOptionModel.value) + .filter(and_(AvailableResponseOptionModel.survey_id.in_( + analytics_survey_id), AvailableResponseOptionModel.is_active == true())) + .subquery()) # Get all the survey responses with the counts for each response specific to a survey id which # are in active status. survey_response = (db.session.query(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value, @@ -59,17 +66,25 @@ def get_survey_result( .group_by(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value) .subquery()) - # Combine the data fetched above such that the result has a format as below - # - position: is a unique value for each question which helps to get the order of question on the survey - # - label: is the the survey question - # - value: user selected response for each question - # - count: number of time the same value is selected as a response to each question - survey_result = (db.session.query((survey_question.c.position).label('position'), - (survey_question.c.label).label('question'), - func.json_agg(func.json_build_object('value', survey_response.c.value, - 'count', survey_response.c.response)) - .label('result')) - .join(survey_response, survey_response.c.request_key == survey_question.c.key) - .group_by(survey_question.c.position, survey_question.c.label)) + # Check if there are records in survey_response before executing the final query + if db.session.query(survey_response.c.request_key).first(): + # Combine the data fetched above such that the result has a format as below + # - position: is a unique value for each question which helps to get the order of question on the survey + # - label: is the the survey question + # - value: user selected response for each question + # - count: number of time the same value is selected as a response to each question + survey_result = (db.session.query((survey_question.c.position).label('position'), + (survey_question.c.label).label('question'), + func.json_agg(func.json_build_object( + 'value', available_response.c.value, + 'count', func.coalesce(survey_response.c.response, 0))) + .label('result')) + .outerjoin(available_response, survey_question.c.key == available_response.c.request_key) + .outerjoin(survey_response, + (available_response.c.value == survey_response.c.value) & + (available_response.c.request_key == survey_response.c.request_key)) + .group_by(survey_question.c.position, survey_question.c.label)) - return survey_result.all() + return survey_result.all() + + return None # Return None indicating no records diff --git a/met-etl/src/etl_project/services/ops/submission_etl_service.py b/met-etl/src/etl_project/services/ops/submission_etl_service.py index 24c46d8e7..059ce4ffd 100644 --- a/met-etl/src/etl_project/services/ops/submission_etl_service.py +++ b/met-etl/src/etl_project/services/ops/submission_etl_service.py @@ -286,7 +286,7 @@ def _save_survey(met_etl_session, context, answer_key, component, survey, partic # id and the key radio_response = EtlResponseTypeOptionModel( survey_id=survey.id, - request_key=component['key'], + request_key=component['key']+'-'+key, value=answer_label, request_id=component['id']+'-'+key, participant_id=getattr(participant, 'id', None), diff --git a/met-etl/src/etl_project/services/ops/survey_etl_service.py b/met-etl/src/etl_project/services/ops/survey_etl_service.py index 46adfd9b0..d0ab4085f 100644 --- a/met-etl/src/etl_project/services/ops/survey_etl_service.py +++ b/met-etl/src/etl_project/services/ops/survey_etl_service.py @@ -1,3 +1,4 @@ +from analytics_api.models.available_response_option import AvailableResponseOption as EtlAvailableResponseOption from analytics_api.models.etlruncycle import EtlRunCycle as EtlRunCycleModel from analytics_api.models.request_type_option import RequestTypeOption as EtlRequestTypeOption from analytics_api.models.response_type_option import ResponseTypeOption as EtlResponseTypeOptionModel @@ -121,7 +122,7 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f context.log.info('Survey Found without any component in form_json: %s. Skipping it', survey.id) return position - _inactivate_old_questions(session, survey.id) + _refresh_questions_and_available_option_status(session, survey.id) for component in form_components: position = position + 1 @@ -144,12 +145,14 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f for survey_id in etl_survey: position = _do_etl_survey_inputs(session, survey_id, component, component_type, survey_new_runcycleid, position) + _load_available_response_option(context, session, survey_id, component, component_type, + survey_new_runcycleid) return position # inactivate if record is existing in analytics database -def _inactivate_old_questions(session, source_survey_id): +def _refresh_questions_and_available_option_status(session, source_survey_id): etl_survey_model = session.query(EtlSurveyModel.id).filter(EtlSurveyModel.source_survey_id == source_survey_id, EtlSurveyModel.is_active == False) if not etl_survey_model: @@ -159,6 +162,8 @@ def _inactivate_old_questions(session, source_survey_id): for survey_id in etl_survey_model: session.query(EtlRequestTypeOption).filter(EtlRequestTypeOption.survey_id == survey_id).update(deactive_flag) + session.query(EtlAvailableResponseOption).filter( + EtlAvailableResponseOption.survey_id == survey_id).update(deactive_flag) def _do_etl_survey_data(session, survey, survey_new_runcycleid): @@ -215,6 +220,66 @@ def _do_etl_survey_inputs(session, survey_id, component, component_type, survey_ return position +# load data to table available response option +def _load_available_response_option(context, session, survey_id, component, component_type, survey_new_runcycleid): + + if component_type == FormIoComponentType.SURVEY.value: + _load_survey_available_response(session, component, survey_id, survey_new_runcycleid) + elif component_type == FormIoComponentType.SELECTLIST.value: + _load_selectlist_available_response(session, component, survey_id, survey_new_runcycleid) + else: + _load_default_available_response(session, component, survey_id, survey_new_runcycleid) + + +def _load_survey_available_response(session, component, survey_id, survey_new_runcycleid): + values = component.get('values', None) + if not values: + return + + questions = component.get('questions', None) + if not questions: + return + + for question in questions: + request_key = component['key'] + '-' + question['value'] + _do_etl_available_response_data(session, component, survey_id, values, + request_key, survey_new_runcycleid) + +def _load_selectlist_available_response(session, component, survey_id, survey_new_runcycleid): + data = component.get('data', None) + values = data.get('values', None) + + if not values: + return + + request_key = component['key'] + _do_etl_available_response_data(session, component, survey_id, values, + request_key, survey_new_runcycleid) + +def _load_default_available_response(session, component, survey_id, survey_new_runcycleid): + values = component.get('values', None) + if not values: + return + + request_key = component['key'] + _do_etl_available_response_data(session, component, survey_id, values, + request_key, survey_new_runcycleid) + + +def _do_etl_available_response_data(session, component, survey_id, values, request_key, survey_new_runcycleid): + for value in values: + model_name = EtlAvailableResponseOption(survey_id=survey_id, + request_key=request_key, + value=value['label'], + request_id=component['id'], + is_active=True, + runcycle_id=survey_new_runcycleid) + + session.add(model_name) + + session.commit() + + def _validate_form_type(context, component_type): component_type = component_type.lower()