Bug fix for Public Dashboard (#87)

* adding send_report column

* dashboard bug fixes

analytics-api/migrations/versions/00e8d4134dbe_add_send_report_on_engagement.py

@@ -0,0 +1,28 @@
+"""add_send_report_on_engagement
+
+Revision ID: 00e8d4134dbe
+Revises: 812b1f67015a
+Create Date: 2024-09-19 16:41:49.478981
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '00e8d4134dbe'
+down_revision = '812b1f67015a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('engagement', sa.Column('send_report', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('engagement', 'send_report')
+    # ### end Alembic commands ###

analytics-api/requirements.txt

@@ -50,6 +50,6 @@ six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.3.2
-typing_extensions==4.5.0
+typing_extensions==4.12.2
 urllib3==1.26.15
 zipp==3.15.0

analytics-api/src/analytics_api/models/engagement.py

@@ -26,6 +26,7 @@ class Engagement(BaseModel):  # pylint: disable=too-few-public-methods
     geojson = db.Column(db.Text())
     marker_label = db.Column(db.String(30))
     status_name = db.Column(db.String(50))
+    send_report = db.Column(db.Boolean, nullable=True)
 
     @classmethod
     def find_by_source_id(cls, source_identifier: int):

analytics-api/src/analytics_api/models/request_type_option.py

@@ -50,61 +50,66 @@ def get_survey_result(
                                                 RequestTypeOption.display.is_(None))))
                                .order_by(RequestTypeOption.position)
                                .subquery())
+        result = db.session.query(survey_question).first()
 
-        # Get all the available responses for each question within the survey.
-        available_response = (db.session.query(AvailableResponseOptionModel.request_key,
-                                               AvailableResponseOptionModel.value)
-                              .filter(and_(AvailableResponseOptionModel.survey_id.in_(
-                                  analytics_survey_id), AvailableResponseOptionModel.is_active == true()))
-                              .subquery())
-        # Get all the survey responses with the counts for each response specific to a survey id which
-        # are in active status.
-        survey_response = (db.session.query(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value,
-                                            func.count(ResponseTypeOptionModel.request_key).label('response'))
-                           .filter(and_(ResponseTypeOptionModel.survey_id.in_(analytics_survey_id),
-                                        ResponseTypeOptionModel.is_active == true()))
-                           .group_by(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value)
-                           .subquery())
+        if result:
+            # Get all the available responses for each question within the survey.
+            available_response = (db.session.query(AvailableResponseOptionModel.request_key,
+                                                   AvailableResponseOptionModel.value)
+                                  .filter(and_(AvailableResponseOptionModel.survey_id.in_(
+                                      analytics_survey_id), AvailableResponseOptionModel.is_active == true()))
+                                  .subquery())
+            # Get all the survey responses with the counts for each response specific to a survey id which
+            # are in active status.
+            survey_response = (db.session.query(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value,
+                                                func.count(ResponseTypeOptionModel.request_key).label('response'))
+                               .filter(and_(ResponseTypeOptionModel.survey_id.in_(analytics_survey_id),
+                                            ResponseTypeOptionModel.is_active == true()))
+                               .group_by(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value)
+                               .subquery())
 
-        survey_response_exists = db.session.query(survey_response.c.request_key).first()
-        available_response_exists = db.session.query(available_response.c.request_key).first()
+            survey_response_exists = db.session.query(survey_response.c.request_key).first()
+            available_response_exists = db.session.query(available_response.c.request_key).first()
 
-        # Combine the data fetched above such that the result has a format as below
-        # - position: is a unique value for each question which helps to get the order of question on the survey
-        # - label: is the the survey question
-        # - value: user selected response for each question
-        # - count: number of time the same value is selected as a response to each question
+            # Combine the data fetched above such that the result has a format as below
+            # - position: is a unique value for each question which helps to get the order of question on the survey
+            # - label: is the the survey question
+            # - value: user selected response for each question
+            # - count: number of time the same value is selected as a response to each question
 
-        # Check if there are records in survey_response and available_response before executing the final query
-        # which fetches all the available responses along with the corresponding reponses.
-        if survey_response_exists and available_response_exists:
-            survey_result = (db.session.query((survey_question.c.position).label('position'),
-                                              (survey_question.c.label).label('question'),
-                                              func.json_agg(func.json_build_object(
-                                                  'value', available_response.c.value,
-                                                  'count', func.coalesce(survey_response.c.response, 0)))
-                             .label('result'))
-                             .outerjoin(available_response, survey_question.c.key == available_response.c.request_key)
-                             .outerjoin(survey_response,
-                                        (available_response.c.value == survey_response.c.value) &
-                                        (available_response.c.request_key == survey_response.c.request_key),
-                                        full=True)
-                             .group_by(survey_question.c.position, survey_question.c.label))
+            # Check if there are records in survey_response and available_response before executing the final query
+            # which fetches all the available responses along with the corresponding reponses.
+            if survey_response_exists and available_response_exists:
+                survey_result = (db.session.query((survey_question.c.position).label('position'),
+                                                  (survey_question.c.label).label('question'),
+                                                  func.json_agg(func.json_build_object(
+                                                      'value', available_response.c.value,
+                                                      'count', func.coalesce(survey_response.c.response, 0)))
+                                 .label('result'))
+                                 .outerjoin(
+                                     available_response, survey_question.c.key == available_response.c.request_key)
+                                 .outerjoin(survey_response,
+                                            (available_response.c.value == survey_response.c.value) &
+                                            (available_response.c.request_key == survey_response.c.request_key),
+                                            full=True)
+                                 .filter(survey_question.c.position.isnot(None))
+                                 .filter(survey_question.c.label.isnot(None))
+                                 .group_by(survey_question.c.position, survey_question.c.label))
 
-            return survey_result.all()
-        # Check if there are records in survey_response before executing the final query which fetches reponses
-        # even if the available_response table is not yet populated.
-        elif survey_response_exists:
-            survey_result = (db.session.query((survey_question.c.position).label('position'),
-                                            (survey_question.c.label).label('question'),
-                                            func.json_agg(func.json_build_object('value',
-                                                                                survey_response.c.value,
-                                                                                'count',
-                                                                                survey_response.c.response))
-                            .label('result'))
-                            .join(survey_response, survey_response.c.request_key == survey_question.c.key)
-                            .group_by(survey_question.c.position, survey_question.c.label))
+                return survey_result.all()
+            # Check if there are records in survey_response before executing the final query which fetches reponses
+            # even if the available_response table is not yet populated.
+            if survey_response_exists:
+                survey_result = (db.session.query((survey_question.c.position).label('position'),
+                                                  (survey_question.c.label).label('question'),
+                                                  func.json_agg(func.json_build_object('value',
+                                                                                       survey_response.c.value,
+                                                                                       'count',
+                                                                                       survey_response.c.response))
+                                 .label('result'))
+                                 .join(survey_response, survey_response.c.request_key == survey_question.c.key)
+                                 .group_by(survey_question.c.position, survey_question.c.label))
 
-            return survey_result.all()
+                return survey_result.all()
 
         return None  # Return None indicating no records

analytics-api/src/analytics_api/utils/engagement_access_validator.py

@@ -1,5 +1,5 @@
 """Check Engagement Access Service."""
-from sqlalchemy import and_, exists
+from sqlalchemy import and_, exists, or_
 from sqlalchemy.sql.expression import true
 from analytics_api.constants.engagement_status import Status
 from analytics_api.models.db import db
@@ -9,18 +9,29 @@
 
 
 def check_engagement_access(engagement_id):
-    """Check if user has access to get engagement details."""
-    is_engagement_unpublished = db.session.query(
+    """
+    Check if user has access to get engagement details.
+
+    Public users will not be able to access engagement details if the engagement is unpublished or
+    if the send report setting is turned off.
+
+    Staff Users with the `ACCESS_DASHBOARD` role, such as administrators or team members,
+    will always have access to engagement details, regardless of the engagement's visibility settings.
+    """
+    engagement_conditions = db.session.query(
         exists()
         .where(
             and_(
                 EngagementModel.source_engagement_id == engagement_id,
                 EngagementModel.is_active == true(),
-                EngagementModel.status_name == Status.Unpublished.value
+                or_(
+                    EngagementModel.status_name == Status.Unpublished.value,
+                    EngagementModel.send_report.is_(False)
+                )
             )
         )
     ).scalar()
 
     user_roles = set(TokenInfo.get_user_roles())
 
-    return not is_engagement_unpublished or Role.ACCESS_DASHBOARD.value in user_roles
+    return not engagement_conditions or Role.ACCESS_DASHBOARD.value in user_roles

met-api/src/met_api/models/comment.py

@@ -14,6 +14,7 @@
 
 from met_api.constants.comment_status import Status as CommentStatus
 from met_api.constants.engagement_status import Status as EngagementStatus
+from met_api.models.engagement_settings import EngagementSettingsModel
 from met_api.models.pagination_options import PaginationOptions
 from met_api.models.engagement import Engagement
 from met_api.models.report_setting import ReportSetting
@@ -90,13 +91,14 @@ def get_comments_by_survey_id_paginated(cls, survey_id, pagination_options: Pagi
 
     @classmethod
     def get_accepted_comments_by_survey_id_paginated(
-            cls, survey_id, pagination_options: PaginationOptions, search_text='', include_unpublished=False):
+            cls, survey_id, pagination_options: PaginationOptions, search_text='', can_view_all_comments=False):
         """Get comments for closed engagements."""
         query = db.session.query(Comment)\
             .join(Submission, Submission.id == Comment.submission_id)\
             .join(CommentStatusModel, Submission.comment_status_id == CommentStatusModel.id)\
             .join(Survey, Survey.id == Submission.survey_id)\
             .join(Engagement, Engagement.id == Survey.engagement_id)\
+            .join(EngagementSettingsModel, Engagement.id == EngagementSettingsModel.engagement_id)\
             .join(ReportSetting, and_(Comment.survey_id == ReportSetting.survey_id,
                                       Comment.component_id == ReportSetting.question_key))\
             .filter(
@@ -106,8 +108,11 @@ def get_accepted_comments_by_survey_id_paginated(
                     ReportSetting.display == true()
                 ))
 
-        if not include_unpublished:
-            query = query.filter(Engagement.status_id != EngagementStatus.Unpublished.value)
+        if not can_view_all_comments:
+            query = query.filter(
+                Engagement.status_id != EngagementStatus.Unpublished.value,
+                EngagementSettingsModel.send_report.is_(True)
+            )
 
         if search_text:
             query = query.filter(Comment.text.ilike('%' + search_text + '%'))

met-api/src/met_api/services/comment_service.py

@@ -80,11 +80,11 @@ def can_view_unapproved_comments(survey_id: int) -> bool:
     @classmethod
     def get_comments_paginated(cls, survey_id, pagination_options: PaginationOptions, search_text=''):
         """Get comments paginated."""
-        include_unpublished = CommentService.can_view_unapproved_comments(survey_id)
+        can_view_all_comments = CommentService.can_view_unapproved_comments(survey_id)
 
         comment_schema = CommentSchema(many=True, only=('text', 'submission_date', 'label', 'submission_id'))
         items, total = Comment.get_accepted_comments_by_survey_id_paginated(
-            survey_id, pagination_options, search_text, include_unpublished)
+            survey_id, pagination_options, search_text, can_view_all_comments)
         return {
             'items': comment_schema.dump(items),
             'total': total

met-api/src/met_api/services/engagement_service.py

@@ -134,7 +134,7 @@ def close_engagements_due():
         engagements = EngagementModel.close_engagements_due()
         for engagement in engagements:
             engagement_settings: EngagementSettingsModel =\
-            EngagementSettingsModel.find_by_id(engagement.id)
+                EngagementSettingsModel.find_by_id(engagement.id)
             if engagement_settings:
                 if engagement_settings.send_report:
                     EngagementService._send_closeout_emails(engagement)

met-etl/container/dagster/Dockerfile

@@ -3,12 +3,14 @@
 
 FROM python:3.8.5-buster
 
+ARG DAGSTER_VERSION=0.23.15
+
 RUN pip install \
-    dagster \
-    dagster-graphql \
-    dagit \
-    dagster-postgres \
-    dagster-docker
+    dagster==1.7.15 \
+    dagster-graphql==1.7.15 \
+    dagit==1.7.15 \
+    dagster-postgres==${DAGSTER_VERSION} \
+    dagster-docker==${DAGSTER_VERSION}
 
 # Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
 ENV DAGSTER_HOME=/opt/dagster/dagster_home/

met-etl/requirements.txt

@@ -1,6 +1,6 @@
-dagster
-dagit
-dagster-docker
+dagster==1.7.15
+dagit==1.7.15
+dagster-docker==0.23.15
 Flask-Cors==3.0.10
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5

met-etl/src/etl_project/services/ops/engagement_etl_service.py

@@ -1,6 +1,7 @@
 from dagster import Out, Output, op
 from met_api.constants.engagement_status import Status as MetEngagementStatus
 from met_api.models.engagement import Engagement as MetEngagementModel
+from met_api.models.engagement_settings import EngagementSettingsModel
 from met_api.models.engagement_status import EngagementStatus as EngagementStatusModel
 from met_api.models.widget_map import WidgetMap as MetWidgetMap
 from analytics_api.models.engagement import Engagement as EtlEngagementModel
@@ -105,6 +106,9 @@ def load_engagement(context, new_engagements, updated_engagements, engagement_ne
 
             engagement_status = met_session.query(EngagementStatusModel).filter(
                 EngagementStatusModel.id == engagement.status_id).first()
+
+            engagement_send_report = met_session.query(EngagementSettingsModel).filter(
+                EngagementSettingsModel.engagement_id == engagement.id).first()
 
             engagement_model = EtlEngagementModel(name=engagement.name,
                                                   source_engagement_id=engagement.id,
@@ -119,7 +123,8 @@ def load_engagement(context, new_engagements, updated_engagements, engagement_ne
                                                   longitude=longitude,
                                                   geojson=geojson,
                                                   marker_label=marker_label,
-                                                  status_name=engagement_status.status_name
+                                                  status_name=engagement_status.status_name,
+                                                  send_report=engagement_send_report.send_report
                                                   )
             session.add(engagement_model)
             session.commit()