From 7839b65a44e5198b7ca9ce8e4c72cc95c1be44b1 Mon Sep 17 00:00:00 2001 From: Sara Burns Date: Mon, 18 Nov 2024 12:28:17 -0500 Subject: [PATCH] fix: video engagement updates --- tutoraspects/plugin.py | 4 +- .../assets/charts/Course_Info_b2fff8.yaml | 9 ++- .../charts/Video_Engagement_5b00f7.yaml | 19 ++++--- .../charts/Video_Engagement_6b7610.yaml | 20 ++++--- .../Course_Comparison_Dashboard.yaml | 29 +++++----- .../assets/datasets/tag_filter.yaml | 24 ++++---- .../datasets/watched_video_duration.yaml | 57 ++++++++++++------- .../jobs/init/clickhouse/init-clickhouse.sh | 2 +- .../queries/watched_video_duration.sql | 29 +++++++++- 9 files changed, 118 insertions(+), 75 deletions(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index ee80a862c..9432f8fb1 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -385,8 +385,8 @@ # DBT_PROFILE_* settings get passed into the dbt_profile.yml file. # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. - ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v3.33.7"), + ("DBT_REPOSITORY", "https://github.com/saraburns1/aspects-dbt"), + ("DBT_BRANCH", "fixrewatched_again"), ("DBT_SSH_KEY", ""), ("DBT_STATE_DIR", "/app/aspects-dbt/state"), ("DBT_PROFILES_DIR", "/app/aspects/dbt/"), diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml index d0cfff243..af3541b6c 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Info_b2fff8.yaml @@ -50,7 +50,7 @@ params: viz_type: table query_context: datasource: - id: 651 + id: 261 type: table force: false form_data: @@ -61,6 +61,7 @@ query_context: operator: TEMPORAL_RANGE subject: emission_time all_columns: [] + annotation_layers: [] color_pn: false column_config: active_count: @@ -74,9 +75,7 @@ query_context: tag_list: truncateLongCells: true conditional_formatting: [] - dashboards: - - 355 - datasource: 651__table + datasource: 261__table extra_form_data: {} force: false groupby: @@ -98,7 +97,7 @@ query_context: result_type: full row_limit: 50000 show_cell_bars: false - slice_id: 1298 + slice_id: 561 table_timestamp_format: smart_date temporal_columns_lookup: {} time_grain_sqla: P1M diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml index 22364c178..263f8e00a 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_5b00f7.yaml @@ -7,6 +7,7 @@ description: null params: adhoc_filters: [] all_columns: [] + annotation_layers: [] color_pn: true column_config: rewatched_percent: @@ -20,9 +21,10 @@ params: - org metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -34,12 +36,13 @@ params: viz_type: table query_context: datasource: - id: 65 + id: 161 type: table force: false form_data: adhoc_filters: [] all_columns: [] + annotation_layers: [] color_pn: true column_config: rewatched_percent: @@ -47,9 +50,7 @@ query_context: watched_percent: d3NumberFormat: ',.1%' conditional_formatting: [] - dashboards: - - 22 - datasource: 65__table + datasource: 161__table extra_form_data: {} force: false groupby: @@ -57,9 +58,10 @@ query_context: - org metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -68,7 +70,7 @@ query_context: result_type: full row_limit: 50000 show_cell_bars: false - slice_id: 82 + slice_id: 1121 table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table @@ -86,9 +88,10 @@ query_context: filters: [] metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_desc: true orderby: - - num_videos diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml index eb8c74164..0ae32e197 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml @@ -7,6 +7,7 @@ description: null params: adhoc_filters: [] all_columns: [] + annotation_layers: [] color_pn: true column_config: rewatched_percent: @@ -20,21 +21,23 @@ params: - course_name metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] query_mode: aggregate row_limit: 50000 + server_page_length: 10 show_cell_bars: false table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table query_context: datasource: - id: 65 + id: 161 type: table force: false form_data: @@ -48,8 +51,8 @@ query_context: d3NumberFormat: ',.1%' conditional_formatting: [] dashboards: - - 22 - datasource: 65__table + - 241 + datasource: 161__table extra_form_data: {} force: false groupby: @@ -57,9 +60,10 @@ query_context: - course_name metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_by_cols: [] order_desc: true percent_metrics: [] @@ -67,8 +71,9 @@ query_context: result_format: json result_type: full row_limit: 50000 + server_page_length: 10 show_cell_bars: false - slice_id: 145 + slice_id: 641 table_timestamp_format: smart_date temporal_columns_lookup: {} viz_type: table @@ -86,9 +91,10 @@ query_context: filters: [] metrics: - num_videos + - avg_video_length + - num_videos_watched - watched_percent - rewatched_percent - - avg_video_length order_desc: true orderby: - - num_videos diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml index be7c7ec51..3556f7f68 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml @@ -171,12 +171,11 @@ metadata: - '#9EE5E5' - '#D1C6BC' cross_filters_enabled: true - default_filters: '{}' expanded_slices: - '1252': true - '399': true - '422': true - '776': true + '101': true + '1461': true + '461': true + '61': true global_chart_configuration: chartsInScope: - 314 @@ -350,7 +349,7 @@ position: children: [] id: CHART-explore-1467-1 meta: - chartId: 963 + chartId: 1121 height: 50 sliceName: Video Engagement uuid: 5b00f792-233c-4ee6-9411-7df49f4b5061 @@ -365,7 +364,7 @@ position: children: [] id: CHART-explore-1500-1 meta: - chartId: 314 + chartId: 541 height: 50 sliceName: Course Info uuid: b2fff8a7-3ace-4075-9a3f-5fa5b99f81bb @@ -380,7 +379,7 @@ position: children: [] id: CHART-explore-1501-1 meta: - chartId: 757 + chartId: 201 height: 50 sliceName: Course Info uuid: ccdd7d98-4722-490b-a0b2-077380a7c7eb @@ -395,7 +394,7 @@ position: children: [] id: CHART-explore-1502-1 meta: - chartId: 1154 + chartId: 161 height: 50 sliceName: Enrollment Counts uuid: 7eaf5bc2-0067-424b-a50b-607cf54c086a @@ -410,7 +409,7 @@ position: children: [] id: CHART-explore-1503-1 meta: - chartId: 407 + chartId: 1141 height: 50 sliceName: Enrollment Counts uuid: 8230a366-957d-4667-a2ff-3cbcb2f25f60 @@ -425,7 +424,7 @@ position: children: [] id: CHART-explore-1504-1 meta: - chartId: 399 + chartId: 461 height: 50 sliceName: Learner Performance Breakdown uuid: b73b0a5c-4861-402e-a39c-d73333a4d911 @@ -440,7 +439,7 @@ position: children: [] id: CHART-explore-1505-1 meta: - chartId: 776 + chartId: 101 height: 50 sliceName: Learner Performance Breakdown uuid: d414b144-467b-456b-b899-9cfb3579faba @@ -455,7 +454,7 @@ position: children: [] id: CHART-explore-1506-1 meta: - chartId: 1252 + chartId: 61 height: 50 sliceName: Learner Performance uuid: b518f341-9ed4-4c23-bb88-6c1d497aa260 @@ -470,7 +469,7 @@ position: children: [] id: CHART-explore-1507-1 meta: - chartId: 422 + chartId: 1461 height: 50 sliceName: Learner Performance uuid: b0e170ce-26f1-4452-a3c0-49b41fa6e529 @@ -485,7 +484,7 @@ position: children: [] id: CHART-explore-1508-1 meta: - chartId: 1407 + chartId: 641 height: 50 sliceName: Video Engagement uuid: 6b7610a7-41d0-4d14-be81-62a5a8f22d62 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml index 99c8f06f1..b7824dbd2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/tag_filter.yaml @@ -2,6 +2,18 @@ _file_name: tag_filter.yaml always_filter_main_dttm: false cache_timeout: null columns: +- advanced_data_type: null + column_name: course_key + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Course Key - advanced_data_type: null column_name: id description: null @@ -38,18 +50,6 @@ columns: python_date_format: null type: String verbose_name: Tag -- advanced_data_type: null - column_name: course_key - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: String - verbose_name: Course Key database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml index d744a10bf..128ddda43 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml @@ -6,34 +6,31 @@ columns: column_name: rewatched_time description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Decimal(38, 2) + type: Decimal(9, 2) verbose_name: Rewatched Time - advanced_data_type: null column_name: watched_time description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Decimal(38, 2) + type: Decimal(9, 2) verbose_name: Watched Time - advanced_data_type: null column_name: course_name description: null expression: '' - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -45,8 +42,7 @@ columns: column_name: video_count description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -58,8 +54,7 @@ columns: column_name: video_duration description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -71,8 +66,7 @@ columns: column_name: actor_id description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -84,8 +78,7 @@ columns: column_name: course_key description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -97,8 +90,7 @@ columns: column_name: course_run description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -110,8 +102,7 @@ columns: column_name: org description: null expression: null - extra: - warning_markdown: null + extra: {} filterable: true groupby: true is_active: true @@ -119,6 +110,18 @@ columns: python_date_format: null type: String verbose_name: Org +- advanced_data_type: null + column_name: object_id + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Object Id database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null @@ -130,7 +133,7 @@ metrics: - currency: null d3format: null description: null - expression: if(sum(video_duration) > 0,(sum(rewatched_time)/count(actor_id))/sum(video_duration),0) + expression: if(sum(video_duration) > 0,sum(rewatched_time)/sum(video_duration),0) extra: warning_markdown: '' metric_name: rewatched_percent @@ -140,7 +143,17 @@ metrics: - currency: null d3format: null description: null - expression: if(sum(watched_time)>0,(sum(watched_time)/count(actor_id))/sum(video_duration),0) + expression: count(distinct object_id) + extra: + warning_markdown: '' + metric_name: num_videos_watched + metric_type: null + verbose_name: Number of Videos Watched + warning_text: null +- currency: null + d3format: null + description: null + expression: if(sum(watched_time)>0,sum(watched_time)/sum(video_duration),0) extra: warning_markdown: '' metric_name: watched_percent diff --git a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh index 92d1721ff..715a5a26e 100644 --- a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh +++ b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh @@ -48,7 +48,7 @@ GRANT {{ ON_CLUSTER }} SELECT ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CL GRANT {{ ON_CLUSTER }} INSERT, SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_CMS_USER }}'; GRANT {{ ON_CLUSTER }} SELECT ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; -GRANT {{ ON_CLUSTER }} CREATE TABLE, DROP TABLE, CREATE VIEW, DROP VIEW, SELECT, INSERT, UPDATE, DELETE ON {{ DBT_PROFILE_TARGET_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; +GRANT {{ ON_CLUSTER }} CREATE TABLE, DROP TABLE, CREATE VIEW, DROP VIEW, SELECT, INSERT, UPDATE, DELETE, dictGet ON {{ DBT_PROFILE_TARGET_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} SELECT, DROP TABLE, DROP VIEW ON {{ ASPECTS_XAPI_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} INSERT, SELECT, DELETE ON {{ ASPECTS_EVENT_SINK_DATABASE }}.* TO '{{ ASPECTS_CLICKHOUSE_CMS_USER }}'; diff --git a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql index 263bf950d..2d4abb73c 100644 --- a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql +++ b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql @@ -1,3 +1,24 @@ +{% raw -%} +with + course_keys as ( + select [] as course_key + {% if filter_values("course_name") != [] %} + union all + select array(course_key) as course_key + from {% endraw -%} {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names {% raw -%} + where course_name in {{ filter_values("course_name") | where_in }} + {% endif %} + {% if filter_values("tag") != [] %} + union distinct + select array(course_key) as course_key + from + {% endraw -%} {{ DBT_PROFILE_TARGET_DATABASE }}.most_recent_course_tags {% raw -%} + where + tag + in (select replaceAll(arrayJoin({{ filter_values("tag") }}), '- ', '')) + {% endif %} + ) + {%- endraw %} select names.org as org, names.course_key as course_key, @@ -7,12 +28,14 @@ select video_count, video_duration, watched_time, - rewatched_time + rewatched_time, + object_id from {{ DBT_PROFILE_TARGET_DATABASE }}.watched_video_duration( {% raw -%} - org_filter ={{ filter_values("org") }}, - course_name_filter ={{ filter_values("course_name") }} + org_filter = coalesce({{ filter_values("org") }}, []), + course_key_filter + = coalesce((select array_concat_agg(course_key) from course_keys), []) {%- endraw %} ) as a left join