From 4bac66d19c757e6cb229844f0e3d13806275ab81 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 14:28:27 -0500 Subject: [PATCH 01/11] chore: automatically refresh datasets on import --- .../apps/superset/pythonpath/create_assets.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index d82e8c6d3..2ee5b5b9b 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -12,12 +12,15 @@ from pathlib import Path from superset import security_manager +from superset.commands.dataset.refresh import RefreshDatasetCommand from superset.examples.utils import load_configs_from_directory from superset.extensions import db from superset.models.dashboard import Dashboard +from superset.connectors.sqla.models import SqlaTable from superset.utils.database import get_or_create_db from superset.models.embedded_dashboard import EmbeddedDashboard from pythonpath.localization import get_translation +from flask import g BASE_DIR = "/app/assets/superset" @@ -91,6 +94,7 @@ def create_assets(): import_assets() update_dashboard_roles(roles) update_embeddable_uuids() + update_datasets() def import_databases(): @@ -282,5 +286,20 @@ def update_embeddable_uuids(): db.session.commit() +def update_datasets(): + """Update the datasets""" + print("Refreshing datasets") + g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}") + datasets = ( + db.session.query(SqlaTable).all() + ) + for dataset in datasets: + print(f"Refreshing dataset {dataset.table_name}") + try: + RefreshDatasetCommand(dataset.id).run() + except Exception as e: + print(f"Failed to refresh dataset {dataset.table_name}") + + if __name__ == "__main__": main() From 32510b653b12e41b896caa8cd96fb8a158f58582 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 15:16:00 -0500 Subject: [PATCH 02/11] feat: upgrade superset to 3.1.2 --- .../templates/aspects/build/aspects-superset/Dockerfile | 2 +- .../templates/aspects/build/aspects-superset/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile b/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile index 7e6f03cad..1bc0d6a4e 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile +++ b/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile @@ -3,7 +3,7 @@ # https://github.com/apache/superset/releases # https://github.com/apache/superset/blob/master/Dockerfile # https://superset.apache.org/docs/databases/installing-database-drivers -FROM apache/superset:3.0.1 +FROM apache/superset:3.1.2 USER root diff --git a/tutoraspects/templates/aspects/build/aspects-superset/requirements.txt b/tutoraspects/templates/aspects/build/aspects-superset/requirements.txt index ebb4ed7c9..da4a4f4b9 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/requirements.txt +++ b/tutoraspects/templates/aspects/build/aspects-superset/requirements.txt @@ -1,4 +1,4 @@ authlib # OAuth2 -clickhouse-connect>0.5,<0.6 +clickhouse-connect>0.5,<1.0 sentry-sdk[flask] urllib3>=1.26.15,<2 From 2bd5fd7864af8b38567d973f1a3244ccb656d923 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 16:05:53 -0500 Subject: [PATCH 03/11] fix: refresh datasets that are not empty --- .../apps/superset/pythonpath/create_assets.py | 18 +++++++++++++++++- .../superset/pythonpath/performance_metrics.py | 2 +- .../assets/datasets/dashboards.yaml | 2 +- .../assets/datasets/superset_action_log.yaml | 2 +- .../fact_learner_problem_course_summary.sql | 8 ++++---- .../queries/fact_pageview_engagement.sql | 4 ++-- .../openedx-assets/queries/posts_per_user.sql | 4 ++-- 7 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index 2ee5b5b9b..93caf04f3 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -21,6 +21,7 @@ from superset.models.embedded_dashboard import EmbeddedDashboard from pythonpath.localization import get_translation from flask import g +from superset.connectors.sqla.utils import get_columns_description BASE_DIR = "/app/assets/superset" @@ -294,12 +295,27 @@ def update_datasets(): db.session.query(SqlaTable).all() ) for dataset in datasets: - print(f"Refreshing dataset {dataset.table_name}") try: + if dataset_is_empty(dataset): + print(f"Dataset {dataset.table_name} is empty") + continue RefreshDatasetCommand(dataset.id).run() + print(f"Refreshing dataset {dataset.table_name}") except Exception as e: print(f"Failed to refresh dataset {dataset.table_name}") +def dataset_is_empty(dataset): + """Check if a dataset is empty""" + sql = dataset.get_template_processor().process_template( + dataset.sql, **dataset.template_params_dict + ) + try: + columns = get_columns_description(dataset.database, dataset.schema, sql) + except Exception as e: + print(f"Failed to get columns for dataset {dataset.table_name}: {e}") + return True + return columns == [] + if __name__ == "__main__": main() diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index 7d3746b22..e0f19c1ba 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -14,7 +14,7 @@ import sqlparse from flask import g from superset import security_manager -from superset.charts.data.commands.get_data_command import ChartDataCommand +from superset.commands.chart.data.get_data_command import ChartDataCommand from superset.charts.schemas import ChartDataQueryContextSchema from superset.extensions import db from superset.models.dashboard import Dashboard diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml index 1d33df5d8..fbb536a3b 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml @@ -227,7 +227,7 @@ normalize_columns: true offset: 0 params: null schema: superset -sql: 'select * from ' +sql: 'select * from dashboards' table_name: dashboards template_params: null uuid: 3dee777a-3dd4-4fcb-a749-4d13651b3444 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml index c0e73bc71..988a65138 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/superset_action_log.yaml @@ -190,7 +190,7 @@ metrics: normalize_columns: false offset: 0 params: null -schema: main +schema: superset sql: |- {% filter indent(width=2) %}{% include 'openedx-assets/queries/superset_action_log.sql' %}{% endfilter %} table_name: superset_action_log diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index 19d27b027..0046b1e91 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -86,10 +86,10 @@ WITH problem_responses AS ( FROM int_problem_results WHERE 1=1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} {% endraw %} @@ -109,10 +109,10 @@ WITH problem_responses AS ( FROM {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints WHERE 1=1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} {% endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql index 8e83fb206..d3f183c3c 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql @@ -20,10 +20,10 @@ with where 1 = 1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and visited_on > date('{{ from_dttm }}') {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and visited_on < date('{{ to_dttm }}') {% endif %} {% endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql b/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql index b8224d6f6..6c879934e 100644 --- a/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql +++ b/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql @@ -10,10 +10,10 @@ from where 1=1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} {% endraw %} From 5a0c1ae77e68f625061d3f674af4de920cd9a0e5 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 16:31:48 -0500 Subject: [PATCH 04/11] chore: re-serialize superset assets --- .../apps/superset/pythonpath/create_assets.py | 1 - .../charts/Course_Grade_Distribution.yaml | 1 + .../Currently_Enrolled_Learners_Per_Day.yaml | 1 + .../assets/charts/Distinct_forum_users.yaml | 1 + .../charts/Distribution_Of_Attempts.yaml | 1 + ...tribution_Of_Hints_Per_Correct_Answer.yaml | 1 + .../Distribution_Of_Problem_Grades.yaml | 1 + .../charts/Distribution_Of_Responses.yaml | 1 + .../Enrollments_By_Enrollment_Mode.yaml | 1 + .../assets/charts/Posts_per_user.yaml | 1 + .../assets/charts/Responses_Per_Problem.yaml | 1 + .../Transcripts_Captions_Per_Video.yaml | 1 + .../assets/charts/Watched_Video_Segments.yaml | 1 + .../assets/charts/Watches_Per_Video.yaml | 1 + .../dashboards/Instructor_Dashboard.yaml | 33 ++++---- .../assets/datasets/course_names.yaml | 9 ++- .../assets/datasets/dim_course_problems.yaml | 15 ++-- .../assets/datasets/dim_course_videos.yaml | 15 ++-- .../assets/datasets/fact_course_grades.yaml | 76 +++++++++++-------- .../assets/datasets/fact_enrollments.yaml | 41 +++++----- .../datasets/fact_enrollments_by_day.yaml | 41 +++++----- .../datasets/fact_forum_interactions.yaml | 21 ++--- .../fact_learner_problem_course_summary.yaml | 23 +++--- .../fact_learner_problem_summary.yaml | 32 +++++--- .../datasets/fact_pageview_engagement.yaml | 1 + .../assets/datasets/fact_problem_grades.yaml | 76 ++++++++++++------- .../datasets/fact_problem_responses.yaml | 56 +++++++++++--- .../datasets/fact_transcript_usage.yaml | 28 ++++--- .../assets/datasets/fact_video_plays.yaml | 28 ++++--- .../datasets/fact_watched_video_segments.yaml | 23 +++--- .../assets/datasets/hints_per_success.yaml | 26 +++++-- .../assets/datasets/posts_per_user.yaml | 22 ++++-- 32 files changed, 360 insertions(+), 220 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index 93caf04f3..ce7b53a96 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -263,7 +263,6 @@ def update_dashboard_roles(roles): dashboard = db.session.query(Dashboard).filter_by(uuid=dashboard_uuid).one() print("Importing dashboard roles", dashboard_uuid, role_ids) dashboard.roles = role_ids - dashboard.published = True if owners: dashboard.owners = owners db.session.commit() diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml index 679a0c2a6..9c0918a0b 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml @@ -12,6 +12,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml index 0695c2526..4559e8b59 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml @@ -24,6 +24,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: enrollment_status_date + annotation_layers: [] color_picker: a: 1 b: 135 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml index da88796b5..60e2c2757 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml @@ -11,6 +11,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] extra_form_data: {} header_font_size: 0.4 metric: diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml index 51841b393..e0f5db329 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml @@ -17,6 +17,7 @@ params: operatorId: EQUALS sqlExpression: null subject: success + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml index ba11490e6..86559e5a7 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml @@ -9,6 +9,7 @@ description: This chart displays counts of the number of times hints (including group everyone into "0". params: adhoc_filters: [] + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml index 7e32dd219..59a8beb21 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml @@ -13,6 +13,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml index 1a3702359..f9deadfc6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml @@ -13,6 +13,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml index cd34a4bb7..e17c95e51 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml @@ -23,6 +23,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] color_scheme: supersetColors date_format: smart_date extra_form_data: {} diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml index 3382ff2cf..e36a74119 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml @@ -6,6 +6,7 @@ dataset_uuid: a2823d2e-54c4-4378-98c2-817f000c14ab description: null params: adhoc_filters: [] + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml index d16ddf9ea..e0e694160 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml @@ -8,6 +8,7 @@ description: The number of students who have responded to a question, and whethe but this chart counts each student only once per question. params: adhoc_filters: [] + annotation_layers: [] bar_stacked: true bottom_margin: auto color_scheme: supersetColors diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml index 0c26e6069..775efaed7 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml @@ -12,6 +12,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml index 87d2df9d2..af76d8415 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml @@ -12,6 +12,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: started_at + annotation_layers: [] bar_stacked: true bottom_margin: auto color_scheme: supersetColors diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml index c847530c1..8ac176b84 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml @@ -13,6 +13,7 @@ params: expressionType: SIMPLE operator: TEMPORAL_RANGE subject: emission_time + annotation_layers: [] bottom_margin: auto color_scheme: supersetColors columns: [] diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml index 200dbec5e..bd12718cd 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml @@ -1,6 +1,8 @@ _file_name: Instructor_Dashboard.yaml _roles: - '{{ SUPERSET_ROLES_MAPPING.instructor }}' +certification_details: null +certified_by: null css: '' dashboard_title: Instructor Dashboard description: null @@ -420,7 +422,7 @@ position: children: [] id: CHART-AZZnl_lpMv meta: - chartId: 229 + chartId: 21 height: 50 sliceName: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 @@ -435,7 +437,7 @@ position: children: [] id: CHART-GFCO8s2cxv meta: - chartId: 625 + chartId: 19 height: 50 sliceName: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 @@ -450,7 +452,7 @@ position: children: [] id: CHART-GJJ8VYQ03v meta: - chartId: 783 + chartId: 6 height: 50 sliceName: Posts per user uuid: bc191ce7-f39d-48db-86a9-d19949f4211d @@ -465,7 +467,7 @@ position: children: [] id: CHART-Jr-gNVms2Q meta: - chartId: 114 + chartId: 12 height: 50 sliceName: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa @@ -480,7 +482,7 @@ position: children: [] id: CHART-OMy4wjRBWt meta: - chartId: 682 + chartId: 27 height: 50 sliceName: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 @@ -495,7 +497,7 @@ position: children: [] id: CHART-RTO33WE9FH meta: - chartId: 597 + chartId: 15 height: 50 sliceName: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed @@ -510,7 +512,7 @@ position: children: [] id: CHART-Tej2oLPBAl meta: - chartId: 701 + chartId: 37 height: 50 sliceName: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 @@ -555,7 +557,7 @@ position: children: [] id: CHART-evjVO-ZSSd meta: - chartId: 156 + chartId: 38 height: 50 sliceName: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 @@ -570,7 +572,7 @@ position: children: [] id: CHART-j3trfNh8_p meta: - chartId: 325 + chartId: 29 height: 50 sliceName: Pageview engagement by section/subsection uuid: 366a8193-30c3-4aaf-a1ac-360609dfa0ed @@ -600,7 +602,7 @@ position: children: [] id: CHART-lTr8DL3XuI meta: - chartId: 695 + chartId: 4 height: 50 sliceName: Distribution Of Hints Per Correct Answer uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d @@ -615,7 +617,7 @@ position: children: [] id: CHART-o56v9yEe2I meta: - chartId: 770 + chartId: 31 height: 50 sliceName: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 @@ -630,7 +632,7 @@ position: children: [] id: CHART-qG1WaGKl_b meta: - chartId: 154 + chartId: 34 height: 50 sliceName: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e @@ -645,7 +647,7 @@ position: children: [] id: CHART-rnb6PSwCOS meta: - chartId: 339 + chartId: 16 height: 50 sliceName: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c @@ -660,7 +662,7 @@ position: children: [] id: CHART-tWnaoVNNTH meta: - chartId: 327 + chartId: 11 height: 50 sliceName: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c @@ -675,7 +677,7 @@ position: children: [] id: CHART-w-k4N2T_L8 meta: - chartId: 516 + chartId: 2 height: 50 sliceName: Course Grade Distribution uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e @@ -1130,6 +1132,7 @@ position: parents: - ROOT_ID type: TABS +published: true slug: instructor-dashboard uuid: 1d6bf904-f53f-47fd-b1c9-6cd7e284d286 version: 1.0.0 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml index be4192ecd..b31d9f161 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml @@ -1,10 +1,11 @@ _file_name: course_names.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_problems.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_problems.yaml index 88b407fea..510f20d04 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_problems.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_problems.yaml @@ -1,10 +1,11 @@ _file_name: dim_course_problems.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: problem_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: problem_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: problem_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_videos.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_videos.yaml index 8b3c32f04..27711de91 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_videos.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dim_course_videos.yaml @@ -1,10 +1,11 @@ _file_name: dim_course_videos.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: video_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: video_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: video_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_course_grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_course_grades.yaml index 63f976309..faf55e872 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_course_grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_course_grades.yaml @@ -1,10 +1,35 @@ _file_name: fact_course_grades.yaml +always_filter_main_dttm: false cache_timeout: null columns: +- advanced_data_type: null + column_name: grade_type + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Nullable(String) + verbose_name: Grade Type +- advanced_data_type: null + column_name: scaled_score + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: Scaled Score - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +41,7 @@ columns: - advanced_data_type: null column_name: grade_bucket description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +77,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +89,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +101,7 @@ columns: - advanced_data_type: null column_name: entity_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -85,34 +110,10 @@ columns: python_date_format: null type: String verbose_name: Entity Name -- advanced_data_type: null - column_name: scaled_score - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: Float - verbose_name: Scaled Score -- advanced_data_type: null - column_name: grade_type - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: String - verbose_name: Grade Type - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -127,8 +128,17 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments.yaml index dc25b95aa..e5ae955c3 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments.yaml @@ -1,22 +1,11 @@ _file_name: fact_enrollments.yaml +always_filter_main_dttm: false cache_timeout: null columns: -- advanced_data_type: null - column_name: enrollment_mode - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: LowCardinality(String) - verbose_name: Enrollment Mode - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +17,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +53,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -73,10 +62,22 @@ columns: python_date_format: null type: DateTime verbose_name: Emission Time +- advanced_data_type: null + column_name: enrollment_mode + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Enrollment Mode - advanced_data_type: null column_name: enrollment_status description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -103,7 +104,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: - currency: null d3format: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml index 08adfa508..49b1fa9e6 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml @@ -1,10 +1,11 @@ _file_name: fact_enrollments_by_day.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: enrollment_status_date description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -13,22 +14,10 @@ columns: python_date_format: null type: Date verbose_name: Enrollment Status Date -- advanced_data_type: null - column_name: enrollment_mode - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: LowCardinality(String) - verbose_name: Enrollment Mode - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -73,10 +62,22 @@ columns: python_date_format: null type: String verbose_name: Course Run +- advanced_data_type: null + column_name: enrollment_mode + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Enrollment Mode - advanced_data_type: null column_name: enrollment_status description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -103,7 +104,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: enrollment_status_date metrics: - currency: null d3format: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_forum_interactions.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_forum_interactions.yaml index cfe1de222..b11ddfe4c 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_forum_interactions.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_forum_interactions.yaml @@ -1,10 +1,11 @@ _file_name: fact_forum_interactions.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: event_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: object_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: verb_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -100,7 +101,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -115,7 +116,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: - currency: null d3format: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_course_summary.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_course_summary.yaml index 026bbc1c5..2e0bb351d 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_course_summary.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_course_summary.yaml @@ -1,10 +1,11 @@ _file_name: fact_learner_problem_course_summary.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: num_answers_displayed description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: num_hints_displayed description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: problem_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: problem_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -100,7 +101,7 @@ columns: - advanced_data_type: null column_name: attempts description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -112,7 +113,7 @@ columns: - advanced_data_type: null column_name: success description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -124,7 +125,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml index 25790f894..3351a9b24 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml @@ -1,10 +1,11 @@ _file_name: fact_learner_problem_summary.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: num_answers_displayed description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: num_hints_displayed description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: problem_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: problem_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -100,7 +101,7 @@ columns: - advanced_data_type: null column_name: attempts description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -112,7 +113,7 @@ columns: - advanced_data_type: null column_name: success description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -124,7 +125,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -141,6 +142,15 @@ fetch_values_predicate: null filter_select_enabled: false main_dttm_col: null metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml index 779c0fa31..3f2f32f9d 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml @@ -1,4 +1,5 @@ _file_name: fact_pageview_engagement.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml index 947105640..f0e3e9010 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml @@ -1,10 +1,35 @@ _file_name: fact_problem_grades.yaml +always_filter_main_dttm: false cache_timeout: null columns: +- advanced_data_type: null + column_name: grade_type + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Nullable(String) + verbose_name: Grade Type +- advanced_data_type: null + column_name: scaled_score + description: null + expression: '' + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Float64 + verbose_name: Scaled Score - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +41,7 @@ columns: - advanced_data_type: null column_name: grade_bucket description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +77,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +89,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +101,7 @@ columns: - advanced_data_type: null column_name: entity_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -86,21 +111,9 @@ columns: type: String verbose_name: Entity Name - advanced_data_type: null - column_name: scaled_score - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: Float - verbose_name: Scaled Score -- advanced_data_type: null - column_name: grade_type + column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -108,27 +121,36 @@ columns: is_dttm: false python_date_format: null type: String - verbose_name: Grade Type + verbose_name: Organization - advanced_data_type: null - column_name: org + column_name: entity_name_with_location description: null expression: null - extra: {} + extra: null filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Organization + type: Nullable(String) + verbose_name: Entity Name With Location database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml index 033a72369..8b57f0cd5 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml @@ -1,10 +1,11 @@ _file_name: fact_problem_responses.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: problem_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: problem_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: attempts description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: success description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -100,7 +101,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -112,7 +113,7 @@ columns: - advanced_data_type: null column_name: responses description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -121,14 +122,47 @@ columns: python_date_format: null type: String verbose_name: Responses +- advanced_data_type: null + column_name: problem_id + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Problem ID +- advanced_data_type: null + column_name: course_key + description: null + expression: null + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Course Key database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_transcript_usage.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_transcript_usage.yaml index ce9b5871e..cd3cfd492 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_transcript_usage.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_transcript_usage.yaml @@ -1,10 +1,11 @@ _file_name: fact_transcript_usage.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: video_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: video_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -103,7 +104,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: - currency: null d3format: null @@ -125,6 +126,15 @@ metrics: metric_type: null verbose_name: Total transcript usage warning_text: null +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null normalize_columns: true offset: 0 params: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml index a1a4398be..b362118d9 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml @@ -1,10 +1,11 @@ _file_name: fact_video_plays.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: video_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: emission_time description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: video_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -103,7 +104,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: emission_time metrics: - currency: null d3format: null @@ -125,6 +126,15 @@ metrics: metric_type: null verbose_name: Total plays warning_text: null +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null normalize_columns: true offset: 0 params: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_watched_video_segments.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_watched_video_segments.yaml index 5061a7911..829d0c18b 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_watched_video_segments.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_watched_video_segments.yaml @@ -1,10 +1,11 @@ _file_name: fact_watched_video_segments.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: video_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,19 +17,19 @@ columns: - advanced_data_type: null column_name: segment_start description: null - expression: null + expression: '' extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Int32 + type: Int64 verbose_name: Segment Start - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: started_at description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: video_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -100,7 +101,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -115,7 +116,7 @@ description: null extra: {} fetch_values_predicate: null filter_select_enabled: false -main_dttm_col: null +main_dttm_col: started_at metrics: - currency: null d3format: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/hints_per_success.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/hints_per_success.yaml index 2b35aa575..e377255f2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/hints_per_success.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/hints_per_success.yaml @@ -1,10 +1,11 @@ _file_name: hints_per_success.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: problem_name_with_location description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: total_hints description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -76,7 +77,7 @@ columns: - advanced_data_type: null column_name: problem_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -88,7 +89,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -105,6 +106,15 @@ fetch_values_predicate: null filter_select_enabled: false main_dttm_col: null metrics: +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null - currency: null d3format: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/posts_per_user.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/posts_per_user.yaml index 9a9417b9d..706d2a50f 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/posts_per_user.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/posts_per_user.yaml @@ -1,10 +1,11 @@ _file_name: posts_per_user.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null column_name: num_posts description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -16,7 +17,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -28,7 +29,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -40,7 +41,7 @@ columns: - advanced_data_type: null column_name: course_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -52,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -64,7 +65,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -90,6 +91,15 @@ metrics: metric_type: null verbose_name: Number Of Users warning_text: null +- currency: null + d3format: null + description: null + expression: COUNT(*) + extra: null + metric_name: count + metric_type: count + verbose_name: COUNT(*) + warning_text: null normalize_columns: true offset: 0 params: null From 7fd2fba1b76d452f353271c54a8541ecae936056 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 17:00:57 -0500 Subject: [PATCH 05/11] fix: use dataset fetch_metadata method --- .../apps/superset/pythonpath/create_assets.py | 12 +--- .../assets/datasets/fact_video_plays.yaml | 63 ++++++++++++++++++- 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index ce7b53a96..305a350ae 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -12,7 +12,6 @@ from pathlib import Path from superset import security_manager -from superset.commands.dataset.refresh import RefreshDatasetCommand from superset.examples.utils import load_configs_from_directory from superset.extensions import db from superset.models.dashboard import Dashboard @@ -20,7 +19,6 @@ from superset.utils.database import get_or_create_db from superset.models.embedded_dashboard import EmbeddedDashboard from pythonpath.localization import get_translation -from flask import g from superset.connectors.sqla.utils import get_columns_description BASE_DIR = "/app/assets/superset" @@ -294,14 +292,8 @@ def update_datasets(): db.session.query(SqlaTable).all() ) for dataset in datasets: - try: - if dataset_is_empty(dataset): - print(f"Dataset {dataset.table_name} is empty") - continue - RefreshDatasetCommand(dataset.id).run() - print(f"Refreshing dataset {dataset.table_name}") - except Exception as e: - print(f"Failed to refresh dataset {dataset.table_name}") + print(f"Refreshing dataset {dataset.table_name}") + dataset.fetch_metadata(commit=True) def dataset_is_empty(dataset): diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml index b362118d9..2ef1233f8 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml @@ -86,6 +86,18 @@ columns: python_date_format: null type: String verbose_name: Video Name +- advanced_data_type: null + column_name: video_position + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Decimal(9, 2) + verbose_name: Video Position - advanced_data_type: null column_name: org description: null @@ -98,6 +110,54 @@ columns: python_date_format: null type: String verbose_name: Organization +- advanced_data_type: null + column_name: video_duration + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Int64 + verbose_name: Video Duration +- advanced_data_type: null + column_name: graded + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Bool + verbose_name: Graded +- advanced_data_type: null + column_name: visualization_bucket + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Visualization Bucket +- advanced_data_type: null + column_name: video_id + description: null + expression: null + extra: {} + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Video ID database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null @@ -130,7 +190,8 @@ metrics: d3format: null description: null expression: COUNT(*) - extra: null + extra: + warning_markdown: '' metric_name: count metric_type: count verbose_name: COUNT(*) From 9f5d2bd804403a7c7bb7bc723f4d4fc550e19155 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 17:01:55 -0500 Subject: [PATCH 06/11] fix: use dataset fetch_metadata method --- .../apps/superset/pythonpath/create_assets.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index 305a350ae..c38c49c18 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -19,7 +19,6 @@ from superset.utils.database import get_or_create_db from superset.models.embedded_dashboard import EmbeddedDashboard from pythonpath.localization import get_translation -from superset.connectors.sqla.utils import get_columns_description BASE_DIR = "/app/assets/superset" @@ -296,17 +295,5 @@ def update_datasets(): dataset.fetch_metadata(commit=True) -def dataset_is_empty(dataset): - """Check if a dataset is empty""" - sql = dataset.get_template_processor().process_template( - dataset.sql, **dataset.template_params_dict - ) - try: - columns = get_columns_description(dataset.database, dataset.schema, sql) - except Exception as e: - print(f"Failed to get columns for dataset {dataset.table_name}: {e}") - return True - return columns == [] - if __name__ == "__main__": main() From fe9c7ebcea54e7b19f486bdcb2e29704015bfa13 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 4 Apr 2024 17:15:11 -0500 Subject: [PATCH 07/11] fix: add * for select video plays --- .../aspects/apps/superset/pythonpath/create_assets.py | 1 - .../openedx-assets/queries/fact_video_plays.sql | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index c38c49c18..0564de7d0 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -286,7 +286,6 @@ def update_embeddable_uuids(): def update_datasets(): """Update the datasets""" print("Refreshing datasets") - g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}") datasets = ( db.session.query(SqlaTable).all() ) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index 9609b93f5..a516366ab 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -15,12 +15,5 @@ where ) select - emission_time, - org, - course_key, - course_name, - course_run, - video_name, - video_name_with_location, - actor_id + * from plays From 074ce600d83d281e2f83c4f63e68b38b5ab0cc48 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Fri, 5 Apr 2024 09:15:07 -0500 Subject: [PATCH 08/11] chore: re-serialize superset assets --- .../dashboards/Instructor_Dashboard.yaml | 28 +++---- .../datasets/fact_pageview_engagement.yaml | 14 ++-- .../assets/datasets/fact_problem_grades.yaml | 24 +++--- .../datasets/fact_problem_responses.yaml | 44 +++++------ .../datasets/fact_video_engagement.yaml | 1 + .../assets/datasets/fact_video_plays.yaml | 78 +++++++++---------- 6 files changed, 95 insertions(+), 94 deletions(-) diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml index bd12718cd..140e38eba 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml @@ -422,7 +422,7 @@ position: children: [] id: CHART-AZZnl_lpMv meta: - chartId: 21 + chartId: 22 height: 50 sliceName: Watches Per Video uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64 @@ -437,7 +437,7 @@ position: children: [] id: CHART-GFCO8s2cxv meta: - chartId: 19 + chartId: 20 height: 50 sliceName: Distribution Of Responses uuid: f1651c44-a8f4-4b44-ad49-962823009319 @@ -467,7 +467,7 @@ position: children: [] id: CHART-Jr-gNVms2Q meta: - chartId: 12 + chartId: 13 height: 50 sliceName: Enrollment Events Per Day uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa @@ -482,7 +482,7 @@ position: children: [] id: CHART-OMy4wjRBWt meta: - chartId: 27 + chartId: 28 height: 50 sliceName: Watched Video Segments uuid: 2985a9db-c338-4008-af52-2930b81ee2e5 @@ -497,7 +497,7 @@ position: children: [] id: CHART-RTO33WE9FH meta: - chartId: 15 + chartId: 16 height: 50 sliceName: Responses Per Problem uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed @@ -512,7 +512,7 @@ position: children: [] id: CHART-Tej2oLPBAl meta: - chartId: 37 + chartId: 38 height: 50 sliceName: Transcripts / Captions Per Video uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8 @@ -557,7 +557,7 @@ position: children: [] id: CHART-evjVO-ZSSd meta: - chartId: 38 + chartId: 39 height: 50 sliceName: Enrollments By Enrollment Mode uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3 @@ -572,7 +572,7 @@ position: children: [] id: CHART-j3trfNh8_p meta: - chartId: 29 + chartId: 30 height: 50 sliceName: Pageview engagement by section/subsection uuid: 366a8193-30c3-4aaf-a1ac-360609dfa0ed @@ -587,8 +587,8 @@ position: children: [] id: CHART-jfm-20qPKn meta: - chartId: 816 - height: 55 + chartId: 11 + height: 50 sliceName: Video engagement by section/subsection uuid: 21861f47-9e8b-4715-b1ff-e7f4aa595c14 width: 12 @@ -617,7 +617,7 @@ position: children: [] id: CHART-o56v9yEe2I meta: - chartId: 31 + chartId: 32 height: 50 sliceName: Distribution Of Problem Grades uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2 @@ -632,7 +632,7 @@ position: children: [] id: CHART-qG1WaGKl_b meta: - chartId: 34 + chartId: 35 height: 50 sliceName: Distinct forum users uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e @@ -647,7 +647,7 @@ position: children: [] id: CHART-rnb6PSwCOS meta: - chartId: 16 + chartId: 17 height: 50 sliceName: Currently Enrolled Learners Per Day uuid: ed2fe731-6544-422f-bc55-42f399f48b2c @@ -662,7 +662,7 @@ position: children: [] id: CHART-tWnaoVNNTH meta: - chartId: 11 + chartId: 12 height: 50 sliceName: Distribution Of Attempts uuid: db90930f-f16e-4c32-8050-0e4abae28f4c diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml index 3f2f32f9d..601ef56b8 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_pageview_engagement.yaml @@ -5,7 +5,7 @@ columns: - advanced_data_type: null column_name: section/subsection page engagement description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -17,7 +17,7 @@ columns: - advanced_data_type: null column_name: section/subsection name description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -29,7 +29,7 @@ columns: - advanced_data_type: null column_name: content level description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -41,7 +41,7 @@ columns: - advanced_data_type: null column_name: actor_id description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -53,7 +53,7 @@ columns: - advanced_data_type: null column_name: course_key description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -65,7 +65,7 @@ columns: - advanced_data_type: null column_name: course_run description: null - expression: null + expression: '' extra: null filterable: true groupby: true @@ -77,7 +77,7 @@ columns: - advanced_data_type: null column_name: org description: null - expression: null + expression: '' extra: null filterable: true groupby: true diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml index f0e3e9010..35694bfb2 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_grades.yaml @@ -2,6 +2,18 @@ _file_name: fact_problem_grades.yaml always_filter_main_dttm: false cache_timeout: null columns: +- advanced_data_type: null + column_name: entity_name_with_location + description: null + expression: '' + extra: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: Nullable(String) + verbose_name: Entity Name With Location - advanced_data_type: null column_name: grade_type description: null @@ -122,18 +134,6 @@ columns: python_date_format: null type: String verbose_name: Organization -- advanced_data_type: null - column_name: entity_name_with_location - description: null - expression: null - extra: null - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: Nullable(String) - verbose_name: Entity Name With Location database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml index 8b57f0cd5..e3c551b6f 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_problem_responses.yaml @@ -75,31 +75,31 @@ columns: type: String verbose_name: Problem Name - advanced_data_type: null - column_name: attempts + column_name: course_key description: null expression: '' - extra: {} + extra: null filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Int16 - verbose_name: Attempts + type: String + verbose_name: Course Key - advanced_data_type: null - column_name: success + column_name: problem_id description: null expression: '' - extra: {} + extra: null filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Bool - verbose_name: Success + type: String + verbose_name: Problem ID - advanced_data_type: null - column_name: org + column_name: attempts description: null expression: '' extra: {} @@ -108,10 +108,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Organization + type: Int16 + verbose_name: Attempts - advanced_data_type: null - column_name: responses + column_name: success description: null expression: '' extra: {} @@ -120,32 +120,32 @@ columns: is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Responses + type: Bool + verbose_name: Success - advanced_data_type: null - column_name: problem_id + column_name: org description: null - expression: null - extra: null + expression: '' + extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null type: String - verbose_name: Problem ID + verbose_name: Organization - advanced_data_type: null - column_name: course_key + column_name: responses description: null - expression: null - extra: null + expression: '' + extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null type: String - verbose_name: Course Key + verbose_name: Responses database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_engagement.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_engagement.yaml index e5900bc4c..f57c0980c 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_engagement.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_engagement.yaml @@ -1,4 +1,5 @@ _file_name: fact_video_engagement.yaml +always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml index 2ef1233f8..f9691cbd1 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/fact_video_plays.yaml @@ -15,7 +15,7 @@ columns: type: String verbose_name: Video Name With Location - advanced_data_type: null - column_name: actor_id + column_name: video_position description: null expression: '' extra: {} @@ -24,10 +24,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Actor ID + type: Decimal(9, 2) + verbose_name: Video Position - advanced_data_type: null - column_name: course_key + column_name: video_duration description: null expression: '' extra: {} @@ -36,10 +36,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Course Key + type: Int64 + verbose_name: Video Duration - advanced_data_type: null - column_name: course_name + column_name: actor_id description: null expression: '' extra: {} @@ -49,9 +49,9 @@ columns: is_dttm: false python_date_format: null type: String - verbose_name: Course Name + verbose_name: Actor ID - advanced_data_type: null - column_name: course_run + column_name: visualization_bucket description: null expression: '' extra: {} @@ -61,21 +61,21 @@ columns: is_dttm: false python_date_format: null type: String - verbose_name: Course Run + verbose_name: Visualization Bucket - advanced_data_type: null - column_name: emission_time + column_name: course_key description: null expression: '' extra: {} filterable: true groupby: true is_active: true - is_dttm: true + is_dttm: false python_date_format: null - type: DateTime - verbose_name: Emission Time + type: String + verbose_name: Course Key - advanced_data_type: null - column_name: video_name + column_name: course_name description: null expression: '' extra: {} @@ -85,71 +85,71 @@ columns: is_dttm: false python_date_format: null type: String - verbose_name: Video Name + verbose_name: Course Name - advanced_data_type: null - column_name: video_position + column_name: course_run description: null - expression: null + expression: '' extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Decimal(9, 2) - verbose_name: Video Position + type: String + verbose_name: Course Run - advanced_data_type: null - column_name: org + column_name: emission_time description: null expression: '' extra: {} filterable: true groupby: true is_active: true - is_dttm: false + is_dttm: true python_date_format: null - type: String - verbose_name: Organization + type: DateTime + verbose_name: Emission Time - advanced_data_type: null - column_name: video_duration + column_name: video_id description: null - expression: null + expression: '' extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Int64 - verbose_name: Video Duration + type: String + verbose_name: Video ID - advanced_data_type: null - column_name: graded + column_name: video_name description: null - expression: null + expression: '' extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: Bool - verbose_name: Graded + type: String + verbose_name: Video Name - advanced_data_type: null - column_name: visualization_bucket + column_name: graded description: null - expression: null + expression: '' extra: {} filterable: true groupby: true is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Visualization Bucket + type: Bool + verbose_name: Graded - advanced_data_type: null - column_name: video_id + column_name: org description: null - expression: null + expression: '' extra: {} filterable: true groupby: true @@ -157,7 +157,7 @@ columns: is_dttm: false python_date_format: null type: String - verbose_name: Video ID + verbose_name: Organization database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null From 65e20006d922c66961817161e63c3a09ba94546f Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 8 Apr 2024 12:53:58 -0500 Subject: [PATCH 09/11] chore: upgrade superset image to v4.0.0 --- .../aspects/apps/superset/pythonpath/superset_config_docker.py | 1 + .../templates/aspects/build/aspects-superset/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py index 592cdcf8c..d498b778b 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py @@ -91,6 +91,7 @@ "EMBEDDABLE_CHARTS": True, "EMBEDDED_SUPERSET": True, "ENABLE_TEMPLATE_PROCESSING": True, + "TAGGING_SYSTEM": True, } # Add this custom template processor which returns the list of courses the current user can access diff --git a/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile b/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile index 1bc0d6a4e..fec5d2f36 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile +++ b/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile @@ -3,7 +3,7 @@ # https://github.com/apache/superset/releases # https://github.com/apache/superset/blob/master/Dockerfile # https://superset.apache.org/docs/databases/installing-database-drivers -FROM apache/superset:3.1.2 +FROM apache/superset:4.0.0 USER root From 89adc0e4366aeb52be1edc3716f273d4b7d578eb Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 8 Apr 2024 13:02:44 -0500 Subject: [PATCH 10/11] chore: fix templating --- .../openedx-assets/queries/fact_problem_engagement.sql | 4 ++-- .../openedx-assets/queries/fact_video_engagement.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql index 83b5918f2..3c254fed1 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql @@ -20,10 +20,10 @@ with where 1 = 1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and attempted_on > date('{{ from_dttm }}') {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and attempted_on < date('{{ to_dttm }}') {% endif %} {% endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql index a0954365c..ca08a53cb 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql @@ -19,10 +19,10 @@ with where 1 = 1 {% raw %} - {% if from_dttm is not none %} + {% if from_dttm %} and viewed_on > date('{{ from_dttm }}') {% endif %} - {% if to_dttm is not none %} + {% if to_dttm %} and viewed_on < date('{{ to_dttm }}') {% endif %} {% endraw %} From 13090086deb5ef207762323fb96d4b1325fdb538 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 8 Apr 2024 13:21:11 -0500 Subject: [PATCH 11/11] chore: only fetch dataset metadata when configured (cherry picked from commit c48616456264d86a8985e1781392eb4ac88fe868) --- tutoraspects/plugin.py | 4 ++++ .../apps/superset/pythonpath/create_assets.py | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 145ea741d..cc0c1ff52 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -372,6 +372,10 @@ # By default it is on if Caddy is enabled, but it can be set separately in # case you are running a different proxy or otherwise have different needs. ("SUPERSET_ENABLE_PROXY_FIX", "{{ENABLE_WEB_PROXY}}"), + # This setting allows operators to automatically refresh the datasets + # in the Superset database. This is useful for keeping the columns up to + # date with the latest changes in DBT. + ("SUPERSET_REFRESH_DATASETS", False), ###################### # dbt Settings # For the most part you shouldn't have to touch these diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py index 0564de7d0..607945429 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py @@ -286,12 +286,13 @@ def update_embeddable_uuids(): def update_datasets(): """Update the datasets""" print("Refreshing datasets") - datasets = ( - db.session.query(SqlaTable).all() - ) - for dataset in datasets: - print(f"Refreshing dataset {dataset.table_name}") - dataset.fetch_metadata(commit=True) + if {{SUPERSET_REFRESH_DATASETS}}: + datasets = ( + db.session.query(SqlaTable).all() + ) + for dataset in datasets: + print(f"Refreshing dataset {dataset.table_name}") + dataset.fetch_metadata(commit=True) if __name__ == "__main__":