feat: import new course dashboard V1

tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py

@@ -19,7 +19,7 @@
 from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
 from pythonpath.localization import get_translation
-
+from pythonpath.create_row_level_security import create_rls_filters
 BASE_DIR = "/app/assets/superset"
 
 ASSET_FOLDER_MAPPING = {
@@ -93,6 +93,7 @@ def create_assets():
     update_dashboard_roles(roles)
     update_embeddable_uuids()
     update_datasets()
+    create_rls_filters()
 
 
 def import_databases():

tutoraspects/templates/aspects/apps/superset/pythonpath/create_row_level_security.py

@@ -1,8 +1,3 @@
-from superset.app import create_app
-
-app = create_app()
-app.app_context().push()
-
 from superset.connectors.sqla.models import (
     RLSFilterRoles,
     RowLevelSecurityFilter,
@@ -15,100 +10,34 @@
 
 ## https://docs.preset.io/docs/row-level-security-rls
 
-VIRTUAL_TABLE_SCHEMA = "main"
+XAPI_SCHEMA = "{{ ASPECTS_XAPI_DATABASE }}"
+DBT_SCHEMA = "{{ DBT_PROFILE_TARGET_DATABASE }}"
+EVENT_SINK_SCHEMA = "{{ ASPECTS_EVENT_SINK_DATABASE }}"
+
 
 SECURITY_FILTERS = [
     {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "{{ASPECTS_XAPI_TABLE}}",
+        "name": f"can_view_courses_{XAPI_SCHEMA}",
+        "schema": XAPI_SCHEMA,
+        "exclude": [],
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
         "clause": '{% raw %}{{can_view_courses(current_username(), "splitByChar(\'/\', course_id)[-1]")}}{% endraw %}',
-        "filter_type": "Regular",
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_enrollments_by_day",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
         "filter_type": "Regular"
     },
     {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_enrollments",
+        "name": f"can_view_courses_{EVENT_SINK_SCHEMA}",
+        "schema": EVENT_SINK_SCHEMA,
+        "exclude": ["user_pii"],
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
         "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
         "filter_type": "Regular"
     },
     {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_learner_problem_summary",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_problem_responses",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_transcript_usage",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_video_plays",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "fact_watched_video_segments",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "hints_per_success",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "course_names",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "course_overviews",
-        "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
-        "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
-        "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
-        "filter_type": "Regular"
-    },
-    {
-        "schema": VIRTUAL_TABLE_SCHEMA,
-        "table_name": "course_blocks",
+        "name": f"can_view_courses_{DBT_SCHEMA}",
+        "schema": DBT_SCHEMA,
+        "exclude": [],
         "role_name": "{{SUPERSET_ROLES_MAPPING.instructor}}",
         "group_key": "{{SUPERSET_ROW_LEVEL_SECURITY_XAPI_GROUP_KEY}}",
         "clause": '{% raw %}{{can_view_courses(current_username(), "course_key")}}{% endraw %}',
@@ -118,67 +47,57 @@
 
 {{patch("superset-row-level-security") | indent(4)}}
 
+def create_rls_filters():
+    for security_filter in SECURITY_FILTERS:
+        # Fetch the table we want to restrict access to
+        (
+            name,
+            schema,
+            exclude,
+            role_name,
+            group_key,
+            clause,
+            filter_type,
+        ) = security_filter.values()
+        tables = (
+            session.query(SqlaTable)
+            .filter(SqlaTable.schema == schema)
+            .filter(SqlaTable.table_name.not_in(exclude))
+            .all()
+        )
+        print(f"Creating RLS filter {name} for {schema} schema")
 
-for security_filter in SECURITY_FILTERS:
-    # Fetch the table we want to restrict access to
-    (
-        schema,
-        table_name,
-        role_name,
-        group_key,
-        clause,
-        filter_type,
-    ) = security_filter.values()
-    table = (
-        session.query(SqlaTable)
-        .filter(SqlaTable.schema == schema)
-        .filter(SqlaTable.table_name == table_name)
-        .first()
-    )
-
-    assert table, (f"{schema}.{table_name} table doesn't exist. If you have changed "
-                   "your database (schema) name, you will need to update the database "
-                   "connection and dataset schema entries in the Superset UI or "
-                   "database. You may also need to rebuild your aspects-superset "
-                   "image after changes.")
+        role = session.query(Role).filter(Role.name == role_name).first()
+        assert role, f"{role_name} role doesn't exist yet?"
+        # See if the Row Level Security Filter already exists
+        rlsf = (
+            session.query(RowLevelSecurityFilter)
+            .filter(RowLevelSecurityFilter.group_key == group_key)
+            .filter(RowLevelSecurityFilter.name == name)
+        ).first()
+        # If it doesn't already exist, create one
+        if not rlsf:
+            rlsf = RowLevelSecurityFilter()
+        # Sync the fields to our expectations
+        rlsf.filter_type = filter_type
+        rlsf.group_key = group_key
+        rlsf.tables = tables
+        rlsf.clause = clause
+        rlsf.name = name
 
-    role = session.query(Role).filter(Role.name == role_name).first()
-    assert role, f"{role_name} role doesn't exist yet?"
-    # See if the Row Level Security Filter already exists
-    rlsf = (
-        session.query(RowLevelSecurityFilter)
-        .filter(RLSFilterRoles.c.role_id.in_((role.id,)))
-        .filter(RowLevelSecurityFilter.group_key == group_key)
-        .filter(RowLevelSecurityFilter.tables.any(id=table.id))
-    ).first()
-    # If it doesn't already exist, create one
-    if rlsf:
-        create = False
-    else:
-        create = True
-        rlsf = RowLevelSecurityFilter()
-    # Sync the fields to our expectations
-    rlsf.filter_type = filter_type
-    rlsf.group_key = group_key
-    rlsf.tables = [table]
-    rlsf.clause = clause
-    rlsf.name = f"{table.table_name} - {role.name}"
-    # Create if needed
-    if create:
         session.add(rlsf)
-        # ...and commit, so we are sure to have an rlsf.id
         session.commit()
-    # Add the filter role if needed
-    rls_filter_roles = (
-        session.query(RLSFilterRoles)
-        .filter(RLSFilterRoles.c.role_id == role.id)
-        .filter(RLSFilterRoles.c.rls_filter_id == rlsf.id)
-    )
-
-    if not rls_filter_roles.count():
-        session.execute(
-            RLSFilterRoles.insert(), [dict(role_id=role.id, rls_filter_id=rlsf.id)]
+        # Add the filter role if needed
+        rls_filter_roles = (
+            session.query(RLSFilterRoles)
+            .filter(RLSFilterRoles.c.role_id == role.id)
+            .filter(RLSFilterRoles.c.rls_filter_id == rlsf.id)
         )
-        session.commit()
 
-print("Successfully create row-level security filters.")
+        if not rls_filter_roles.count():
+            session.execute(
+                RLSFilterRoles.insert(), [dict(role_id=role.id, rls_filter_id=rlsf.id)]
+            )
+            session.commit()
+
+    print("Successfully create row-level security filters.")

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Users_Over_Time_v2.yaml

@@ -11,6 +11,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   color_picker:
     a: 1
     b: 135
@@ -36,6 +37,7 @@ params:
   viz_type: big_number
   x_axis: emission_time
   y_axis_format: SMART_NUMBER
+query_context: null
 slice_name: Active Users Over Time v2
 uuid: e1d619a2-6238-4264-a000-6b6856027edd
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Users_Per_Organization.yaml

@@ -11,6 +11,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   color_scheme: supersetColors
   date_format: smart_date
   extra_form_data: {}
@@ -54,6 +55,7 @@ params:
   show_legend: true
   sort_by_metric: true
   viz_type: pie
+query_context: null
 slice_name: Active Users Per Organization
 uuid: 02af00f5-6539-428f-b3ab-632997de7528
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Chart_Count.yaml

@@ -6,6 +6,7 @@ dataset_uuid: aaf1ae12-73c1-4b88-b07d-7a8988333018
 description: Count of charts created in Superset over the selected time period.
 params:
   adhoc_filters: []
+  annotation_layers: []
   color_picker:
     a: 1
     b: 150
@@ -30,6 +31,7 @@ params:
   url_params: {}
   viz_type: big_number
   y_axis_format: SMART_NUMBER
+query_context: null
 slice_name: Chart Count
 uuid: a5e0f18b-fb83-4d65-81e0-5f0e620fa05b
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Charts_by_Type.yaml

@@ -7,6 +7,7 @@ description: Count of the various types of charts created in Superset over the s
   time period.
 params:
   adhoc_filters: []
+  annotation_layers: []
   color_scheme: htg_color_scheme
   date_format: smart_date
   extra_form_data: {}
@@ -25,6 +26,7 @@ params:
   - exclusive
   url_params: {}
   viz_type: treemap_v2
+query_context: null
 slice_name: Charts by Type
 uuid: 84393abe-04ab-4084-a195-116ad568badd
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/ClickHouse_metrics.yaml

@@ -10,6 +10,7 @@ params:
   - name
   - value
   - description
+  annotation_layers: []
   color_pn: true
   extra_form_data: {}
   groupby: []
@@ -24,6 +25,7 @@ params:
   temporal_columns_lookup: {}
   time_grain_sqla: P1D
   viz_type: table
+query_context: null
 slice_name: ClickHouse metrics
 uuid: 2f84cbc3-4068-4d4d-bd40-ae348e4be95d
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Enrollments_Over_Time.yaml

@@ -83,6 +83,7 @@ params:
   y_axis_title: Registrations
   y_axis_title_margin: 15
   y_axis_title_position: Left
+query_context: null
 slice_name: Course Enrollments Over Time
 uuid: bf4f4671-c276-4185-9b9a-b10864efea6c
 version: 1.0.0