From 4f466ea699643fd9dae5d97f3df5dfc51dac2970 Mon Sep 17 00:00:00 2001 From: Sara Burns Date: Thu, 7 Nov 2024 13:55:35 -0500 Subject: [PATCH] feat: update dbt and use new param view --- .github/workflows/integration-test.yml | 28 +++++----- tutoraspects/plugin.py | 2 +- .../datasets/watched_video_duration.yaml | 53 ++++++++++--------- .../queries/watched_video_duration.sql | 14 +++-- 4 files changed, 53 insertions(+), 44 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index b561d3913..0c37e5134 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -155,6 +155,20 @@ jobs: k8s: runs-on: ubuntu-latest steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: true + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: false + swap-storage: true - name: Checkout uses: actions/checkout@v4 - name: setup python @@ -177,20 +191,6 @@ jobs: kustomize build $TUTOR_ROOT/env | kubeconform -strict -ignore-missing-schemas -kubernetes-version 1.22.0 - name: Setup Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be - with: - # this might remove tools that are actually needed, - # if set to "true" but frees about 6 GB - tool-cache: false - - # all of these default to true, but feel free to set to - # "false" if necessary for your workflow - android: true - dotnet: true - haskell: true - large-packages: false - swap-storage: true - name: Tutor build openedx run: tutor images build openedx aspects aspects-superset - name: Create k8s Kind Cluster diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index 44974b3cf..f47cc4ddd 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -385,7 +385,7 @@ # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v3.33.5"), + ("DBT_BRANCH", "v3.33.7"), ("DBT_SSH_KEY", ""), ("DBT_STATE_DIR", "/app/aspects-dbt/state"), ("DBT_PROFILES_DIR", "/app/aspects/dbt/"), diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml index 6a67dcba8..d744a10bf 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/watched_video_duration.yaml @@ -3,9 +3,9 @@ always_filter_main_dttm: false cache_timeout: null columns: - advanced_data_type: null - column_name: course_name + column_name: rewatched_time description: null - expression: '' + expression: null extra: warning_markdown: null filterable: true @@ -13,10 +13,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: String - verbose_name: Course Name + type: Decimal(38, 2) + verbose_name: Rewatched Time - advanced_data_type: null - column_name: video_duration + column_name: watched_time description: null expression: null extra: @@ -26,10 +26,23 @@ columns: is_active: true is_dttm: false python_date_format: null - type: Int64 - verbose_name: Video Duration + type: Decimal(38, 2) + verbose_name: Watched Time - advanced_data_type: null - column_name: rewatched_time + column_name: course_name + description: null + expression: '' + extra: + warning_markdown: null + filterable: true + groupby: true + is_active: true + is_dttm: false + python_date_format: null + type: String + verbose_name: Course Name +- advanced_data_type: null + column_name: video_count description: null expression: null extra: @@ -39,10 +52,10 @@ columns: is_active: true is_dttm: false python_date_format: null - type: Int64 - verbose_name: Rewatched Time + type: Int32 + verbose_name: Video Count - advanced_data_type: null - column_name: watched_time + column_name: video_duration description: null expression: null extra: @@ -53,7 +66,7 @@ columns: is_dttm: false python_date_format: null type: Int64 - verbose_name: Watched Time + verbose_name: Video Duration - advanced_data_type: null column_name: actor_id description: null @@ -106,18 +119,6 @@ columns: python_date_format: null type: String verbose_name: Org -- advanced_data_type: null - column_name: video_count - description: null - expression: null - extra: {} - filterable: true - groupby: true - is_active: true - is_dttm: false - python_date_format: null - type: Int32 - verbose_name: Video Count database_uuid: 21174b6c-4d40-4958-8161-d6c3cf5e77b6 default_endpoint: null description: null @@ -129,7 +130,7 @@ metrics: - currency: null d3format: null description: null - expression: (sum(rewatched_time)/count(actor_id))/sum(video_duration) + expression: if(sum(video_duration) > 0,(sum(rewatched_time)/count(actor_id))/sum(video_duration),0) extra: warning_markdown: '' metric_name: rewatched_percent @@ -139,7 +140,7 @@ metrics: - currency: null d3format: null description: null - expression: (sum(watched_time)/count(actor_id))/sum(video_duration) + expression: if(sum(watched_time)>0,(sum(watched_time)/count(actor_id))/sum(video_duration),0) extra: warning_markdown: '' metric_name: watched_percent diff --git a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql index 1310c39a6..263bf950d 100644 --- a/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql +++ b/tutoraspects/templates/openedx-assets/queries/watched_video_duration.sql @@ -8,7 +8,15 @@ select video_duration, watched_time, rewatched_time -from {{ ASPECTS_XAPI_DATABASE }}.watched_video_duration -left join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as names -using org, course_key +from + {{ DBT_PROFILE_TARGET_DATABASE }}.watched_video_duration( + {% raw -%} + org_filter ={{ filter_values("org") }}, + course_name_filter ={{ filter_values("course_name") }} + {%- endraw %} + ) as a +left join + {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as names + on a.org = names.org + and a.course_key = names.course_key where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %}