From 2542a5af132314c40224c1fd3012a8a70086792b Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 1 Apr 2024 11:09:22 -0500 Subject: [PATCH] feat: upgrade to clickhouse 24.3 LTS --- tutoraspects/plugin.py | 4 ++-- .../aspects/jobs/init/clickhouse/init-clickhouse.sh | 1 + .../openedx-assets/queries/fact_enrollments.sql | 4 ++-- .../queries/fact_learner_problem_summary.sql | 4 ++-- .../openedx-assets/queries/fact_pageview_engagement.sql | 9 +++++++-- .../openedx-assets/queries/int_problem_responses.sql | 4 ++-- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py index b43d9de70..bf6b11807 100644 --- a/tutoraspects/plugin.py +++ b/tutoraspects/plugin.py @@ -34,7 +34,7 @@ ("RUN_RALPH", True), ("RUN_SUPERSET", True), ("DOCKER_IMAGE_ASPECTS", "edunext/aspects:{{ ASPECTS_VERSION }}"), - ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:23.8"), + ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:24.3"), ("DOCKER_IMAGE_RALPH", "fundocker/ralph:4.1.0"), ("DOCKER_IMAGE_SUPERSET", "edunext/aspects-superset:{{ ASPECTS_VERSION }}"), ("DOCKER_IMAGE_VECTOR", "timberio/vector:0.30.0-alpine"), @@ -373,7 +373,7 @@ # For now we are pulling this from github, which should allow maximum # flexibility for forking, running branches, specific versions, etc. ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"), - ("DBT_BRANCH", "v3.9.1"), + ("DBT_BRANCH", "v3.9.2"), ("DBT_SSH_KEY", ""), ("DBT_STATE_DIR", "/app/aspects/dbt_state/"), # This is a pip compliant list of Python packages to install to run dbt diff --git a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh index 74fdee54d..92d1721ff 100644 --- a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh +++ b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh @@ -60,6 +60,7 @@ GRANT {{ ON_CLUSTER }} SELECT ON system.disks TO '{{ ASPECTS_CLICKHOUSE_REPORT_U GRANT {{ ON_CLUSTER }} SELECT ON system.events TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} SELECT ON system.metrics TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; GRANT {{ ON_CLUSTER }} SELECT ON system.replication_queue TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; +GRANT {{ ON_CLUSTER }} SELECT ON system.query_log TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}'; -- Patch from clickhouse-extra-sql follows... {{ patch("clickhouse-extra-sql") }} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql index 2af0fb504..091d2f566 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql @@ -1,4 +1,4 @@ -with enrollments as ( +with enrollments_base as ( select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments where @@ -15,4 +15,4 @@ select actor_id, enrollment_mode, enrollment_status -from enrollments +from enrollments_base diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index 692f06beb..81dcbded4 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -70,7 +70,7 @@ WITH problem_responses AS ( attempts FROM problem_responses INNER JOIN final_responses USING (org, course_key, problem_id, actor_id, emission_time) -), summary AS ( +), summary_base AS ( SELECT org, course_key, @@ -115,7 +115,7 @@ SELECT coalesce(any(attempts), 0) AS attempts, sum(num_hints_displayed) AS num_hints_displayed, sum(num_answers_displayed) AS num_answers_displayed -FROM summary +FROM summary_base where {% raw %} {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql index f768694bc..8e83fb206 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql @@ -3,11 +3,12 @@ with select org, course_key, + course_run, section_with_name, subsection_with_name, actor_id, page_count, - countdistinct(block_id) as pages_visited, + COUNT(DISTINCT block_id) as pages_visited, case when pages_visited = 0 then 'No pages viewed yet' @@ -30,6 +31,7 @@ with group by org, course_key, + course_run, section_with_name, subsection_with_name, actor_id, @@ -39,6 +41,7 @@ with select org, course_key, + course_run, section_with_name, actor_id, sum(page_count) as page_count, @@ -51,12 +54,13 @@ with else 'At least one page viewed' end as engagement_level from subsection_counts - group by org, course_key, section_with_name, actor_id + group by org, course_key, course_run, section_with_name, actor_id ) select org, course_key, + course_run, subsection_with_name as `section/subsection name`, 'subsection' as `content level`, actor_id as actor_id, @@ -66,6 +70,7 @@ union all select org, course_key, + course_run, section_with_name as `section/subsection name`, 'section' as `content level`, actor_id as actor_id, diff --git a/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql index 57b4142f0..75e38e67a 100644 --- a/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql @@ -1,4 +1,4 @@ -with problem_responses as ( +with problem_responses_base as ( select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_problem_responses where 1=1 @@ -19,4 +19,4 @@ select success, responses from - problem_responses + problem_responses_base