Merge pull request #685 from openedx/cag/clickhouse-24.3

feat: upgrade to clickhouse 24.3 LTS
openedx · Apr 2, 2024 · 6c751ad · 6c751ad
2 parents 27aa42f + 2542a5a
commit 6c751ad
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 10 deletions.
diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py
@@ -35,7 +35,7 @@
         ("RUN_RALPH", True),
         ("RUN_SUPERSET", True),
         ("DOCKER_IMAGE_ASPECTS", "edunext/aspects:{{ ASPECTS_VERSION }}"),
-        ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:23.8"),
+        ("DOCKER_IMAGE_CLICKHOUSE", "clickhouse/clickhouse-server:24.3"),
         ("DOCKER_IMAGE_RALPH", "fundocker/ralph:4.1.0"),
         ("DOCKER_IMAGE_SUPERSET", "edunext/aspects-superset:{{ ASPECTS_VERSION }}"),
         ("DOCKER_IMAGE_VECTOR", "timberio/vector:0.30.0-alpine"),
@@ -376,7 +376,7 @@
         # For now we are pulling this from github, which should allow maximum
         # flexibility for forking, running branches, specific versions, etc.
         ("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"),
-        ("DBT_BRANCH", "v3.9.1"),
+        ("DBT_BRANCH", "v3.9.2"),
         ("DBT_SSH_KEY", ""),
         ("DBT_STATE_DIR", "/app/aspects/dbt_state/"),
         # This is a pip compliant list of Python packages to install to run dbt

diff --git a/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh b/tutoraspects/templates/aspects/jobs/init/clickhouse/init-clickhouse.sh
@@ -60,6 +60,7 @@ GRANT {{ ON_CLUSTER }} SELECT ON system.disks TO '{{ ASPECTS_CLICKHOUSE_REPORT_U
 GRANT {{ ON_CLUSTER }} SELECT ON system.events TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
 GRANT {{ ON_CLUSTER }} SELECT ON system.metrics TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
 GRANT {{ ON_CLUSTER }} SELECT ON system.replication_queue TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
+GRANT {{ ON_CLUSTER }} SELECT ON system.query_log TO '{{ ASPECTS_CLICKHOUSE_REPORT_USER }}';
 
 -- Patch from clickhouse-extra-sql follows...
 {{ patch("clickhouse-extra-sql") }}

diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql
@@ -1,4 +1,4 @@
-with enrollments as (
+with enrollments_base as (
 select *
 from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments
 where
@@ -15,4 +15,4 @@ select
     actor_id,
     enrollment_mode,
     enrollment_status
-from enrollments
+from enrollments_base
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql
@@ -70,7 +70,7 @@ WITH problem_responses AS (
         attempts
     FROM problem_responses
     INNER JOIN final_responses USING (org, course_key, problem_id, actor_id, emission_time)
-), summary AS (
+), summary_base AS (
     SELECT
         org,
         course_key,
@@ -115,7 +115,7 @@ SELECT
     coalesce(any(attempts), 0) AS attempts,
     sum(num_hints_displayed) AS num_hints_displayed,
     sum(num_answers_displayed) AS num_answers_displayed
-FROM summary
+FROM summary_base
 where
     {% raw %}
     {% if get_filters('problem_name_with_location', remove_filter=True) == [] %}

diff --git a/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_pageview_engagement.sql
@@ -3,11 +3,12 @@ with
         select
             org,
             course_key,
+            course_run,
             section_with_name,
             subsection_with_name,
             actor_id,
             page_count,
-            countdistinct(block_id) as pages_visited,
+            COUNT(DISTINCT block_id) as pages_visited,
             case
                 when pages_visited = 0
                 then 'No pages viewed yet'
@@ -30,6 +31,7 @@ with
         group by
             org,
             course_key,
+            course_run,
             section_with_name,
             subsection_with_name,
             actor_id,
@@ -39,6 +41,7 @@ with
         select
             org,
             course_key,
+            course_run,
             section_with_name,
             actor_id,
             sum(page_count) as page_count,
@@ -51,12 +54,13 @@ with
                 else 'At least one page viewed'
             end as engagement_level
         from subsection_counts
-        group by org, course_key, section_with_name, actor_id
+        group by org, course_key, course_run, section_with_name, actor_id
     )
 
 select
     org,
     course_key,
+    course_run,
     subsection_with_name as `section/subsection name`,
     'subsection' as `content level`,
     actor_id as actor_id,
@@ -66,6 +70,7 @@ union all
 select
     org,
     course_key,
+    course_run,
     section_with_name as `section/subsection name`,
     'section' as `content level`,
     actor_id as actor_id,

diff --git a/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql
@@ -1,4 +1,4 @@
-with problem_responses as (
+with problem_responses_base as (
 select *
 from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_problem_responses
 where 1=1
@@ -19,4 +19,4 @@ select
     success,
     responses
 from
-    problem_responses
+    problem_responses_base