fix: log missing rows

openedx · Dec 19, 2024 · cfae721 · cfae721
1 parent 0300e33
commit cfae721
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py
@@ -232,8 +232,9 @@ def get_query_log_from_clickhouse(report, query_contexts, print_sql, fail_on_err
     # Run CH query until results for all slices are returned
     ch_count = 6
     while ch_count > 0:
-        if ch_chart_result["queries"][0]["rowcount"] < chart_count:
-            logger.info("Waiting for clickhouse log...")
+        missing_rows = chart_count - ch_chart_result["queries"][0]["rowcount"]
+        if missing_rows > 0:
+            logger.info(f"Waiting for {missing_rows} clickhouse logs...")
             time.sleep(5)
             ch_chart_result = measure_chart(slice, query_context, fail_on_error)
             ch_count -= 1

diff --git a/...ts/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml b/...ts/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml
@@ -118,7 +118,7 @@ sql: |
       query,
       http_user_agent
   FROM system.query_log
-  WHERE databases <> ['system'] AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish')
+  WHERE (has(databases, '{{ASPECTS_XAPI_DATABASE}}') OR has(databases, '{{ASPECTS_EVENT_SINK_DATABASE}}') OR has(databases, '{{DBT_PROFILE_TARGET_DATABASE}}')) AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish')
   ORDER BY event_time DESC
 table_name: query_log
 template_params: null