From cfae721aa4112ad61272de7b439de3538b860310 Mon Sep 17 00:00:00 2001 From: Sara Burns Date: Thu, 19 Dec 2024 09:18:20 -0500 Subject: [PATCH] fix: log missing rows --- .../aspects/apps/superset/pythonpath/performance_metrics.py | 5 +++-- .../openedx-assets/assets/datasets/query_log.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py index 89997862b..ac0b77f02 100644 --- a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py +++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py @@ -232,8 +232,9 @@ def get_query_log_from_clickhouse(report, query_contexts, print_sql, fail_on_err # Run CH query until results for all slices are returned ch_count = 6 while ch_count > 0: - if ch_chart_result["queries"][0]["rowcount"] < chart_count: - logger.info("Waiting for clickhouse log...") + missing_rows = chart_count - ch_chart_result["queries"][0]["rowcount"] + if missing_rows > 0: + logger.info(f"Waiting for {missing_rows} clickhouse logs...") time.sleep(5) ch_chart_result = measure_chart(slice, query_context, fail_on_error) ch_count -= 1 diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml index b46e09d33..b9c1b9988 100644 --- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml +++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/query_log.yaml @@ -118,7 +118,7 @@ sql: | query, http_user_agent FROM system.query_log - WHERE databases <> ['system'] AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish') + WHERE (has(databases, '{{ASPECTS_XAPI_DATABASE}}') OR has(databases, '{{ASPECTS_EVENT_SINK_DATABASE}}') OR has(databases, '{{DBT_PROFILE_TARGET_DATABASE}}')) AND (http_user_agent LIKE 'aspects-%') AND (type = 'QueryFinish') ORDER BY event_time DESC table_name: query_log template_params: null