fix: Reduce CH memory usage by using uniqCombinedMerge

openedx · Apr 29, 2024 · ae355dd · ae355dd
1 parent aaf8d0c
commit ae355dd
Show file tree

Hide file tree

Showing 16 changed files with 173 additions and 166 deletions.
diff --git a/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Courses.yaml b/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Courses.yaml
@@ -2,7 +2,7 @@ _file_name: Active_Courses.yaml
 cache_timeout: null
 certification_details: null
 certified_by: null
-dataset_uuid: fed94e41-966a-5faa-b2ad-d7e967e304c0
+dataset_uuid: f13a4d42-2f44-58a4-99d2-2f480d63fdf1
 description: null
 params:
   adhoc_filters:
@@ -24,30 +24,14 @@ params:
   markerEnabled: false
   markerSize: 6
   metrics:
-  - aggregate: MAX
-    column:
-      advanced_data_type: null
-      certification_details: null
-      certified_by: null
-      column_name: courses_cnt
-      description: null
-      expression: null
-      filterable: true
-      groupby: true
-      id: 889
-      is_certified: false
-      is_dttm: false
-      python_date_format: null
-      type: UINT64
-      type_generic: 0
-      verbose_name: null
-      warning_markdown: null
+  - aggregate: null
+    column: null
     datasourceWarning: false
-    expressionType: SIMPLE
+    expressionType: SQL
     hasCustomLabel: true
     label: Active Courses
     optionName: metric_hvxlgfij23e_68a6guxh855
-    sqlExpression: null
+    sqlExpression: uniqCombinedMerge(courses_cnt)
   only_total: true
   opacity: 0.2
   order_desc: true
@@ -75,13 +59,13 @@ params:
   y_axis_title_margin: 15
   y_axis_title_position: Left
   zoomable: true
-query_context: '{"datasource":{"id":84,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_hour","op":"TEMPORAL_RANGE","val":"No
-  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"courses_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":889,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null}],"orderby":[[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"courses_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":889,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
-  Courses":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"84__table","viz_type":"echarts_timeseries_line","slice_id":3402,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"courses_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":889,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
-  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":false,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","truncateXAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[1942],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+query_context: '{"datasource":{"id":829,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_hour","op":"TEMPORAL_RANGE","val":"No
+  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
+  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"}],"orderby":[[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
+  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
+  Courses":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"829__table","viz_type":"echarts_timeseries_line","slice_id":3402,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"aggregate":null,"column":null,"datasourceWarning":false,"expressionType":"SQL","hasCustomLabel":true,"label":"Active
+  Courses","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":"uniqCombinedMerge(courses_cnt)"}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
+  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":false,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","truncateXAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[2422],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
 slice_name: Active Courses
 uuid: 138eb72a-f73f-42b5-96d6-3417715c3519
 version: 1.0.0

diff --git a/...emplates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Learners.yaml b/...emplates/aspects/build/aspects-superset/openedx-assets/assets/charts/Active_Learners.yaml
@@ -2,7 +2,7 @@ _file_name: Active_Learners.yaml
 cache_timeout: null
 certification_details: null
 certified_by: null
-dataset_uuid: fed94e41-966a-5faa-b2ad-d7e967e304c0
+dataset_uuid: 5d7fa3b0-5692-5229-b8fc-4711b88d5401
 description: null
 params:
   adhoc_filters:
@@ -25,30 +25,14 @@ params:
   markerEnabled: false
   markerSize: 6
   metrics:
-  - aggregate: MAX
-    column:
-      advanced_data_type: null
-      certification_details: null
-      certified_by: null
-      column_name: actors_cnt
-      description: null
-      expression: null
-      filterable: true
-      groupby: true
-      id: 890
-      is_certified: false
-      is_dttm: false
-      python_date_format: null
-      type: UINT64
-      type_generic: 0
-      verbose_name: null
-      warning_markdown: null
+  - aggregate: null
+    column: null
     datasourceWarning: false
-    expressionType: SIMPLE
+    expressionType: SQL
     hasCustomLabel: true
     label: Active Learners
     optionName: metric_hvxlgfij23e_68a6guxh855
-    sqlExpression: null
+    sqlExpression: uniqCombinedMerge(actors_cnt)
   minorSplitLine: false
   minorTicks: false
   only_total: true
@@ -80,12 +64,12 @@ params:
   y_axis_title_position: Left
   zoomable: true
 query_context: '{"datasource":{"id":84,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_hour","op":"TEMPORAL_RANGE","val":"No
-  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actors_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":890,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null}],"orderby":[[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actors_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":890,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
-  Learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"84__table","viz_type":"echarts_timeseries_line","slice_id":3403,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"aggregate":"MAX","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actors_cnt","description":null,"expression":null,"filterable":true,"groupby":true,"id":890,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"UINT64","type_generic":0,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Active
-  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855","sqlExpression":null}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
-  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"area":false,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"minorTicks":false,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","minorSplitLine":false,"truncateXAxis":false,"truncateYAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[1942],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+  filter"}],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1D","columnType":"BASE_AXIS","sqlExpression":"emission_hour","label":"emission_hour","expressionType":"SQL"}],"metrics":[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
+  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
+  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_hour"],"columns":[],"aggregates":{"Active
+  Learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"84__table","viz_type":"echarts_timeseries_line","slice_id":3403,"x_axis":"emission_hour","time_grain_sqla":"P1D","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"uniqCombinedMerge(actors_cnt)","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Active
+  Learners","optionName":"metric_hvxlgfij23e_68a6guxh855"}],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
+  filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_hour"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","only_total":true,"area":false,"opacity":0.2,"markerEnabled":false,"markerSize":6,"zoomable":true,"minorTicks":false,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"%Y-%m-%d","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"SMART_NUMBER","minorSplitLine":false,"truncateXAxis":false,"truncateYAxis":false,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[2422],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
 slice_name: Active Learners
 uuid: b9b602ec-001b-4584-aa24-9201f3b8746f
 version: 1.0.0

diff --git a/...ts/build/aspects-superset/openedx-assets/assets/charts/Active_Users_Per_Organization.yaml b/...ts/build/aspects-superset/openedx-assets/assets/charts/Active_Users_Per_Organization.yaml
@@ -11,6 +11,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   color_scheme: supersetColors
   date_format: smart_date
   extra_form_data: {}
@@ -57,9 +58,7 @@ params:
   show_total: true
   sort_by_metric: true
   viz_type: pie
-query_context: '{"datasource":{"id":52,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"Last
-  quarter"}],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":["org"],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":44,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_rxofgwpgk7d_1xcp4i69sjj","sqlExpression":null}],"orderby":[[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":44,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_rxofgwpgk7d_1xcp4i69sjj","sqlExpression":null},false]],"annotation_layers":[],"row_limit":250,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"52__table","viz_type":"pie","slice_id":2741,"groupby":["org"],"metric":{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":44,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"expressionType":"SIMPLE","hasCustomLabel":false,"isNew":false,"label":"COUNT_DISTINCT(actor_id)","optionName":"metric_rxofgwpgk7d_1xcp4i69sjj","sqlExpression":null},"adhoc_filters":[{"clause":"WHERE","comparator":"Last
-  quarter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"emission_time"}],"row_limit":250,"sort_by_metric":true,"color_scheme":"supersetColors","show_labels_threshold":5,"show_legend":false,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"label_type":"key","number_format":",d","date_format":"smart_date","show_labels":true,"labels_outside":true,"label_line":true,"show_total":true,"outerRadius":70,"innerRadius":30,"extra_form_data":{},"dashboards":[1942],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+query_context: null
 slice_name: Active Users Per Organization
 uuid: 02af00f5-6539-428f-b3ab-632997de7528
 version: 1.0.0