From 05d9dc6ee5d261cc4a61628e96f99ab9e3e66eb8 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Wed, 15 Jan 2025 10:54:16 -0500 Subject: [PATCH 1/7] Set retention on client_level tables for glean_usage tables --- .../templates/baseline_clients_daily_v1.metadata.yaml | 1 + .../templates/baseline_clients_last_seen_v1.metadata.yaml | 1 + .../glean_usage/templates/events_stream_v1.metadata.yaml | 1 + .../glean_usage/templates/metrics_clients_daily.metadata.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml index 0629872a6c7..73bb37c5750 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml @@ -19,6 +19,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - normalized_channel diff --git a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml index d1dd9250034..a1e40dc1221 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml @@ -21,6 +21,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - normalized_channel diff --git a/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml b/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml index 18aa2d4e2b6..763b81fc513 100644 --- a/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml @@ -26,6 +26,7 @@ bigquery: type: day field: submission_timestamp require_partition_filter: true + expiration_days: 775 clustering: fields: - sample_id diff --git a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml index e3ef42a109e..cbd5e319607 100644 --- a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml +++ b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml @@ -15,6 +15,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - normalized_channel From 7c30878a66359f5746321f99a0d412f55b5efba7 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Wed, 15 Jan 2025 11:27:27 -0500 Subject: [PATCH 2/7] Add table_type label for event_stream table --- .../glean_usage/templates/events_stream_v1.metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml b/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml index 763b81fc513..9b0dfa68ae3 100644 --- a/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/events_stream_v1.metadata.yaml @@ -11,6 +11,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} From 051ee24eac567f567b44c4626db6148d2fe9e689 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Wed, 15 Jan 2025 12:56:47 -0500 Subject: [PATCH 3/7] change 780 to 775 --- .../firefox_desktop_derived/pageload_1pct_v1/metadata.yaml | 3 ++- .../firefox_desktop_derived/pageload_nightly_v1/metadata.yaml | 3 ++- .../templates/baseline_clients_last_seen_v1.metadata.yaml | 1 + .../glean_usage/templates/metrics_clients_daily.metadata.yaml | 1 + 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_1pct_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_1pct_v1/metadata.yaml index 609263975d4..6eeecda2725 100644 --- a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_1pct_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_1pct_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: - dpalmeiro@mozilla.com labels: incremental: true + table_type: client_level scheduling: dag_name: bqetl_pageload_v1 task_name: firefox_desktop_pageload_1pct__v1 @@ -16,7 +17,7 @@ bigquery: type: day field: submission_timestamp require_partition_filter: true - expiration_days: 780 + expiration_days: 775 range_partitioning: null clustering: fields: diff --git a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_nightly_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_nightly_v1/metadata.yaml index 41066f0c65f..1892031ea10 100644 --- a/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_nightly_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_desktop_derived/pageload_nightly_v1/metadata.yaml @@ -7,6 +7,7 @@ owners: - dpalmeiro@mozilla.com labels: incremental: true + table_type: client_level scheduling: dag_name: bqetl_pageload_v1 task_name: firefox_desktop_pageload_nightly__v1 @@ -16,6 +17,6 @@ bigquery: type: day field: submission_timestamp require_partition_filter: true - expiration_days: 780 + expiration_days: 775 range_partitioning: null references: {} diff --git a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml index a1e40dc1221..f7c6e4725f5 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml @@ -12,6 +12,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} diff --git a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml index cbd5e319607..88d7a21e489 100644 --- a/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml +++ b/sql_generators/glean_usage/templates/metrics_clients_daily.metadata.yaml @@ -7,6 +7,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} From 628dfdab23a39ea8762bb2556d46f015569b1cf8 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Thu, 16 Jan 2025 09:54:55 -0500 Subject: [PATCH 4/7] Add some more client level glean usage tables --- .../clients_last_seen_joined_v1/metadata.yaml | 5 +++-- .../templates/baseline_clients_daily_v1.metadata.yaml | 1 + .../templates/clients_last_seen_joined.metadata.yaml | 2 ++ .../templates/metrics_clients_last_seen.metadata.yaml | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_joined_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_joined_v1/metadata.yaml index e0549e71965..653e387ebec 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_joined_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_joined_v1/metadata.yaml @@ -16,7 +16,8 @@ labels: application: firefox schedule: daily dag: bqetl_main_summary - owner1: ascholtzs + owner1: ascholtz + table_type: client_level scheduling: dag_name: bqetl_main_summary start_date: '2021-01-19' @@ -27,7 +28,7 @@ bigquery: type: day field: submission_date require_partition_filter: false - expiration_days: null + expiration_days: 775 range_partitioning: null clustering: fields: diff --git a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml index 73bb37c5750..20048c7119a 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml @@ -11,6 +11,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} diff --git a/sql_generators/glean_usage/templates/clients_last_seen_joined.metadata.yaml b/sql_generators/glean_usage/templates/clients_last_seen_joined.metadata.yaml index 26dd50a028b..778b6b4399b 100644 --- a/sql_generators/glean_usage/templates/clients_last_seen_joined.metadata.yaml +++ b/sql_generators/glean_usage/templates/clients_last_seen_joined.metadata.yaml @@ -16,6 +16,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} @@ -25,6 +26,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - normalized_channel diff --git a/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml b/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml index 737f3d4b153..4b271b0254a 100644 --- a/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml +++ b/sql_generators/glean_usage/templates/metrics_clients_last_seen.metadata.yaml @@ -12,6 +12,7 @@ owners: labels: incremental: true schedule: daily + table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} @@ -20,6 +21,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - normalized_channel From ecd153f22f7b9c597431d776ddb0a3764953dcf8 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Thu, 16 Jan 2025 10:31:04 -0500 Subject: [PATCH 5/7] Add some more client level glean usage tables --- .../telemetry_derived/clients_daily_event_v1/metadata.yaml | 3 ++- .../clients_last_seen_event_v1/metadata.yaml | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml index 362c82264b8..8295cffde5e 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml @@ -8,8 +8,9 @@ description: |- labels: application: firefox schedule: daily + table_type: client_level owners: - - jklukas@mozilla.com + - ascholtz@mozilla.com scheduling: dag_name: bqetl_main_summary start_date: "2021-01-19" diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_event_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_event_v1/metadata.yaml index 2e5829851ac..3593e5ca1ab 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_event_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_last_seen_event_v1/metadata.yaml @@ -6,12 +6,12 @@ description: |- We join main ping and event data together in clients_last_seen_joined_v1, which then becomes the source for the user-facing view. owners: -- jklukas@mozilla.com +- ascholtz@mozilla.com labels: application: firefox schedule: daily dag: bqetl_main_summary - owner1: jklukas + owner1: ascholtz scheduling: dag_name: bqetl_main_summary start_date: '2021-01-19' @@ -22,7 +22,7 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: null + expiration_days: 775 clustering: fields: - sample_id From 1cbaefa5c680f5dacd8b77ea8f3046d365621e0e Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Fri, 31 Jan 2025 14:47:45 -0500 Subject: [PATCH 6/7] Fix based on reviews --- .../telemetry_derived/clients_daily_event_v1/metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml index 8295cffde5e..90c67d6bb06 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/clients_daily_event_v1/metadata.yaml @@ -20,6 +20,7 @@ bigquery: type: day field: submission_date require_partition_filter: true + expiration_days: 775 clustering: fields: - sample_id From 5be997cdb84fa907ad92c3bd0482b1dc1b21532d Mon Sep 17 00:00:00 2001 From: Katie Windau Date: Tue, 4 Feb 2025 14:17:19 -0600 Subject: [PATCH 7/7] Moving baseline_clients_daily_v1 and baseline_clients_last_seen_v1 to a separate PR --- .../templates/baseline_clients_daily_v1.metadata.yaml | 2 -- .../templates/baseline_clients_last_seen_v1.metadata.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml index 20048c7119a..0629872a6c7 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_daily_v1.metadata.yaml @@ -11,7 +11,6 @@ owners: labels: incremental: true schedule: daily - table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} @@ -20,7 +19,6 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: 775 clustering: fields: - normalized_channel diff --git a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml index f7c6e4725f5..d1dd9250034 100644 --- a/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml +++ b/sql_generators/glean_usage/templates/baseline_clients_last_seen_v1.metadata.yaml @@ -12,7 +12,6 @@ owners: labels: incremental: true schedule: daily - table_type: client_level scheduling: dag_name: bqetl_glean_usage task_group: {{ app_name }} @@ -22,7 +21,6 @@ bigquery: type: day field: submission_date require_partition_filter: true - expiration_days: 775 clustering: fields: - normalized_channel