[DENG-8890] Add BI engine information to monitoring.jobs_by_organization and monitoring.bigquery_usage (#8242)

Author: scholtzan

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql

@@ -28,6 +28,9 @@ WITH jobs_by_org AS (
     error_result.reason AS error_reason,
     error_result.message AS error_message,
     query_info_resource_warning AS resource_warning,
+    bi_engine_mode,
+    acceleration_mode,
+    bi_engine_reasons
   FROM
     `moz-fx-data-shared-prod.monitoring_derived.jobs_by_organization_v1` AS jobs
   LEFT JOIN
@@ -89,6 +92,9 @@ SELECT DISTINCT
   @submission_date AS submission_date,
   jp.labels,
   jp.is_materialized_view_refresh,
+  jo.bi_engine_mode,
+  jo.acceleration_mode,
+  jo.bi_engine_reasons,
 FROM
   jobs_by_org AS jo
 LEFT JOIN

sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml

@@ -148,3 +148,27 @@ fields:
   name: is_materialized_view_refresh
   type: BOOLEAN
   description: Whether or not the query is a materialized view refresh, based on the query string.
+
+- mode: NULLABLE
+  name: bi_engine_mode
+  type: STRING
+  description: High-level BI Engine status for the job (e.g., ENABLED, DISABLED, PARTIAL).
+
+- mode: NULLABLE
+  name: acceleration_mode
+  type: STRING
+  description: How BI Engine was used when acceleration occurred (e.g., FULL_QUERY, FULL_INPUT, PARTIAL_INPUT).
+
+- mode: REPEATED
+  name: bi_engine_reasons
+  type: RECORD
+  description: Reasons explaining why BI Engine was PARTIAL or DISABLED for this job.
+  fields:
+  - mode: NULLABLE
+    name: code
+    type: STRING
+    description: Machine-readable reason code.
+  - mode: NULLABLE
+    name: message
+    type: STRING
+    description: Human-readable explanation of the reason.

sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/query.py

@@ -62,6 +62,9 @@ def create_query(job_date: date, project: str):
           DATE(creation_time) as creation_date,
           materialized_view_statistics,
           query_dialect,
+          bi_engine_statistics.bi_engine_mode AS bi_engine_mode,
+          bi_engine_statistics.acceleration_mode AS acceleration_mode,
+          bi_engine_statistics.bi_engine_reasons AS bi_engine_reasons,
         FROM
           `{project}.region-us.INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION`
         WHERE

sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/schema.yaml

@@ -222,3 +222,27 @@ fields:
   name: query_dialect
   type: STRING
   description: The query dialect used for the job.
+
+- mode: NULLABLE
+  name: bi_engine_mode
+  type: STRING
+  description: High-level BI Engine status for the job (e.g., ENABLED, DISABLED, PARTIAL).
+
+- mode: NULLABLE
+  name: acceleration_mode
+  type: STRING
+  description: How BI Engine was used when acceleration occurred (e.g., FULL_QUERY, FULL_INPUT, PARTIAL_INPUT).
+
+- mode: REPEATED
+  name: bi_engine_reasons
+  type: RECORD
+  description: Reasons explaining why BI Engine was PARTIAL or DISABLED for this job.
+  fields:
+  - mode: NULLABLE
+    name: code
+    type: STRING
+    description: Machine-readable reason code.
+  - mode: NULLABLE
+    name: message
+    type: STRING
+    description: Human-readable explanation of the reason.