diff --git a/models/dim_dbt__current_models.sql b/models/dim_dbt__current_models.sql index b92cd5e4..4a84accb 100644 --- a/models/dim_dbt__current_models.sql +++ b/models/dim_dbt__current_models.sql @@ -22,9 +22,7 @@ model_executions as ( -- Get the most recent comile run latest_compile as ( - select - command_invocation_id, - dbt_cloud_run_id + select artifact_run_id from run_results where execution_command = 'run' order by artifact_generated_at desc @@ -39,8 +37,7 @@ latest_models as ( from models -- In a local deploy, the command id is sufficient, but not in cloud - that requires the cloud run id to achieve a match. inner join latest_compile - on models.command_invocation_id = latest_compile.command_invocation_id - or models.dbt_cloud_run_id = latest_compile.dbt_cloud_run_id + on models.artifact_run_id = latest_compile.artifact_run_id ), diff --git a/models/fct_dbt__critical_path.sql b/models/fct_dbt__critical_path.sql index 41e0a8a5..27953a3e 100644 --- a/models/fct_dbt__critical_path.sql +++ b/models/fct_dbt__critical_path.sql @@ -15,9 +15,7 @@ latest_executions as ( latest_id as ( -- Find the latest full, incremental execution - select - any_value(command_invocation_id) as command_invocation_id, - any_value(dbt_cloud_run_id) as dbt_cloud_run_id + select any_value(artifact_run_id) as artifact_run_id from latest_executions ), @@ -31,8 +29,7 @@ latest_models as ( models.model_materialization from latest_id left join models on - latest_id.command_invocation_id = models.command_invocation_id - or latest_id.dbt_cloud_run_id = models.dbt_cloud_run_id + latest_id.artifact_run_id = models.artifact_run_id ), diff --git a/models/fct_dbt__latest_full_model_executions.sql b/models/fct_dbt__latest_full_model_executions.sql index b6f7fe14..6af2e633 100644 --- a/models/fct_dbt__latest_full_model_executions.sql +++ b/models/fct_dbt__latest_full_model_executions.sql @@ -28,8 +28,7 @@ joined as ( model_executions.* from latest_full left join model_executions on - model_executions.command_invocation_id = latest_full.command_invocation_id - or model_executions.dbt_cloud_run_id = latest_full.dbt_cloud_run_id + model_executions.artifact_run_id = latest_full.artifact_run_id ), @@ -39,6 +38,7 @@ fields as ( artifact_generated_at, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, compile_started_at, query_completed_at, total_node_runtime, diff --git a/models/incremental/dim_dbt__exposures.sql b/models/incremental/dim_dbt__exposures.sql index 1c388fe1..0244def6 100644 --- a/models/incremental/dim_dbt__exposures.sql +++ b/models/incremental/dim_dbt__exposures.sql @@ -25,8 +25,7 @@ dbt_exposures_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_exposures.command_invocation_id = run_results.command_invocation_id - or dbt_exposures.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_exposures.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -41,6 +40,7 @@ fields as ( t.manifest_exposure_id, t.command_invocation_id, t.dbt_cloud_run_id, + t.artifact_run_id, t.artifact_generated_at, t.node_id, t.name, diff --git a/models/incremental/dim_dbt__models.sql b/models/incremental/dim_dbt__models.sql index 87e02b82..fa019e28 100644 --- a/models/incremental/dim_dbt__models.sql +++ b/models/incremental/dim_dbt__models.sql @@ -20,8 +20,7 @@ dbt_models_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_models.command_invocation_id = run_results.command_invocation_id - or dbt_models.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_models.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -36,6 +35,7 @@ fields as ( manifest_model_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, model_database, diff --git a/models/incremental/dim_dbt__seeds.sql b/models/incremental/dim_dbt__seeds.sql index b314f62e..7edba10d 100644 --- a/models/incremental/dim_dbt__seeds.sql +++ b/models/incremental/dim_dbt__seeds.sql @@ -20,8 +20,7 @@ dbt_seeds_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_seeds.command_invocation_id = run_results.command_invocation_id - or dbt_seeds.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_seeds.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -36,6 +35,7 @@ fields as ( manifest_seed_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, seed_database, diff --git a/models/incremental/dim_dbt__snapshots.sql b/models/incremental/dim_dbt__snapshots.sql index d662500b..899f32e8 100644 --- a/models/incremental/dim_dbt__snapshots.sql +++ b/models/incremental/dim_dbt__snapshots.sql @@ -20,8 +20,7 @@ dbt_snapshots_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_snapshots.command_invocation_id = run_results.command_invocation_id - or dbt_snapshots.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_snapshots.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -36,6 +35,7 @@ fields as ( manifest_snapshot_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, snapshot_database, diff --git a/models/incremental/dim_dbt__sources.sql b/models/incremental/dim_dbt__sources.sql index 65a3ad0d..c8734c0c 100644 --- a/models/incremental/dim_dbt__sources.sql +++ b/models/incremental/dim_dbt__sources.sql @@ -20,8 +20,7 @@ dbt_sources_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_sources.command_invocation_id = run_results.command_invocation_id - or dbt_sources.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_sources.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -36,6 +35,7 @@ fields as ( manifest_source_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, name, diff --git a/models/incremental/dim_dbt__tests.sql b/models/incremental/dim_dbt__tests.sql index 3181b949..4d5ce9e3 100644 --- a/models/incremental/dim_dbt__tests.sql +++ b/models/incremental/dim_dbt__tests.sql @@ -20,8 +20,7 @@ dbt_tests_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - dbt_tests.command_invocation_id = run_results.command_invocation_id - or dbt_tests.dbt_cloud_run_id = run_results.dbt_cloud_run_id + dbt_tests.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -36,6 +35,7 @@ fields as ( manifest_test_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, name, diff --git a/models/incremental/fct_dbt__model_executions.sql b/models/incremental/fct_dbt__model_executions.sql index 61127a65..f7bb4d7f 100644 --- a/models/incremental/fct_dbt__model_executions.sql +++ b/models/incremental/fct_dbt__model_executions.sql @@ -28,8 +28,7 @@ model_executions_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - model_executions.command_invocation_id = run_results.command_invocation_id - or model_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id + model_executions.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -61,6 +60,7 @@ fields as ( model_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/incremental/fct_dbt__run_results.sql b/models/incremental/fct_dbt__run_results.sql index ef4ccf9d..69456114 100644 --- a/models/incremental/fct_dbt__run_results.sql +++ b/models/incremental/fct_dbt__run_results.sql @@ -27,6 +27,7 @@ fields as ( artifact_generated_at, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, dbt_version, elapsed_time, execution_command, diff --git a/models/incremental/fct_dbt__seed_executions.sql b/models/incremental/fct_dbt__seed_executions.sql index 00297f5f..b0505b5e 100644 --- a/models/incremental/fct_dbt__seed_executions.sql +++ b/models/incremental/fct_dbt__seed_executions.sql @@ -28,8 +28,7 @@ seed_executions_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - seed_executions.command_invocation_id = run_results.command_invocation_id - or seed_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id + seed_executions.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -60,6 +59,7 @@ fields as ( seed_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/incremental/fct_dbt__snapshot_executions.sql b/models/incremental/fct_dbt__snapshot_executions.sql index f451f522..77d84405 100644 --- a/models/incremental/fct_dbt__snapshot_executions.sql +++ b/models/incremental/fct_dbt__snapshot_executions.sql @@ -28,8 +28,7 @@ snapshot_executions_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - snapshot_executions.command_invocation_id = run_results.command_invocation_id - or snapshot_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id + snapshot_executions.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -60,6 +59,7 @@ fields as ( snapshot_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/incremental/fct_dbt__test_executions.sql b/models/incremental/fct_dbt__test_executions.sql index ffefd468..31d32d8d 100644 --- a/models/incremental/fct_dbt__test_executions.sql +++ b/models/incremental/fct_dbt__test_executions.sql @@ -21,8 +21,7 @@ test_executions_incremental as ( -- Inner join with run results to enforce consistency and avoid race conditions. -- https://github.com/brooklyn-data/dbt_artifacts/issues/75 inner join run_results on - test_executions.command_invocation_id = run_results.command_invocation_id - or test_executions.dbt_cloud_run_id = run_results.dbt_cloud_run_id + test_executions.artifact_run_id = run_results.artifact_run_id {% if is_incremental() %} -- this filter will only be applied on an incremental run @@ -37,6 +36,7 @@ fields as ( test_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/incremental/int_dbt__model_executions.sql b/models/incremental/int_dbt__model_executions.sql index f4a36506..1983ac9c 100644 --- a/models/incremental/int_dbt__model_executions.sql +++ b/models/incremental/int_dbt__model_executions.sql @@ -26,6 +26,7 @@ fields as ( model_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/staging/stg_dbt__artifacts.sql b/models/staging/stg_dbt__artifacts.sql index 8c91c629..b1fa5cc3 100644 --- a/models/staging/stg_dbt__artifacts.sql +++ b/models/staging/stg_dbt__artifacts.sql @@ -36,6 +36,8 @@ artifacts as ( select command_invocation_id, dbt_cloud_run_id, + -- This ID provides a reliable ID, regardless of whether running in a local or cloud environment. + sha2_hex(coalesce(dbt_cloud_run_id::string, command_invocation_id::string), 256) as artifact_run_id, generated_at, path, artifact_type, diff --git a/models/staging/stg_dbt__exposures.sql b/models/staging/stg_dbt__exposures.sql index c2aabe57..ac87f41a 100644 --- a/models/staging/stg_dbt__exposures.sql +++ b/models/staging/stg_dbt__exposures.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:name::string as name, @@ -38,6 +39,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_exposure_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, name, diff --git a/models/staging/stg_dbt__model_executions.sql b/models/staging/stg_dbt__model_executions.sql index 09267beb..f023444b 100644 --- a/models/staging/stg_dbt__model_executions.sql +++ b/models/staging/stg_dbt__model_executions.sql @@ -26,6 +26,7 @@ fields as ( select dbt_run.command_invocation_id, dbt_run.dbt_cloud_run_id, + dbt_run.artifact_run_id, dbt_run.generated_at as artifact_generated_at, coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh, result.value:unique_id::string as node_id, @@ -54,6 +55,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as model_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/staging/stg_dbt__models.sql b/models/staging/stg_dbt__models.sql index 5866e857..0f3bdc46 100644 --- a/models/staging/stg_dbt__models.sql +++ b/models/staging/stg_dbt__models.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:database::string as model_database, @@ -40,6 +41,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_model_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, model_database, diff --git a/models/staging/stg_dbt__run_results.sql b/models/staging/stg_dbt__run_results.sql index b732f57a..c1762ce4 100644 --- a/models/staging/stg_dbt__run_results.sql +++ b/models/staging/stg_dbt__run_results.sql @@ -27,6 +27,7 @@ fields as ( generated_at as artifact_generated_at, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, data:metadata:dbt_version::string as dbt_version, data:metadata:env as env, data:elapsed_time::float as elapsed_time, diff --git a/models/staging/stg_dbt__seed_executions.sql b/models/staging/stg_dbt__seed_executions.sql index 374809f6..32fa5404 100644 --- a/models/staging/stg_dbt__seed_executions.sql +++ b/models/staging/stg_dbt__seed_executions.sql @@ -26,6 +26,7 @@ fields as ( select dbt_run.command_invocation_id, dbt_run.dbt_cloud_run_id, + dbt_run.artifact_run_id, dbt_run.generated_at as artifact_generated_at, coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh, result.value:unique_id::string as node_id, @@ -55,6 +56,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as seed_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/staging/stg_dbt__seeds.sql b/models/staging/stg_dbt__seeds.sql index 74c6902e..5fcabe9c 100644 --- a/models/staging/stg_dbt__seeds.sql +++ b/models/staging/stg_dbt__seeds.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:database::string as seed_database, @@ -39,6 +40,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_seed_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, seed_database, diff --git a/models/staging/stg_dbt__snapshot_executions.sql b/models/staging/stg_dbt__snapshot_executions.sql index f584711e..5ba17721 100644 --- a/models/staging/stg_dbt__snapshot_executions.sql +++ b/models/staging/stg_dbt__snapshot_executions.sql @@ -26,6 +26,7 @@ fields as ( select dbt_run.command_invocation_id, dbt_run.dbt_cloud_run_id, + dbt_run.artifact_run_id, dbt_run.generated_at as artifact_generated_at, coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh, result.value:unique_id::string as node_id, @@ -55,6 +56,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as snapshot_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/staging/stg_dbt__snapshots.sql b/models/staging/stg_dbt__snapshots.sql index 2a5f9a06..0e8a0ab1 100644 --- a/models/staging/stg_dbt__snapshots.sql +++ b/models/staging/stg_dbt__snapshots.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:database::string as snapshot_database, @@ -39,6 +40,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_snapshot_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, snapshot_database, diff --git a/models/staging/stg_dbt__sources.sql b/models/staging/stg_dbt__sources.sql index bf971c59..c2db06a2 100644 --- a/models/staging/stg_dbt__sources.sql +++ b/models/staging/stg_dbt__sources.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:name::string as name, @@ -38,6 +39,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_source_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, name, diff --git a/models/staging/stg_dbt__test_executions.sql b/models/staging/stg_dbt__test_executions.sql index b1dc4f30..7f9687f8 100644 --- a/models/staging/stg_dbt__test_executions.sql +++ b/models/staging/stg_dbt__test_executions.sql @@ -26,6 +26,7 @@ fields as ( select dbt_run.command_invocation_id, dbt_run.dbt_cloud_run_id, + dbt_run.artifact_run_id, dbt_run.generated_at as artifact_generated_at, coalesce(dbt_run.data:args:full_refresh, 'false')::boolean as was_full_refresh, result.value:unique_id::string as node_id, @@ -55,6 +56,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as test_execution_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, was_full_refresh, node_id, diff --git a/models/staging/stg_dbt__tests.sql b/models/staging/stg_dbt__tests.sql index 4ec9ae68..85451aee 100644 --- a/models/staging/stg_dbt__tests.sql +++ b/models/staging/stg_dbt__tests.sql @@ -18,6 +18,7 @@ flatten as ( select manifests.command_invocation_id, manifests.dbt_cloud_run_id, + manifests.artifact_run_id, manifests.generated_at as artifact_generated_at, node.key as node_id, node.value:name::string as name, @@ -36,6 +37,7 @@ surrogate_key as ( {{ dbt_utils.surrogate_key(['command_invocation_id', 'node_id']) }} as manifest_test_id, command_invocation_id, dbt_cloud_run_id, + artifact_run_id, artifact_generated_at, node_id, name,