-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: use dynamodb schedules from the deployments table (#668)
- Loading branch information
1 parent
02c5443
commit d79519a
Showing
5 changed files
with
81 additions
and
102 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 10 additions & 46 deletions
56
data/transform/models/staging/dynamodb/stg_dynamodb__project_deployments.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,46 +1,10 @@ | ||
WITH source AS ( | ||
|
||
SELECT | ||
*, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 1 -- noqa: RF05 | ||
) AS tenant_resource_key, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 2 -- noqa: RF05 | ||
) AS cloud_project_id | ||
FROM {{ source('tap_dynamodb', 'project_deployments_table') }} | ||
), | ||
|
||
clean_source AS ( | ||
SELECT | ||
*, | ||
{{ dbt_utils.surrogate_key( | ||
['tenant_resource_key', 'cloud_project_id', 'deployment_name'] | ||
) }} AS deployment_surrogate_key, | ||
ROW_NUMBER() OVER ( | ||
PARTITION BY deployment_surrogate_key | ||
ORDER BY CAST(_sdc_batched_at AS TIMESTAMP_NTZ) DESC | ||
) AS row_num | ||
FROM source | ||
), | ||
|
||
renamed AS ( | ||
|
||
SELECT | ||
deployment_surrogate_key, | ||
tenant_resource_key, | ||
cloud_project_id, | ||
git_rev, | ||
git_rev_hash, | ||
SHA2_HEX(deployment_name) AS cloud_deployment_name_hash, | ||
SHA2_HEX(environment_name) AS cloud_environment_name_hash, | ||
CAST( | ||
last_deployed_timestamp AS TIMESTAMP_NTZ | ||
) AS last_deployed_timestamp | ||
FROM clean_source | ||
WHERE row_num = 1 | ||
|
||
) | ||
|
||
SELECT * | ||
FROM renamed | ||
SELECT | ||
deployment_surrogate_key, | ||
tenant_resource_key, | ||
cloud_project_id, | ||
git_rev, | ||
git_rev_hash, | ||
cloud_deployment_name_hash, | ||
cloud_environment_name_hash, | ||
last_deployed_timestamp | ||
FROM {{ ref('stg_dynamodb__project_deployments_base') }} |
47 changes: 47 additions & 0 deletions
47
data/transform/models/staging/dynamodb/stg_dynamodb__project_deployments_base.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
WITH source AS ( | ||
|
||
SELECT | ||
*, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 1 -- noqa: RF05 | ||
) AS tenant_resource_key, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 2 -- noqa: RF05 | ||
) AS cloud_project_id | ||
FROM {{ source('tap_dynamodb', 'project_deployments_table') }} | ||
), | ||
|
||
clean_source AS ( | ||
SELECT | ||
*, | ||
{{ dbt_utils.surrogate_key( | ||
['tenant_resource_key', 'cloud_project_id', 'deployment_name'] | ||
) }} AS deployment_surrogate_key, | ||
ROW_NUMBER() OVER ( | ||
PARTITION BY deployment_surrogate_key | ||
ORDER BY CAST(_sdc_batched_at AS TIMESTAMP_NTZ) DESC | ||
) AS row_num | ||
FROM source | ||
), | ||
|
||
renamed AS ( | ||
|
||
SELECT | ||
deployment_surrogate_key, | ||
tenant_resource_key, | ||
cloud_project_id, | ||
git_rev, | ||
git_rev_hash, | ||
schedules AS schedules_obj, | ||
SHA2_HEX(deployment_name) AS cloud_deployment_name_hash, | ||
SHA2_HEX(environment_name) AS cloud_environment_name_hash, | ||
CAST( | ||
last_deployed_timestamp AS TIMESTAMP_NTZ | ||
) AS last_deployed_timestamp | ||
FROM clean_source | ||
WHERE row_num = 1 | ||
|
||
) | ||
|
||
SELECT * | ||
FROM renamed |
73 changes: 19 additions & 54 deletions
73
data/transform/models/staging/dynamodb/stg_dynamodb__project_schedules_table.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,54 +1,19 @@ | ||
WITH source AS ( | ||
|
||
SELECT | ||
*, | ||
SPLIT_PART( | ||
"DEPLOYMENT_NAME::SCHEDULE_NAME", '::', 1 -- noqa: RF05 | ||
) AS cloud_deployment_name, | ||
REPLACE( | ||
SPLIT_PART( | ||
"DEPLOYMENT_NAME::SCHEDULE_NAME", -- noqa: RF05 | ||
'::', | ||
2 | ||
), '-', '_' | ||
) AS cloud_schedule_name, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 1 -- noqa: RF05 | ||
) AS tenant_resource_key, | ||
SPLIT_PART( | ||
"TENANT_RESOURCE_KEY::PROJECT_ID", '::', 2 -- noqa: RF05 | ||
) AS cloud_project_id | ||
FROM {{ source('tap_dynamodb', 'project_schedules_table') }} | ||
), | ||
|
||
clean_source AS ( | ||
SELECT | ||
*, | ||
{{ dbt_utils.surrogate_key( | ||
['tenant_resource_key', 'cloud_project_id', 'cloud_deployment_name', 'cloud_schedule_name'] | ||
) }} AS schedule_surrogate_key, | ||
ROW_NUMBER() OVER ( | ||
PARTITION BY schedule_surrogate_key | ||
ORDER BY CAST(_sdc_batched_at AS TIMESTAMP_NTZ) DESC | ||
) AS row_num | ||
FROM source | ||
), | ||
|
||
renamed AS ( | ||
|
||
SELECT | ||
schedule_surrogate_key, | ||
interval, | ||
enabled AS is_enabled, | ||
tenant_resource_key, | ||
cloud_project_id, | ||
eventbridge_name, | ||
SHA2_HEX(cloud_deployment_name) AS cloud_deployment_name_hash, | ||
SHA2_HEX(cloud_schedule_name) AS cloud_schedule_name_hash | ||
FROM clean_source | ||
WHERE row_num = 1 | ||
|
||
) | ||
|
||
SELECT * | ||
FROM renamed | ||
SELECT | ||
{{ dbt_utils.surrogate_key( | ||
[ | ||
'stg_dynamodb__project_deployments_base.deployment_surrogate_key', | ||
'CAST(schedule_elem.value:schedule_name AS STRING)' | ||
] | ||
) }} AS schedule_surrogate_key, | ||
stg_dynamodb__project_deployments_base.deployment_surrogate_key, | ||
CAST(schedule_elem.value:interval AS STRING) AS schedule_interval, | ||
CAST(schedule_elem.value:enabled AS BOOLEAN) AS schedule_is_enabled, | ||
stg_dynamodb__project_deployments_base.tenant_resource_key, | ||
stg_dynamodb__project_deployments_base.cloud_project_id, | ||
CAST(schedule_elem.value:eventbridge_name AS STRING) AS eventbridge_name, | ||
stg_dynamodb__project_deployments_base.cloud_deployment_name_hash, | ||
SHA2_HEX( | ||
CAST(schedule_elem.value:schedule_name AS STRING) | ||
) AS cloud_schedule_name_hash | ||
FROM {{ ref('stg_dynamodb__project_deployments_base') }}, | ||
LATERAL FLATTEN(input => schedules_obj) AS schedule_elem |