NTD: staging and mart tables for 2022 agency data (#3605)

* add new tables to yml files

* add new staging and mart tables for 2022 agency information

* fix duplicate entries in yml file

warehouse/models/mart/ntd_annual_reporting/dim_2022_agency_information.sql

@@ -0,0 +1,56 @@
+WITH staging_agency_information AS (
+    SELECT *
+    FROM {{ ref('stg_ntd__2022_agency_information') }}
+),
+
+dim_2022_agency_information AS (
+    SELECT *
+    FROM staging_agency_information
+)
+
+SELECT
+    number_of_state_counties,
+    tam_tier,
+    personal_vehicles,
+    uza_name,
+    tribal_area_name,
+    service_area_sq_miles,
+    voms_do,
+    url,
+    region,
+    state_admin_funds_expended,
+    zip_code_ext,
+    zip_code,
+    ueid,
+    address_line_2,
+    number_of_counties_with_service,
+    reporter_acronym,
+    original_due_date,
+    sq_miles,
+    address_line_1,
+    p_o__box,
+    fy_end_date,
+    service_area_pop,
+    state,
+    subrecipient_type,
+    population,
+    reporting_module,
+    volunteer_drivers,
+    doing_business_as,
+    reporter_type,
+    legacy_ntd_id,
+    total_voms,
+    primary_uza_uace_code,
+    reported_by_name,
+    fta_recipient_id,
+    city,
+    voms_pt,
+    organization_type,
+    agency_name,
+    ntd_id,
+    reported_by_ntd_id,
+    density,
+    state_parent_ntd_id,
+    dt,
+    execution_ts
+FROM dim_2022_agency_information

warehouse/models/staging/ntd_annual_reporting/_src.yml

@@ -179,6 +179,9 @@ sources:
     database: "{{ env_var('DBT_SOURCE_DATABASE', var('SOURCE_DATABASE')) }}"
     schema: external_ntd__annual_reporting
     tables:
+      - name: 2022__annual_database_agency_information
+      - name: 2023__annual_database_agency_information
+      - name: 2023__annual_database_contractual_relationships
       - name: multi_year__breakdowns
       - name: multi_year__breakdowns_by_agency
       - name: multi_year__capital_expenses_by_capital_use
@@ -501,5 +504,3 @@ sources:
       - name: multi_year__track_and_roadway_guideway_age_distribution
       - name: multi_year__vehicles_age_distribution
       - name: multi_year__vehicles_type_count_by_agency
-      - name: 2023__annual_database_agency_information
-      - name: 2023__annual_database_contractual_relationships

warehouse/models/staging/ntd_annual_reporting/_stg_ntd_annual_reporting.yml

@@ -174,6 +174,9 @@ x-common-fields:
     description: '{{ doc("ntd_execution_timestamp") }}'
 
 models:
+  - name: stg_ntd__2022_agency_information
+  - name: stg_ntd__2023_agency_information
+  - name: stg_ntd__2023_contractual_relationships
   - name: stg_ntd__breakdowns
   - name: stg_ntd__breakdowns_by_agency
   - name: stg_ntd__capital_expenses_by_capital_use
@@ -495,5 +498,3 @@ models:
   - name: stg_ntd__track_and_roadway_guideway_age_distribution
   - name: stg_ntd__vehicles_age_distribution
   - name: stg_ntd__vehicles_type_count_by_agency
-  - name: stg_ntd__2023_agency_information
-  - name: stg_ntd__2023_contractual_relationships

warehouse/models/staging/ntd_annual_reporting/stg_ntd__2022_agency_information.sql

@@ -0,0 +1,63 @@
+WITH external_agency_information AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__annual_database_agency_information') }}
+),
+
+get_latest_extract AS(
+    SELECT *
+    FROM external_agency_information
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd__2022_agency_information AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT
+    SAFE_CAST(number_of_state_counties AS NUMERIC) AS number_of_state_counties,
+    {{ trim_make_empty_string_null('tam_tier') }} AS tam_tier,
+    SAFE_CAST(personal_vehicles AS NUMERIC) AS personal_vehicles,
+    {{ trim_make_empty_string_null('uza_name') }} AS uza_name,
+    {{ trim_make_empty_string_null('tribal_area_name') }} AS tribal_area_name,
+    SAFE_CAST(service_area_sq_miles AS NUMERIC) AS service_area_sq_miles,
+    SAFE_CAST(voms_do AS NUMERIC) AS voms_do,
+    {{ trim_make_empty_string_null('url') }} AS url,
+    SAFE_CAST(region AS INTEGER) AS region,
+    SAFE_CAST(state_admin_funds_expended AS NUMERIC) AS state_admin_funds_expended,
+    SAFE_CAST(zip_code_ext AS NUMERIC) AS zip_code_ext,
+    SAFE_CAST(zip_code AS NUMERIC) AS zip_code,
+    {{ trim_make_empty_string_null('ueid') }} AS ueid,
+    {{ trim_make_empty_string_null('address_line_2') }} AS address_line_2,
+    SAFE_CAST(number_of_counties_with_service AS NUMERIC) AS number_of_counties_with_service,
+    {{ trim_make_empty_string_null('reporter_acronym') }} AS reporter_acronym,
+    SAFE_CAST(original_due_date AS INTEGER) AS original_due_date,
+    SAFE_CAST(sq_miles AS NUMERIC) AS sq_miles,
+    {{ trim_make_empty_string_null('address_line_1') }} AS address_line_1,
+    {{ trim_make_empty_string_null('p_o__box') }} AS p_o__box,
+    SAFE_CAST(fy_end_date AS INTEGER) AS fy_end_date,
+    SAFE_CAST(service_area_pop AS NUMERIC) AS service_area_pop,
+    {{ trim_make_empty_string_null('state') }} AS state,
+    {{ trim_make_empty_string_null('subrecipient_type') }} AS subrecipient_type,
+    SAFE_CAST(population AS NUMERIC) AS population,
+    {{ trim_make_empty_string_null('reporting_module') }} AS reporting_module,
+    SAFE_CAST(volunteer_drivers AS NUMERIC) AS volunteer_drivers,
+    {{ trim_make_empty_string_null('doing_business_as') }} AS doing_business_as,
+    {{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
+    {{ trim_make_empty_string_null('legacy_ntd_id') }} AS legacy_ntd_id,
+    SAFE_CAST(total_voms AS INTEGER) AS total_voms,
+    SAFE_CAST(primary_uza_uace_code AS NUMERIC) AS primary_uza_uace_code,
+    {{ trim_make_empty_string_null('reported_by_name') }} AS reported_by_name,
+    SAFE_CAST(fta_recipient_id AS NUMERIC) AS fta_recipient_id,
+    {{ trim_make_empty_string_null('city') }} AS city,
+    SAFE_CAST(voms_pt AS NUMERIC) AS voms_pt,
+    {{ trim_make_empty_string_null('organization_type') }} AS organization_type,
+    {{ trim_make_empty_string_null('agency_name') }} AS agency_name,
+    {{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
+    {{ trim_make_empty_string_null('reported_by_ntd_id') }} AS reported_by_ntd_id,
+    SAFE_CAST(density AS NUMERIC) AS density,
+    {{ trim_make_empty_string_null('state_parent_ntd_id') }} AS state_parent_ntd_id,
+    dt,
+    execution_ts
+FROM stg_ntd__2022_agency_information