Ntd: typecast annual_reporting and security_and_security staging tabl…

…es (#3601)

* ntd: add typecasting for strings and numerics in annual and safety tables

* ntd: add typecasting for strings and numerics in safety tables

* fix error typecasting to wrong data type

warehouse/models/staging/ntd_annual_reporting/stg_ntd__2023_agency_information.sql

@@ -16,49 +16,49 @@ stg_ntd__2023_agency_information AS (
 )
 
 SELECT
-    number_of_state_counties,
-    tam_tier,
-    personal_vehicles,
-    uza_name,
-    tribal_area_name,
-    service_area_sq_miles,
-    voms_do,
-    url,
-    region,
-    state_admin_funds_expended,
-    zip_code_ext,
-    zip_code,
-    ueid,
-    address_line_2,
-    number_of_counties_with_service,
-    reporter_acronym,
-    original_due_date,
-    sq_miles,
-    address_line_1,
-    p_o__box,
-    division_department,
-    fy_end_date,
-    service_area_pop,
-    state,
-    subrecipient_type,
-    primary_uza_uace_code,
-    reported_by_name,
-    population,
-    reporting_module,
-    volunteer_drivers,
-    doing_business_as,
-    reporter_type,
-    legacy_ntd_id,
-    total_voms,
-    fta_recipient_id,
-    city,
-    voms_pt,
-    organization_type,
-    agency_name,
-    ntd_id,
-    reported_by_ntd_id,
-    density,
-    state_parent_ntd_id,
+    SAFE_CAST(number_of_state_counties AS NUMERIC) AS number_of_state_counties,
+    {{ trim_make_empty_string_null('tam_tier') }} AS tam_tier,
+    SAFE_CAST(personal_vehicles AS NUMERIC) AS personal_vehicles,
+    {{ trim_make_empty_string_null('uza_name') }} AS uza_name,
+    {{ trim_make_empty_string_null('tribal_area_name') }} AS tribal_area_name,
+    SAFE_CAST(service_area_sq_miles AS NUMERIC) AS service_area_sq_miles,
+    SAFE_CAST(voms_do AS NUMERIC) AS voms_do,
+    {{ trim_make_empty_string_null('url') }} AS url,
+    SAFE_CAST(region AS INTEGER) AS region,
+    SAFE_CAST(state_admin_funds_expended AS NUMERIC) AS state_admin_funds_expended,
+    SAFE_CAST(zip_code_ext AS NUMERIC) AS zip_code_ext,
+    SAFE_CAST(zip_code AS NUMERIC) AS zip_code,
+    {{ trim_make_empty_string_null('ueid') }} AS ueid,
+    {{ trim_make_empty_string_null('address_line_2') }} AS address_line_2,
+    SAFE_CAST(number_of_counties_with_service AS NUMERIC) AS number_of_counties_with_service,
+    {{ trim_make_empty_string_null('reporter_acronym') }} AS reporter_acronym,
+    SAFE_CAST(original_due_date AS INTEGER) AS original_due_date,
+    SAFE_CAST(sq_miles AS NUMERIC) AS sq_miles,
+    {{ trim_make_empty_string_null('address_line_1') }} AS address_line_1,
+    {{ trim_make_empty_string_null('p_o__box') }} AS p_o__box,
+    {{ trim_make_empty_string_null('division_department') }} AS division_department,
+    SAFE_CAST(fy_end_date AS INTEGER) AS fy_end_date,
+    SAFE_CAST(service_area_pop AS NUMERIC) AS service_area_pop,
+    {{ trim_make_empty_string_null('state') }} AS state,
+    {{ trim_make_empty_string_null('subrecipient_type') }} AS subrecipient_type,
+    SAFE_CAST(primary_uza_uace_code AS NUMERIC) AS primary_uza_uace_code,
+    {{ trim_make_empty_string_null('reported_by_name') }} AS reported_by_name,
+    SAFE_CAST(population AS NUMERIC) AS population,
+    {{ trim_make_empty_string_null('reporting_module') }} AS reporting_module,
+    SAFE_CAST(volunteer_drivers AS NUMERIC) AS volunteer_drivers,
+    {{ trim_make_empty_string_null('doing_business_as') }} AS doing_business_as,
+    {{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
+    {{ trim_make_empty_string_null('legacy_ntd_id') }} AS legacy_ntd_id,
+    SAFE_CAST(total_voms AS INTEGER) AS total_voms,
+    SAFE_CAST(fta_recipient_id AS NUMERIC) AS fta_recipient_id,
+    {{ trim_make_empty_string_null('city') }} AS city,
+    SAFE_CAST(voms_pt AS NUMERIC) AS voms_pt,
+    {{ trim_make_empty_string_null('organization_type') }} AS organization_type,
+    {{ trim_make_empty_string_null('agency_name') }} AS agency_name,
+    {{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
+    {{ trim_make_empty_string_null('reported_by_ntd_id') }} AS reported_by_ntd_id,
+    SAFE_CAST(density AS NUMERIC) AS density,
+    {{ trim_make_empty_string_null('state_parent_ntd_id') }} AS state_parent_ntd_id,
     dt,
     execution_ts
 FROM stg_ntd__2023_agency_information

warehouse/models/staging/ntd_annual_reporting/stg_ntd__2023_contractual_relationships.sql

@@ -16,33 +16,33 @@ stg_ntd__2023_contractual_relationships AS (
 )
 
 SELECT
-    other_reconciling_item_expenses_incurred_by_the_buyer,
-    total_modal_expenses,
-    contract_capital_leasing_expenses,
-    direct_payment_agency_subsidy,
-    months_seller_operated_in_fy,
-    primary_feature,
-    voms_under_contract,
-    service_captured,
-    fares_retained_by,
-    other_party,
-    other_public_assets_provided,
-    buyer_supplies_vehicles_to_seller,
-    contractee_ntd_id,
-    pt_fare_revenues_passenger_fees,
-    agency_name,
-    tos,
-    type_of_contract,
-    reporter_contractual_position,
-    other_operating_expenses_incurred_by_the_buyer,
-    passenger_out_of_pocket_expenses,
-    buyer_provides_maintenance_facility_to_seller,
-    contractee_operator_name,
-    mode,
-    reporting_module,
-    reporter_type,
-    other_public_assets_provided_desc,
-    ntd_id,
+    SAFE_CAST(other_reconciling_item_expenses_incurred_by_the_buyer AS INTEGER) AS other_reconciling_item_expenses_incurred_by_the_buyer,
+    SAFE_CAST(total_modal_expenses AS INTEGER) AS total_modal_expenses,
+    SAFE_CAST(contract_capital_leasing_expenses AS INTEGER) AS contract_capital_leasing_expenses,
+    SAFE_CAST(direct_payment_agency_subsidy AS INTEGER) AS direct_payment_agency_subsidy,
+    SAFE_CAST(months_seller_operated_in_fy AS INTEGER) AS months_seller_operated_in_fy,
+    {{ trim_make_empty_string_null('primary_feature') }} AS primary_feature,
+    SAFE_CAST(voms_under_contract AS INTEGER) AS voms_under_contract,
+    {{ trim_make_empty_string_null('service_captured') }} AS service_captured,
+    {{ trim_make_empty_string_null('fares_retained_by') }} AS fares_retained_by,
+    {{ trim_make_empty_string_null('other_party') }} AS other_party,
+    SAFE_CAST(other_public_assets_provided AS BOOLEAN) AS other_public_assets_provided,
+    SAFE_CAST(buyer_supplies_vehicles_to_seller AS BOOLEAN) AS buyer_supplies_vehicles_to_seller,
+    {{ trim_make_empty_string_null('contractee_ntd_id') }} AS contractee_ntd_id,
+    SAFE_CAST(pt_fare_revenues_passenger_fees AS INTEGER) AS pt_fare_revenues_passenger_fees,
+    {{ trim_make_empty_string_null('agency_name') }} AS agency_name,
+    {{ trim_make_empty_string_null('tos') }} AS tos,
+    {{ trim_make_empty_string_null('type_of_contract') }} AS type_of_contract,
+    {{ trim_make_empty_string_null('reporter_contractual_position') }} AS reporter_contractual_position,
+    SAFE_CAST(other_operating_expenses_incurred_by_the_buyer AS INTEGER) AS other_operating_expenses_incurred_by_the_buyer,
+    SAFE_CAST(passenger_out_of_pocket_expenses AS INTEGER) AS passenger_out_of_pocket_expenses,
+    SAFE_CAST(buyer_provides_maintenance_facility_to_seller AS BOOLEAN) AS buyer_provides_maintenance_facility_to_seller,
+    {{ trim_make_empty_string_null('contractee_operator_name') }} AS contractee_operator_name,
+    {{ trim_make_empty_string_null('mode') }} AS mode,
+    {{ trim_make_empty_string_null('reporting_module') }} AS reporting_module,
+    {{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
+    {{ trim_make_empty_string_null('other_public_assets_provided_desc') }} AS other_public_assets_provided_desc,
+    SAFE_CAST(ntd_id AS INTEGER) AS ntd_id,
     dt,
     execution_ts
 FROM stg_ntd__2023_contractual_relationships

warehouse/models/staging/ntd_annual_reporting/stg_ntd__breakdowns.sql

@@ -16,35 +16,35 @@ stg_ntd__breakdowns AS (
 )
 
 SELECT
-    agency,
-    agency_voms,
-    city,
-    major_mechanical_failures,
-    major_mechanical_failures_1,
-    mode,
-    mode_name,
-    mode_voms,
-    ntd_id,
-    organization_type,
-    other_mechanical_failures,
-    other_mechanical_failures_1,
-    primary_uza_population,
-    report_year,
-    reporter_type,
-    state,
-    total_mechanical_failures,
-    total_mechanical_failures_1,
-    train_miles,
-    train_miles_questionable,
-    train_revenue_miles,
-    train_revenue_miles_1,
-    type_of_service,
-    uace_code,
-    uza_name,
-    vehicle_passenger_car_miles,
-    vehicle_passenger_car_miles_1,
-    vehicle_passenger_car_miles_2,
-    vehicle_passenger_car_revenue,
+    {{ trim_make_empty_string_null('agency') }} AS agency,
+    SAFE_CAST(agency_voms AS NUMERIC) AS agency_voms,
+    {{ trim_make_empty_string_null('city') }} AS city,
+    SAFE_CAST(major_mechanical_failures AS NUMERIC) AS major_mechanical_failures,
+    {{ trim_make_empty_string_null('major_mechanical_failures_1') }} AS major_mechanical_failures_1,
+    {{ trim_make_empty_string_null('mode') }} AS mode,
+    {{ trim_make_empty_string_null('mode_name') }} AS mode_name,
+    SAFE_CAST(mode_voms AS NUMERIC) AS mode_voms,
+    {{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
+    {{ trim_make_empty_string_null('organization_type') }} AS organization_type,
+    SAFE_CAST(other_mechanical_failures AS NUMERIC) AS other_mechanical_failures,
+    {{ trim_make_empty_string_null('other_mechanical_failures_1') }} AS other_mechanical_failures_1,
+    SAFE_CAST(primary_uza_population AS NUMERIC) AS primary_uza_population,
+    {{ trim_make_empty_string_null('report_year') }} AS report_year,
+    {{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
+    {{ trim_make_empty_string_null('state') }} AS state,
+    SAFE_CAST(total_mechanical_failures AS NUMERIC) AS total_mechanical_failures,
+    {{ trim_make_empty_string_null('total_mechanical_failures_1') }} AS total_mechanical_failures_1,
+    SAFE_CAST(train_miles AS NUMERIC) AS train_miles,
+    {{ trim_make_empty_string_null('train_miles_questionable') }} AS train_miles_questionable,
+    SAFE_CAST(train_revenue_miles AS NUMERIC) AS train_revenue_miles,
+    {{ trim_make_empty_string_null('train_revenue_miles_1') }} AS train_revenue_miles_1,
+    {{ trim_make_empty_string_null('type_of_service') }} AS type_of_service,
+    {{ trim_make_empty_string_null('uace_code') }} AS uace_code,
+    {{ trim_make_empty_string_null('uza_name') }} AS uza_name,
+    SAFE_CAST(vehicle_passenger_car_miles AS NUMERIC) AS vehicle_passenger_car_miles,
+    {{ trim_make_empty_string_null('vehicle_passenger_car_miles_1') }} AS vehicle_passenger_car_miles_1,
+    {{ trim_make_empty_string_null('vehicle_passenger_car_miles_2') }} AS vehicle_passenger_car_miles_2,
+    SAFE_CAST(vehicle_passenger_car_revenue AS NUMERIC) AS vehicle_passenger_car_revenue,
     dt,
     execution_ts
 FROM stg_ntd__breakdowns

warehouse/models/staging/ntd_annual_reporting/stg_ntd__breakdowns_by_agency.sql

@@ -16,30 +16,30 @@ stg_ntd__breakdowns_by_agency AS (
 )
 
 SELECT
-    count_major_mechanical_failures_questionable,
-    count_other_mechanical_failures_questionable,
-    count_total_mechanical_failures_questionable,
-    count_train_miles_questionable,
-    count_train_revenue_miles_questionable,
-    count_vehicle_passenger_car_miles_questionable,
-    max_agency,
-    max_agency_voms,
-    max_city,
-    max_organization_type,
-    max_primary_uza_population,
-    max_reporter_type,
-    max_state,
-    max_uace_code,
-    max_uza_name,
-    ntd_id,
-    report_year,
-    sum_major_mechanical_failures,
-    sum_other_mechanical_failures,
-    sum_total_mechanical_failures,
-    sum_train_miles,
-    sum_train_revenue_miles,
-    sum_vehicle_passenger_car_miles,
-    sum_vehicle_passenger_car_revenue,
+    SAFE_CAST(count_major_mechanical_failures_questionable AS NUMERIC) AS count_major_mechanical_failures_questionable,
+    SAFE_CAST(count_other_mechanical_failures_questionable AS NUMERIC) AS count_other_mechanical_failures_questionable,
+    SAFE_CAST(count_total_mechanical_failures_questionable AS NUMERIC) AS count_total_mechanical_failures_questionable,
+    SAFE_CAST(count_train_miles_questionable AS NUMERIC) AS count_train_miles_questionable,
+    SAFE_CAST(count_train_revenue_miles_questionable AS NUMERIC) AS count_train_revenue_miles_questionable,
+    SAFE_CAST(count_vehicle_passenger_car_miles_questionable AS NUMERIC) AS count_vehicle_passenger_car_miles_questionable,
+    {{ trim_make_empty_string_null('max_agency') }} AS max_agency,
+    SAFE_CAST(max_agency_voms AS NUMERIC) AS max_agency_voms,
+    {{ trim_make_empty_string_null('max_city') }} AS max_city,
+    {{ trim_make_empty_string_null('max_organization_type') }} AS max_organization_type,
+    SAFE_CAST(max_primary_uza_population AS NUMERIC) AS max_primary_uza_population,
+    {{ trim_make_empty_string_null('max_reporter_type') }} AS max_reporter_type,
+    {{ trim_make_empty_string_null('max_state') }} AS max_state,
+    {{ trim_make_empty_string_null('max_uace_code') }} AS max_uace_code,
+    {{ trim_make_empty_string_null('max_uza_name') }} AS max_uza_name,
+    {{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
+    {{ trim_make_empty_string_null('report_year') }} AS report_year,
+    SAFE_CAST(sum_major_mechanical_failures AS NUMERIC) AS sum_major_mechanical_failures,
+    SAFE_CAST(sum_other_mechanical_failures AS NUMERIC) AS sum_other_mechanical_failures,
+    SAFE_CAST(sum_total_mechanical_failures AS NUMERIC) AS sum_total_mechanical_failures,
+    SAFE_CAST(sum_train_miles AS NUMERIC) AS sum_train_miles,
+    SAFE_CAST(sum_train_revenue_miles AS NUMERIC) AS sum_train_revenue_miles,
+    SAFE_CAST(sum_vehicle_passenger_car_miles AS NUMERIC) AS sum_vehicle_passenger_car_miles,
+    SAFE_CAST(sum_vehicle_passenger_car_revenue AS NUMERIC) AS sum_vehicle_passenger_car_revenue,
     dt,
     execution_ts
 FROM stg_ntd__breakdowns_by_agency

warehouse/models/staging/ntd_annual_reporting/stg_ntd__capital_expenses_by_capital_use.sql

@@ -16,44 +16,44 @@ stg_ntd__capital_expenses_by_capital_use AS (
 )
 
 SELECT
-    administrative_buildings,
-    administrative_buildings_1,
-    agency,
-    agency_voms,
-    city,
-    communication_information,
-    communication_information_1,
-    fare_collection_equipment,
-    fare_collection_equipment_1,
-    form_type,
-    guideway,
-    guideway_questionable,
-    maintenance_buildings,
-    maintenance_buildings_1,
-    mode_name,
-    mode_voms,
-    modecd,
-    ntd_id,
-    organization_type,
-    other,
-    other_questionable,
-    other_vehicles,
-    other_vehicles_questionable,
-    passenger_vehicles,
-    passenger_vehicles_1,
-    primary_uza_population,
-    reduced_reporter,
-    reduced_reporter_questionable,
-    report_year,
-    reporter_type,
-    state,
-    stations,
-    stations_questionable,
-    total,
-    total_questionable,
-    typeofservicecd,
-    uace_code,
-    uza_name,
+    SAFE_CAST(administrative_buildings AS NUMERIC) AS administrative_buildings,
+    {{ trim_make_empty_string_null('administrative_buildings_1') }} AS administrative_buildings_1,
+    {{ trim_make_empty_string_null('agency') }} AS agency,
+    SAFE_CAST(agency_voms AS NUMERIC) AS agency_voms,
+    {{ trim_make_empty_string_null('city') }} AS city,
+    SAFE_CAST(communication_information AS NUMERIC) AS communication_information,
+    {{ trim_make_empty_string_null('communication_information_1') }} AS communication_information_1,
+    SAFE_CAST(fare_collection_equipment AS NUMERIC) AS fare_collection_equipment,
+    {{ trim_make_empty_string_null('fare_collection_equipment_1') }} AS fare_collection_equipment_1,
+    {{ trim_make_empty_string_null('form_type') }} AS form_type,
+    SAFE_CAST(guideway AS NUMERIC) AS guideway,
+    SAFE_CAST(guideway_questionable AS NUMERIC) AS guideway_questionable,
+    SAFE_CAST(maintenance_buildings AS NUMERIC) AS maintenance_buildings,
+    {{ trim_make_empty_string_null('maintenance_buildings_1') }} AS maintenance_buildings_1,
+    {{ trim_make_empty_string_null('mode_name') }} AS mode_name,
+    SAFE_CAST(mode_voms AS NUMERIC) AS mode_voms,
+    {{ trim_make_empty_string_null('modecd') }} AS modecd,
+    {{ trim_make_empty_string_null('ntd_id') }} AS ntd_id,
+    {{ trim_make_empty_string_null('organization_type') }} AS organization_type,
+    SAFE_CAST(other AS NUMERIC) AS other,
+    {{ trim_make_empty_string_null('other_questionable') }} AS other_questionable,
+    SAFE_CAST(other_vehicles AS NUMERIC) AS other_vehicles,
+    {{ trim_make_empty_string_null('other_vehicles_questionable') }} AS other_vehicles_questionable,
+    SAFE_CAST(passenger_vehicles AS NUMERIC) AS passenger_vehicles,
+    {{ trim_make_empty_string_null('passenger_vehicles_1') }} AS passenger_vehicles_1,
+    SAFE_CAST(primary_uza_population AS NUMERIC) AS primary_uza_population,
+    SAFE_CAST(reduced_reporter AS NUMERIC) AS reduced_reporter,
+    {{ trim_make_empty_string_null('reduced_reporter_questionable') }} AS reduced_reporter_questionable,
+    {{ trim_make_empty_string_null('report_year') }} AS report_year,
+    {{ trim_make_empty_string_null('reporter_type') }} AS reporter_type,
+    {{ trim_make_empty_string_null('state') }} AS state,
+    SAFE_CAST(stations AS NUMERIC) AS stations,
+    {{ trim_make_empty_string_null('stations_questionable') }} AS stations_questionable,
+    SAFE_CAST(total AS NUMERIC) AS total,
+    {{ trim_make_empty_string_null('total_questionable') }} AS total_questionable,
+    {{ trim_make_empty_string_null('typeofservicecd') }} AS typeofservicecd,
+    {{ trim_make_empty_string_null('uace_code') }} AS uace_code,
+    {{ trim_make_empty_string_null('uza_name') }} AS uza_name,
     dt,
     execution_ts
 FROM stg_ntd__capital_expenses_by_capital_use