diff --git a/airflow/.gitignore b/airflow/.gitignore index c75dc7c3df..408cd73648 100644 --- a/airflow/.gitignore +++ b/airflow/.gitignore @@ -1 +1,2 @@ logs/* +airflow.sh diff --git a/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_organizations.yml b/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_organizations.yml index c8a591447b..c2e42527dd 100644 --- a/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_organizations.yml +++ b/airflow/dags/create_external_tables/airtable/external_airtable_california_transit_organizations.yml @@ -254,3 +254,6 @@ schema_fields: - name: raw_ntd_id type: STRING mode: NULLABLE + - name: ntd_id_2022 + type: STRING + mode: NULLABLE diff --git a/warehouse/models/intermediate/transit_database/dimensions/int_transit_database__organizations_dim.sql b/warehouse/models/intermediate/transit_database/dimensions/int_transit_database__organizations_dim.sql index 02e09c7f99..d546c5eb55 100644 --- a/warehouse/models/intermediate/transit_database/dimensions/int_transit_database__organizations_dim.sql +++ b/warehouse/models/intermediate/transit_database/dimensions/int_transit_database__organizations_dim.sql @@ -36,6 +36,7 @@ int_transit_database__organizations_dim AS ( hq_county_geography, is_public_entity, raw_ntd_id, + ntd_id_2022, public_currently_operating, public_currently_operating_fixed_route, _is_current, diff --git a/warehouse/models/mart/transit_database/_mart_transit_database.yml b/warehouse/models/mart/transit_database/_mart_transit_database.yml index 8e2c80afc5..20feddbee4 100644 --- a/warehouse/models/mart/transit_database/_mart_transit_database.yml +++ b/warehouse/models/mart/transit_database/_mart_transit_database.yml @@ -86,7 +86,7 @@ models: renamed to `_deprecated__ntd_agency_to_organizations`). - name: ntd_id_2022 description: | - Just pulls out the 5 digit component of the raw_ntd_id + Pulled from airtable, the 5 digit component of raw_ntd_id - name: name description: Organization name - name: organization_type diff --git a/warehouse/models/mart/transit_database/dim_organizations.sql b/warehouse/models/mart/transit_database/dim_organizations.sql index 353802be84..72e85df821 100644 --- a/warehouse/models/mart/transit_database/dim_organizations.sql +++ b/warehouse/models/mart/transit_database/dim_organizations.sql @@ -32,9 +32,7 @@ dim_organizations AS ( WHEN _valid_from >= '2023-05-23' THEN raw_ntd_id ELSE ntd_to_org.ntd_id END AS ntd_id, - IF(LENGTH(ntd_id) >= 10, - SUBSTR(ntd_id, -5), - ntd_id) AS ntd_id_2022, + ntd_id_2022, public_currently_operating, public_currently_operating_fixed_route, _is_current, diff --git a/warehouse/models/staging/transit_database/stg_transit_database__organizations.sql b/warehouse/models/staging/transit_database/stg_transit_database__organizations.sql index e894977cf2..7cd883f8fc 100644 --- a/warehouse/models/staging/transit_database/stg_transit_database__organizations.sql +++ b/warehouse/models/staging/transit_database/stg_transit_database__organizations.sql @@ -41,6 +41,7 @@ stg_transit_database__organizations AS ( hq_county_geography, is_public_entity = "Yes" AS is_public_entity, raw_ntd_id, + ntd_id_2022, public_currently_operating = "Yes" AS public_currently_operating, public_currently_operating_fixed_route = "Yes" AS public_currently_operating_fixed_route, FROM once_daily_organizations