Merge pull request #59 from Urban-Analytics-Technology-Platform/trave…

…l_time_pipeline_errors

travel time pipeline fixes

scripts/2_match_households_and_individuals.py

@@ -929,7 +929,7 @@ def get_interim_path(
         df1_id="hid",
         df2_id="HouseholdID",
         matches_hh=matches_hh_level_sample,
-        show_progress=False,
+        show_progress=True,
     )
 
     # Add matches_ind values to spc_edited using map

scripts/3.1_assign_primary_feasible_zones.py

@@ -96,6 +96,11 @@ def main(config_file):
         # Create a new travel time matrix based on distances between zones
         travel_times = zones_to_time_matrix(zones=boundaries, id_col=config.zone_id)
         logger.info("Travel time estimates created")
+        # save travel_times as parquet
+
+        travel_times.to_parquet(
+            acbm.root_path / "data/interim/assigning/travel_time_estimates.parquet"
+        )
 
     # --- Intrazonal trip times
     #
@@ -112,6 +117,10 @@ def main(config_file):
 
     intrazone_times = intrazone_time(zones=boundaries, key_column=config.zone_id)
 
+    # save intrazone_times to pickle
+    with open(acbm.root_path / "data/interim/assigning/intrazone_times.pkl", "wb") as f:
+        pkl.dump(intrazone_times, f)
+
     logger.info("Intrazonal travel time estimates created")
 
     # replace intrazonal travel times with estimates from intrazone_times

scripts/3.2.1_assign_primary_zone_edu.py

@@ -33,16 +33,13 @@ def main(config_file):
     )
 
     # --- boundaries
-    logger.info("Loading boundaries")
-
-    where_clause = "MSOA21NM LIKE '%Leeds%'"
+    logger.info("Loading study area boundaries")
 
     boundaries = gpd.read_file(
-        acbm.root_path / "data/external/boundaries/oa_england.geojson",
-        where=where_clause,
+        acbm.root_path / "data/external/boundaries/study_area_zones.geojson"
     )
 
-    boundaries = boundaries.to_crs(epsg=4326)
+    logger.info("Study area boundaries loaded")
 
     # --- osm POI data
     logger.info("Loading OSM POI data")
@@ -102,8 +99,8 @@ def main(config_file):
     # --- travel time estimates
     logger.info("Loading travel time estimates")
 
-    travel_time_estimates = pd.read_pickle(
-        acbm.root_path / "data/interim/assigning/travel_time_estimates.pkl"
+    travel_time_estimates = pd.read_parquet(
+        acbm.root_path / "data/interim/assigning/travel_time_estimates.parquet"
     )
 
     #### ASSIGN TO ZONE FROM FEASIBLE ZONES ####

scripts/3.2.3_assign_secondary_zone.py

@@ -305,9 +305,16 @@ def merge_columns_from_other(df: pd.DataFrame, other: pd.DataFrame) -> pd.DataFr
     # --- Load travel time estimates
     logger.info("Analysis (matrices): Step 1 - Loading travel time data")
 
-    travel_times = pd.read_parquet(
-        acbm.root_path / "data/external/travel_times/oa/travel_time_matrix.parquet"
-    )
+    # load in the travel times (path differs for estimated ones)
+    # TODO: improve / save in same directory / add paths to config
+    if config.parameters.travel_times:
+        travel_times = pd.read_parquet(
+            acbm.root_path / "data/external/travel_times/oa/travel_time_matrix.parquet"
+        )
+    else:
+        travel_times = pd.read_parquet(
+            acbm.root_path / "data/interim/assigning/travel_time_estimates.parquet"
+        )
 
     # Edit modes
     logger.info("Analysis (matrices): Step 2 - Editing modes")
@@ -357,7 +364,9 @@ def merge_columns_from_other(df: pd.DataFrame, other: pd.DataFrame) -> pd.DataFr
 
     # Merge to get floor_area for origin
     merged_df = travel_times.merge(
-        activities_per_zone, left_on="OA21CD_to", right_on=config.zone_id
+        activities_per_zone,
+        left_on=config.destination_zone_id(zone_id),
+        right_on=config.zone_id,
     )
 
     # Calculate the visit_probability: it is a funciton of floor_area and travel time

src/acbm/assigning/feasible_zones_primary.py

@@ -50,7 +50,7 @@
 travel_times_schema = DataFrameSchema(
     {
         "mode": Column(str),
-        "weekday": Column(pa.Float, Check.isin([0, 1]), nullable=True),
+        # "weekday": Column(pa.Float, Check.isin([0, 1]), nullable=True), # Does not exist if we make our own estimate
         # "time_of_day": Column(str, nullable=True),
         "time": Column(float),
     },
@@ -161,9 +161,16 @@ def get_possible_zones(
 
         # if the mode is public transport, we need to filter the travel_times data based on time_of_day and weekday/weekend
         # this only applies if we have the time_of_day column in the travel_times dataframe (not the case if we've estimated
-        # travel times)
+        # travel times).
 
-        if mode == "pt" and "time_of_day" in travel_times.columns:
+        # if "weekday" does not exist, we skip. Our travel_time_estimates (from zones_to_time_matrix())
+        # don't have weekday information
+
+        if (
+            mode == "pt"
+            and "time_of_day" in travel_times.columns
+            and "weekday" in travel_times.columns
+        ):
             for time_of_day in list_of_times_of_day:
                 print(f"Processing time of day: {time_of_day} | mode: {mode}")
                 for day_type in day_types:

src/acbm/assigning/select_zone_primary.py

@@ -256,6 +256,15 @@ def _get_zones_using_time_estimate(
         for k, v in estimated_times.items()
         if k[0] == from_zone and k[1] in to_zones_set
     }
+
+    # Check if the filtered dictionary is empty
+    if not filtered_dict:
+        # Handle the case where there are no travel time estimates for the target zone
+        print(
+            f"No travel time estimates found for from_zone: {from_zone} to any of the to_zones: {to_zones}"
+        )
+        return None
+
     # get to_zone where time_average is closest to "time"
     if mode is not None:
         closest_to_zone = min(

src/acbm/config.py

@@ -51,6 +51,10 @@ def origin_zone_id(cls, zone_id: str) -> str:
     def destination_zone_id(cls, zone_id: str) -> str:
         return zone_id + "_to"
 
+    @property
+    def boundary_geography(self) -> str:
+        return self.parameters.boundary_geography
+
     # TODO: consider moving to method in config
     def init_rng(self):
         try: