Skip to content

Commit 92dc961

Browse files
tiffanychu90tiffanychu90
authored
Merge pull request #1080 from cal-itp/apr-open-data
Apr open data
2 parents 5a60971 + e3b1567 commit 92dc961

31 files changed

+147
-66
lines changed

‎_shared_utils/shared_utils/gtfs_analytics_data.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ digest_tables:
6464

6565
stop_segments:
6666
dir: ${gcs_paths.SEGMENT_GCS}
67-
stage1: ${speeds_tables.usable_vp_file}
67+
stage1: ${speeds_tables.usable_vp}
6868
stage2: "nearest/nearest_vp_shape_segments"
6969
stage3: "stop_arrivals"
7070
stage4: "speeds_stop_segments"
@@ -77,7 +77,7 @@ stop_segments:
7777

7878
rt_stop_times:
7979
dir: ${gcs_paths.SEGMENT_GCS}
80-
stage1: ${speeds_tables.usable_vp_file}
80+
stage1: ${speeds_tables.usable_vp}
8181
stage2: "nearest/nearest_vp_rt_stop_times"
8282
stage3: "rt_stop_times/stop_arrivals"
8383
stage4: "rt_stop_times/speeds"
@@ -92,7 +92,7 @@ rt_stop_times:
9292

9393
road_segments:
9494
dir: ${gcs_paths.SEGMENT_GCS}
95-
stage1: ${speeds_tables.usable_vp_file}
95+
stage1: ${speeds_tables.usable_vp}
9696
stage2: "nearest/nearest_vp_roads"
9797
stage3: "road_segments/stop_arrivals"
9898
stage4: "speeds_road_segments"

‎_shared_utils/shared_utils/rt_dates.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"jan2024": "2024-01-17",
5353
"feb2024": "2024-02-14",
5454
"mar2024": "2024-03-13",
55+
"apr2024": "2024-04-17",
5556
}
5657

5758
y2023_dates = [

‎gtfs_funnel/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ preprocess:
1616
python cleanup.py
1717
python vp_condenser.py
1818
make route_typologies_data
19-
python operator_scheduled_stats.py
19+
python operator_scheduled_stats.py
2020

2121

2222
route_typologies_data:
@@ -36,4 +36,4 @@ monthly_scheduled_data:
3636

3737

3838
funnel_gtfs_data:
39-
make download_gtfs_data && make preprocess && make timeseries_preprocessing
39+
make download_gtfs_data && make preprocess && make timeseries_preprocessing

‎gtfs_funnel/cleanup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
for analysis_date in analysis_date_list:
1414

15-
INPUT_FILE = GTFS_DATA_DICT.speed_tables.usable_vp
15+
INPUT_FILE = GTFS_DATA_DICT.speeds_tables.usable_vp
1616

1717
helpers.if_exists_then_delete(
1818
f"{SEGMENT_GCS}{INPUT_FILE}_{analysis_date}_stage"

‎gtfs_funnel/logs/download_data.log

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -320,3 +320,19 @@
320320
2024-03-14 11:44:27.599 | INFO | __main__:download_one_day:56 - execution time: 0:01:22.625555
321321
2024-03-19 16:50:51.742 | INFO | __main__:download_one_year:35 - execution time: 0:00:43.062868
322322
2024-04-02 09:03:24.949 | INFO | __main__:download_one_year:35 - execution time: 0:00:14.017626
323+
2024-04-18 10:27:32.761 | INFO | __main__:download_one_day:45 - Analysis date: 2024-04-17
324+
2024-04-18 10:27:35.561 | INFO | __main__:download_one_day:52 - # operators to run: 198
325+
2024-04-18 10:27:35.562 | INFO | __main__:download_one_day:56 - *********** Download trips data ***********
326+
2024-04-18 10:28:15.325 | INFO | __main__:download_one_day:86 - execution time: 0:00:42.561678
327+
2024-04-18 10:28:35.712 | INFO | __main__:download_one_day:22 - Analysis date: 2024-04-17
328+
2024-04-18 10:28:38.053 | INFO | __main__:download_one_day:29 - # operators to run: 198
329+
2024-04-18 10:28:38.054 | INFO | __main__:download_one_day:33 - *********** Download stops data ***********
330+
2024-04-18 10:28:51.192 | INFO | __main__:download_one_day:64 - execution time: 0:00:15.478316
331+
2024-04-18 10:29:14.197 | INFO | __main__:download_one_day:22 - Analysis date: 2024-04-17
332+
2024-04-18 10:29:16.683 | INFO | __main__:download_one_day:29 - # operators to run: 198
333+
2024-04-18 10:29:16.683 | INFO | __main__:download_one_day:33 - *********** Download routelines data ***********
334+
2024-04-18 10:32:02.705 | INFO | __main__:download_one_day:63 - execution time: 0:02:48.507170
335+
2024-04-18 10:32:26.078 | INFO | __main__:download_one_day:21 - Analysis date: 2024-04-17
336+
2024-04-18 10:32:27.682 | INFO | __main__:download_one_day:29 - # operators to run: 171
337+
2024-04-18 10:32:27.683 | INFO | __main__:download_one_day:33 - *********** Download st data ***********
338+
2024-04-18 10:34:57.448 | INFO | __main__:download_one_day:56 - execution time: 0:02:31.368995

‎gtfs_funnel/logs/download_vp_v2.log

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,3 +207,14 @@
207207
2024-03-14 11:58:41.151 | INFO | __main__:<module>:110 - export concatenated vp: 0:03:05.913001
208208
2024-03-14 12:01:43.033 | INFO | __main__:<module>:132 - remove batched parquets
209209
2024-03-14 12:01:43.035 | INFO | __main__:<module>:135 - execution time: 0:06:14.791580
210+
2024-04-18 10:35:22.856 | INFO | __main__:<module>:148 - Analysis date: 2024-04-17
211+
2024-04-18 10:38:25.816 | INFO | __main__:loop_through_batches_and_download_vp:111 - exported batch 0 to GCS: 0:03:02.950348
212+
2024-04-18 10:39:56.253 | INFO | __main__:loop_through_batches_and_download_vp:111 - exported batch 1 to GCS: 0:01:30.434267
213+
2024-04-18 10:45:13.007 | INFO | __main__:loop_through_batches_and_download_vp:111 - exported batch 2 to GCS: 0:05:16.754172
214+
2024-04-18 10:47:08.452 | INFO | __main__:loop_through_batches_and_download_vp:111 - exported batch 3 to GCS: 0:01:55.425286
215+
2024-04-18 10:47:08.453 | INFO | __main__:<module>:155 - execution time: 0:11:45.586863
216+
2024-04-18 10:47:29.679 | INFO | __main__:<module>:97 - Analysis date: 2024-04-17
217+
2024-04-18 10:47:37.286 | INFO | __main__:<module>:105 - concat and filter batched data: 0:00:07.606835
218+
2024-04-18 10:51:44.225 | INFO | __main__:<module>:112 - export concatenated vp: 0:04:06.939409
219+
2024-04-18 10:56:04.569 | INFO | __main__:<module>:134 - remove batched parquets
220+
2024-04-18 10:56:04.570 | INFO | __main__:<module>:137 - execution time: 0:08:34.890652

‎gtfs_funnel/logs/vp_preprocessing.log

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,11 @@
2020
2024-03-14 12:16:08.742 | INFO | __main__:<module>:202 - 2024-03-13: vp_direction script execution time: 0:07:35.556127
2121
2024-03-14 12:43:58.062 | INFO | __main__:<module>:153 - 2024-03-13: condense vp for trip 0:04:45.267623
2222
2024-03-14 12:56:43.421 | INFO | __main__:<module>:161 - 2024-03-13: prepare vp to use in nearest neighbor: 0:12:45.358549
23+
2024-04-18 11:04:43.747 | INFO | __main__:<module>:169 - 2024-04-17: pare down vp: 0:02:22.947760
24+
2024-04-18 11:08:55.978 | INFO | __main__:attach_prior_vp_add_direction:90 - persist vp gddf: 0:03:51.080485
25+
2024-04-18 11:13:14.285 | INFO | __main__:attach_prior_vp_add_direction:122 - np vectorize arrays for direction: 0:04:18.306848
26+
2024-04-18 11:13:22.726 | INFO | __main__:<module>:194 - 2024-04-17: export vp direction: 0:08:17.828237
27+
2024-04-18 11:14:49.190 | INFO | __main__:<module>:200 - 2024-04-17: export usable vp with direction: 0:01:26.463779
28+
2024-04-18 11:14:49.191 | INFO | __main__:<module>:203 - 2024-04-17: vp_direction script execution time: 0:09:44.292016
29+
2024-04-18 11:34:37.603 | INFO | __main__:<module>:120 - 2024-04-17: condense vp for trip 0:06:16.809244
30+
2024-04-18 11:52:47.882 | INFO | __main__:<module>:128 - 2024-04-17: prepare vp to use in nearest neighbor: 0:18:10.278588

‎gtfs_funnel/operator_scheduled_stats.py

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -121,26 +121,19 @@ def operator_typology_breakdown(df: pd.DataFrame) -> pd.DataFrame:
121121
Get a count of how many routes (not route-dir)
122122
have a certain primary typology.
123123
"""
124-
df2 = (df
125-
.groupby(
126-
["schedule_gtfs_dataset_key", "primary_typology"])
127-
.agg({"route_id": "nunique"})
128-
.reset_index()
129-
)
130-
131-
df_wide = df2.pivot(
132-
index="schedule_gtfs_dataset_key",
133-
columns = "primary_typology",
134-
values="route_id"
135-
).reset_index().fillna(0)
136-
137-
typology_values = ["downtown_local", "local",
138-
"rapid", "coverage"]
139-
140-
df_wide[typology_values] = df_wide[typology_values].astype(int)
124+
typology_values = [
125+
f"is_{i}" for i in
126+
["downtown_local", "local", "rapid", "coverage"]
127+
]
141128

142-
rename_dict = {old_name: f"n_{old_name}_routes"
129+
df_wide = (df.groupby("schedule_gtfs_dataset_key")
130+
.agg({**{c: "sum" for c in typology_values}})
131+
.reset_index()
132+
)
133+
134+
rename_dict = {old_name: f"n_{old_name.replace('is_', '')}_routes"
143135
for old_name in typology_values}
136+
144137
df_wide = df_wide.rename(columns = rename_dict)
145138

146139
return df_wide
@@ -152,7 +145,7 @@ def operator_typology_breakdown(df: pd.DataFrame) -> pd.DataFrame:
152145

153146
ROUTE_TYPOLOGY = GTFS_DATA_DICT.schedule_tables.route_typologies
154147
OPERATOR_EXPORT = GTFS_DATA_DICT.schedule_tables.operator_scheduled_stats
155-
OPERATOR_ROUTE_EXPORT = GTFS_DATA_DICT.schedule_tables.operator_route
148+
OPERATOR_ROUTE_EXPORT = GTFS_DATA_DICT.schedule_tables.operator_routes
156149

157150
for analysis_date in analysis_date_list:
158151
start = datetime.datetime.now()

‎gtfs_funnel/route_typologies.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -244,7 +244,7 @@ def primary_secondary_typology(
244244

245245
start = datetime.datetime.now()
246246

247-
#roads = delayed(prep_roads)(GTFS_DATA_DICT)
247+
roads = delayed(prep_roads)(GTFS_DATA_DICT)
248248
ROAD_BUFFER_METERS = 20
249249
TYPOLOGY_THRESHOLD = 0.10
250250

‎gtfs_funnel/update_vars.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
rt_dates.oct_week + rt_dates.apr_week)
55

66
analysis_date_list = [
7-
rt_dates.DATES["mar2024"]
7+
rt_dates.DATES["apr2024"]
88
]
99

1010
GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")

‎high_quality_transit_areas/logs/hqta_processing.log

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,10 @@
8787
2024-03-21 12:04:01.449 | INFO | __main__:<module>:163 - C3_create_bus_hqta_types 2024-03-13 execution time: 0:00:19.553787
8888
2024-03-21 12:04:42.807 | INFO | __main__:<module>:295 - D1_assemble_hqta_points 2024-03-13 execution time: 0:00:22.988739
8989
2024-03-21 12:05:20.102 | INFO | __main__:<module>:167 - D2_assemble_hqta_polygons 2024-03-13 execution time: 0:00:19.166756
90+
2024-04-18 12:02:44.870 | INFO | __main__:<module>:354 - A1_rail_ferry_brt_stops 2024-04-17 execution time: 0:00:59.115933
91+
2024-04-18 12:09:06.425 | INFO | __main__:<module>:256 - B2_sjoin_stops_to_segments 2024-04-17 execution time: 0:00:50.678918
92+
2024-04-18 12:09:36.340 | INFO | __main__:<module>:142 - C1_prep_pairwise_intersections 2024-04-17 execution time: 0:00:07.719892
93+
2024-04-18 12:10:31.226 | INFO | __main__:<module>:125 - C2_find_intersections 2024-04-17 execution time: 0:00:33.802270
94+
2024-04-18 12:11:31.609 | INFO | __main__:<module>:163 - C3_create_bus_hqta_types 2024-04-17 execution time: 0:00:37.330690
95+
2024-04-18 12:12:28.853 | INFO | __main__:<module>:296 - D1_assemble_hqta_points 2024-04-17 execution time: 0:00:31.955298
96+
2024-04-18 12:13:36.294 | INFO | __main__:<module>:167 - D2_assemble_hqta_polygons 2024-04-17 execution time: 0:00:40.596021

‎high_quality_transit_areas/update_vars.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
from shared_utils import rt_dates
22

3-
analysis_date = rt_dates.DATES["mar2024"]
3+
analysis_date = rt_dates.DATES["apr2024"]
44

55
GCS_FILE_PATH = ("gs://calitp-analytics-data/data-analyses/"
66
"high_quality_transit_areas/")

‎open_data/update_vars.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from pathlib import Path
22
from shared_utils import rt_dates
33

4-
analysis_date = rt_dates.DATES["mar2024"]
4+
analysis_date = rt_dates.DATES["apr2024"]
55

66
GCS_FILE_PATH = "gs://calitp-analytics-data/data-analyses/"
77
COMPILED_CACHED_VIEWS = f"{GCS_FILE_PATH}rt_delay/compiled_cached_views/"

‎rt_scheduled_v_ran/logs/rt_v_scheduled_route_metrics.log

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,4 @@
4949
2024-03-29 12:11:15.513 | INFO | __main__:route_metrics:47 - route aggregation 2023-04-14: 0:00:01.444968
5050
2024-03-29 12:11:16.851 | INFO | __main__:route_metrics:47 - route aggregation 2023-04-15: 0:00:01.334524
5151
2024-03-29 12:11:18.456 | INFO | __main__:route_metrics:47 - route aggregation 2023-04-16: 0:00:01.601076
52+
2024-04-18 13:34:04.718 | INFO | __main__:route_metrics:74 - route aggregation 2024-04-17: 0:00:02.236708

‎rt_scheduled_v_ran/logs/rt_v_scheduled_trip_metrics.log

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,3 +397,7 @@
397397
2024-03-29 12:01:25.746 | INFO | __main__:rt_schedule_trip_metrics:337 - Total run time for metrics on 2023-10-12: 0:01:03.980200
398398
2024-03-29 12:02:28.269 | INFO | __main__:rt_schedule_trip_metrics:337 - Total run time for metrics on 2023-10-13: 0:01:02.513314
399399
2024-03-29 12:03:14.207 | INFO | __main__:rt_schedule_trip_metrics:337 - Total run time for metrics on 2023-10-14: 0:00:45.927788
400+
2024-04-18 12:23:17.745 | INFO | __main__:rt_schedule_trip_metrics:280 - tabular trip metrics 2024-04-17: 0:04:28.804318
401+
2024-04-18 13:02:41.637 | INFO | __main__:rt_schedule_trip_metrics:280 - tabular trip metrics 2024-04-17: 0:03:45.566584
402+
2024-04-18 13:32:29.183 | INFO | __main__:rt_schedule_trip_metrics:285 - spatial trip metrics 2024-04-17: 0:29:47.546487
403+
2024-04-18 13:33:34.374 | INFO | __main__:rt_schedule_trip_metrics:333 - Total run time for metrics on 2024-04-17: 0:34:38.304007

‎rt_scheduled_v_ran/scripts/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,6 @@ rt_sched_pipeline:
44
python rt_v_scheduled_routes.py
55

66

7+
# this can be run after rt_segment_speeds make rt_stop_times_pipeline is run
78
schedule_rt_stop_times_table:
89
python rt_stop_times.py

‎rt_scheduled_v_ran/scripts/rt_stop_times.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import pandas as pd
77

88
from segment_speed_utils import helpers, segment_calcs
9-
from update_vars import SEGMENT_GCS, RT_SCHED_GCS
9+
from update_vars import GTFS_DATA_DICT, SEGMENT_GCS, RT_SCHED_GCS
1010

1111
def prep_scheduled_stop_times(
1212
analysis_date: str

‎rt_scheduled_v_ran/scripts/rt_v_scheduled_trip.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def buffer_shapes(
160160
crs=PROJECT_CRS,
161161
get_pandas=True,
162162
**kwargs
163-
).dropna(
163+
).pipe(helpers.remove_shapes_outside_ca).dropna(
164164
subset="geometry"
165165
).query("shape_array_key not in @amtrak_outside_ca")
166166

‎rt_scheduled_v_ran/scripts/update_vars.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@
44
oct_week = rt_dates.get_week("oct2023", exclude_wed=True)
55
apr_week = rt_dates.get_week("apr2023", exclude_wed=True)
66

7-
analysis_date_list = rt_dates.y2024_dates + rt_dates.y2023_dates + oct_week + apr_week
7+
analysis_date_list = [
8+
rt_dates.DATES["apr2024"]
9+
]
810

911
GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")
1012

‎rt_segment_speeds/logs/avg_speeds.log

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,3 +132,10 @@
132132
2024-03-28 18:45:04.105 | INFO | __main__:<module>:304 - average rollups for ['2023-10-09', '2023-10-10', '2023-10-11', '2023-10-12', '2023-10-13', '2023-10-14', '2023-10-15']: 0:07:22.839496
133133
2024-03-28 18:52:09.308 | INFO | __main__:multi_day_segment_averages:240 - multi day segment execution time: 0:07:05.201271
134134
2024-03-28 18:52:09.420 | INFO | __main__:<module>:304 - average rollups for ['2023-04-10', '2023-04-11', '2023-04-12', '2023-04-13', '2023-04-14', '2023-04-15', '2023-04-16']: 0:07:05.312950
135+
2024-04-18 15:42:58.828 | INFO | __main__:single_day_summary_averages:82 - trip avg 0:00:14.291404
136+
2024-04-18 15:43:09.833 | INFO | __main__:single_day_summary_averages:122 - route dir avg: 0:00:11.005300
137+
2024-04-18 15:43:09.834 | INFO | __main__:single_day_summary_averages:123 - single day summary speed execution time: 0:00:25.296704
138+
2024-04-18 15:43:09.912 | INFO | __main__:<module>:246 - average rollups for 2024-04-17: 0:00:25.384157
139+
2024-04-18 15:49:30.073 | INFO | __main__:single_day_segment_averages:167 - shape seg avg 0:05:04.862313
140+
2024-04-18 15:53:11.195 | INFO | __main__:single_day_segment_averages:183 - route dir seg avg 0:03:41.121253
141+
2024-04-18 15:53:11.196 | INFO | __main__:single_day_segment_averages:184 - single day segment execution time: 0:08:45.983566

‎rt_segment_speeds/logs/cut_stop_segments.log

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
0:15:29.6603982024-02-15 13:36:56.379 | INFO | __main__:<module>:156 - cut segments 2024-02-14: 0:13:55.835925
22
2024-03-14 13:33:18.528 | INFO | __main__:<module>:156 - cut segments 2024-03-13: 0:26:02.810762
3+
2024-04-18 12:44:11.808 | INFO | __main__:<module>:155 - cut segments 2024-04-17: 0:33:24.198635

‎rt_segment_speeds/logs/interpolate_stop_arrival.log

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
11
2024-02-15 14:18:47.082 | INFO | __main__:interpolate_stop_arrivals:110 - get stop arrivals 2024-02-14: 0:09:37.655118
2+
2024-04-18 14:30:37.401 | INFO | interpolate_stop_arrival:interpolate_stop_arrivals:203 - get stop arrivals 2024-04-17: 0:18:48.633416
3+
2024-04-18 14:36:29.213 | INFO | interpolate_stop_arrival:interpolate_stop_arrivals:214 - interpolate arrivals for stop_segments 2024-04-17: 2024-04-17: 0:24:40.445216
4+
2024-04-18 15:40:32.431 | INFO | interpolate_stop_arrival:interpolate_stop_arrivals:214 - interpolate arrivals for rt_stop_times 2024-04-17: 2024-04-17: 0:18:11.754862
5+
2024-04-18 15:42:16.358 | INFO |

‎rt_segment_speeds/logs/nearest_vp.log

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
2024-02-15 13:48:26.270 | INFO | __main__:nearest_neighbor_shape_segments:203 - shape segments 2024-02-14: 0:10:31.474349
22
2024-02-15 14:08:52.783 | INFO | __main__:nearest_neighbor_rt_stop_times:135 - RT stop times 2024-02-14: 0:11:07.883083
3+
2024-04-18 13:19:11.258 | INFO | nearest_vp_to_stop:nearest_neighbor_for_stop:134 - nearest neighbor for stop_segments 2024-04-17: 0:27:28.612602
4+
2024-04-18 15:22:16.322 | INFO | nearest_vp_to_stop:nearest_neighbor_for_stop:134 - nearest neighbor for rt_stop_times 2024-04-17: 0:17:44.714181
5+

‎rt_segment_speeds/logs/speeds_by_segment_trip.log

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
11
2024-02-15 14:19:36.965 | INFO | __main__:calculate_speed_from_stop_arrivals:132 - speeds by segment: 2024-02-14: 0:00:31.989535
22
2024-02-15 14:45:36.035 | INFO | __main__:calculate_speed_from_stop_arrivals:132 - speeds by segment: 2024-01-17: 0:00:32.109610
33
2024-03-28 13:11:29.272 | INFO | __main__:calculate_speed_from_stop_arrivals:148 - speeds by segment for stop_segments 2024-03-13: 0:01:36.402022
4+
2024-04-18 15:04:52.606 | INFO | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:148 - speeds by segment for stop_segments 2024-04-17: 0:01:48.092886
5+
2024-04-18 15:42:16.358 | INFO | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:148 - speeds by segment for rt_stop_times 2024-04-17: 0:01:43.666310

‎rt_segment_speeds/scripts/average_segment_speeds.py

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99
from dask import delayed, compute
1010
from loguru import logger
11-
from pathlib import Path
1211
from typing import Literal
1312

1413
from calitp_data_analysis.geography_utils import WGS84
@@ -240,33 +239,6 @@ def multi_day_segment_averages(analysis_date_list: list, dict_inputs: dict):
240239
logger.info(f"multi day segment execution time: {end - start}")
241240

242241
return
243-
244-
def stage_open_data_exports(analysis_date: str, dict_inputs: dict):
245-
"""
246-
For the datasets we publish to Geoportal,
247-
export them to a stable GCS URL so we can always
248-
read it in open_data/catalog.yml.
249-
"""
250-
datasets = [
251-
dict_inputs["route_dir_single_segment"],
252-
dict_inputs["route_dir_single_summary"]
253-
]
254-
255-
for d in datasets:
256-
gdf = gpd.read_parquet(
257-
f"{SEGMENT_GCS}{d}_{analysis_date}.parquet"
258-
)
259-
260-
utils.geoparquet_gcs_export(
261-
gdf,
262-
f"{SEGMENT_GCS}export/",
263-
f"{Path(d).stem}"
264-
)
265-
del gdf
266-
267-
print(f"overwrite {datasets}")
268-
269-
return
270242

271243

272244
if __name__ == "__main__":
@@ -287,7 +259,6 @@ def stage_open_data_exports(analysis_date: str, dict_inputs: dict):
287259
start = datetime.datetime.now()
288260

289261
single_day_segment_averages(analysis_date, STOP_SEG_DICT)
290-
stage_open_data_exports(analysis_date, STOP_SEG_DICT)
291262

292263
end = datetime.datetime.now()
293264

‎rt_segment_speeds/scripts/cut_stop_segments.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from calitp_data_analysis import utils
1818
from calitp_data_analysis.geography_utils import WGS84
19-
from segment_speed_utils import gtfs_schedule_wrangling
19+
from segment_speed_utils import gtfs_schedule_wrangling, helpers
2020
from update_vars import GTFS_DATA_DICT, SEGMENT_GCS
2121
from segment_speed_utils.project_vars import PROJECT_CRS
2222

‎rt_segment_speeds/scripts/interpolate_stop_arrival.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from pathlib import Path
1313
from typing import Literal, Optional
1414

15-
from segment_speed_utils import (array_utils,
15+
from segment_speed_utils import (array_utils, helpers,
1616
segment_calcs, wrangle_shapes)
1717
from update_vars import SEGMENT_GCS, GTFS_DATA_DICT
1818
from segment_speed_utils.project_vars import PROJECT_CRS, SEGMENT_TYPES

‎rt_segment_speeds/scripts/pipe.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
interpolate_stop_arrivals.py,
55
and calculate_speed_from_stop_arrivals.py
66
"""
7+
import sys
8+
from loguru import logger
79
from pathlib import Path
810
from typing import Literal, Optional
911

@@ -24,6 +26,7 @@ def nearest_neigbor_to_speed(
2426
interpolation of stop arrival, deriving segment speeds
2527
between stops.
2628
"""
29+
2730
LOG_FILE = "../logs/nearest_vp.log"
2831
logger.add(LOG_FILE, retention="3 months")
2932
logger.add(sys.stderr,
@@ -47,7 +50,7 @@ def nearest_neigbor_to_speed(
4750
segment_type = segment_type,
4851
config_path = config_path
4952
)
50-
53+
5154
LOG_FILE = "../logs/speeds_by_segment_trip.log"
5255
logger.add(LOG_FILE, retention="3 months")
5356
logger.add(sys.stderr,

0 commit comments

Comments
 (0)