Skip to content

Commit e88462a

Browse files
tiffanychu90tiffanychu90
authored
Merge pull request #962 from cal-itp/add-speeds-to-public-gcs
Add speeds to public gcs
2 parents 04aac64 + 3dc6e6b commit e88462a

28 files changed

+1084
-432
lines changed

‎_shared_utils/shared_utils/rt_dates.py

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,23 @@
2828
"apr2023b": "2023-04-11",
2929
"apr2023": "2023-04-12", # main April date
3030
"apr2023c": "2023-04-13",
31+
"apr2023d": "2023-04-14",
32+
"apr2023e": "2023-04-15",
33+
"apr2023f": "2023-04-16",
3134
"may2023": "2023-05-17",
3235
"jun2023": "2023-06-14",
3336
"jul2023": "2023-07-12",
3437
# "aug2023": "2023-08-16", # this date is missing Muni
3538
"aug2023": "2023-08-15",
3639
"aug2023a": "2023-08-23", # date used for speedmaps
3740
"sep2023": "2023-09-13",
41+
"oct2023a": "2023-10-09",
42+
"oct2023b": "2023-10-10",
3843
"oct2023": "2023-10-11",
39-
"oct2023a": "2023-10-13", # add weekend dates for SB 125 service increase
40-
"oct2023b": "2023-10-14",
44+
"oct2023c": "2023-10-12",
45+
"oct2023d": "2023-10-13",
46+
"oct2023e": "2023-10-14", # add weekend dates for SB 125 service increase
47+
"oct2023f": "2023-10-15",
4148
"nov2023": "2023-11-15",
4249
}
4350

‎gtfs_funnel/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ download_gtfs_data:
55
python download_shapes.py
66
python download_stop_times.py
77
python download_vehicle_positions.py
8-
python concatenate_vehicle_positions.py
8+
python concatenate_vehicle_positions.py
99

1010
preprocess:
1111
python stop_times_with_direction.py
1212
python vp_keep_usable.py
1313
python vp_direction.py
1414
python cleanup.py
15-
python route_typologies.py
15+
#python route_typologies.py
1616

1717

1818
funnel_gtfs_data:

‎gtfs_funnel/concatenate_vehicle_positions.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
import shapely
1111
import sys
1212

13-
from dask import delayed
13+
from dask import delayed, compute
1414
from loguru import logger
1515

16-
from shared_utils import dask_utils, schedule_rt_utils
16+
from shared_utils import schedule_rt_utils
1717
from calitp_data_analysis import utils
1818
from segment_speed_utils.project_vars import SEGMENT_GCS
1919

@@ -109,12 +109,18 @@ def remove_batched_parquets(analysis_date: str):
109109
time2 = datetime.datetime.now()
110110
logger.info(f"export concatenated vp: {time2 - time1}")
111111

112+
# Delete objects once it's saved out
113+
# Loop to save out multiple dates of vp may cause kernel to crash
114+
del concatenated_vp_df
115+
112116
# Import concatenated tabular vp and make it a gdf
113-
vp = pd.read_parquet(
117+
vp = delayed(pd.read_parquet)(
114118
f"{SEGMENT_GCS}vp_{analysis_date}_concat/"
115119
).reset_index(drop=True)
116120

117-
vp_gdf = vp_into_gdf(vp)
121+
vp_gdf = delayed(vp_into_gdf)(vp)
122+
123+
vp_gdf = compute(vp_gdf)[0]
118124

119125
utils.geoparquet_gcs_export(
120126
vp_gdf,
@@ -127,5 +133,5 @@ def remove_batched_parquets(analysis_date: str):
127133

128134
end = datetime.datetime.now()
129135
logger.info(f"execution time: {end - start}")
130-
131-
136+
137+
del vp_gdf

‎gtfs_funnel/logs/download_data.log

Lines changed: 160 additions & 0 deletions
Large diffs are not rendered by default.

‎gtfs_funnel/logs/download_vp_v2.log

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,112 @@
4343
2023-11-16 08:54:49.396 | INFO | __main__:<module>:110 - export concatenated vp: 0:03:07.799421
4444
2023-11-16 08:57:34.095 | INFO | __main__:<module>:126 - remove batched parquets
4545
2023-11-16 08:57:34.098 | INFO | __main__:<module>:129 - execution time: 0:06:00.023771
46+
2023-12-06 15:53:25.425 | INFO | __main__:<module>:146 - Analysis date: 2023-04-14
47+
2023-12-06 15:54:55.818 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:30.358555
48+
2023-12-06 15:55:59.221 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:01:03.401890
49+
2023-12-06 15:58:43.657 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:02:44.435456
50+
2023-12-06 16:00:35.795 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:52.136564
51+
2023-12-06 16:00:38.019 | INFO | __main__:<module>:153 - execution time: 0:07:12.559036
52+
2023-12-06 16:00:38.019 | INFO | __main__:<module>:146 - Analysis date: 2023-04-15
53+
2023-12-06 16:01:49.707 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:11.687373
54+
2023-12-06 16:02:25.927 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:36.219273
55+
2023-12-06 16:04:06.562 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:01:40.634251
56+
2023-12-06 16:05:19.630 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:13.067106
57+
2023-12-06 16:05:20.923 | INFO | __main__:<module>:153 - execution time: 0:04:42.903343
58+
2023-12-06 16:05:20.924 | INFO | __main__:<module>:146 - Analysis date: 2023-04-16
59+
2023-12-06 16:06:28.174 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:07.250476
60+
2023-12-06 16:06:59.570 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:31.394971
61+
2023-12-06 16:08:25.014 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:01:25.443304
62+
2023-12-06 16:09:40.831 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:15.816201
63+
2023-12-06 16:09:42.430 | INFO | __main__:<module>:153 - execution time: 0:04:21.505770
64+
2023-12-06 16:10:00.996 | INFO | __main__:<module>:95 - Analysis date: 2023-04-14
65+
2023-12-06 16:10:08.379 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:07.381973
66+
2023-12-06 16:12:57.935 | INFO | __main__:<module>:110 - export concatenated vp: 0:02:49.555680
67+
2023-12-06 16:15:29.834 | INFO | __main__:<module>:126 - remove batched parquets
68+
2023-12-06 16:15:29.857 | INFO | __main__:<module>:129 - execution time: 0:05:28.859895
69+
2023-12-06 16:15:29.858 | INFO | __main__:<module>:95 - Analysis date: 2023-04-15
70+
2023-12-06 16:15:33.774 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:03.915208
71+
2023-12-06 16:17:12.548 | INFO | __main__:<module>:110 - export concatenated vp: 0:01:38.774064
72+
2023-12-06 16:18:56.325 | INFO | __main__:<module>:126 - remove batched parquets
73+
2023-12-06 16:18:56.327 | INFO | __main__:<module>:129 - execution time: 0:03:26.468585
74+
2023-12-06 16:18:56.328 | INFO | __main__:<module>:95 - Analysis date: 2023-04-16
75+
2023-12-06 16:19:00.361 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:04.032685
76+
2023-12-06 16:20:24.447 | INFO | __main__:<module>:110 - export concatenated vp: 0:01:24.086557
77+
2023-12-06 16:22:01.112 | INFO | __main__:<module>:126 - remove batched parquets
78+
2023-12-06 16:22:01.113 | INFO | __main__:<module>:129 - execution time: 0:03:04.784816
79+
2023-12-07 10:44:19.668 | INFO | __main__:<module>:146 - Analysis date: 2023-10-09
80+
2023-12-07 10:46:23.356 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:02:03.666694
81+
2023-12-07 10:47:15.802 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:52.445184
82+
2023-12-07 10:50:39.219 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:03:23.415247
83+
2023-12-07 10:52:11.742 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:32.522360
84+
2023-12-07 10:52:13.063 | INFO | __main__:<module>:153 - execution time: 0:07:53.373404
85+
2023-12-07 10:52:13.063 | INFO | __main__:<module>:146 - Analysis date: 2023-10-10
86+
2023-12-07 10:55:12.799 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:02:59.735493
87+
2023-12-07 10:56:16.570 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:01:03.769905
88+
2023-12-07 11:00:26.622 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:04:10.051981
89+
2023-12-07 11:03:05.019 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:02:38.370515
90+
2023-12-07 11:03:07.136 | INFO | __main__:<module>:153 - execution time: 0:10:54.072633
91+
2023-12-07 11:16:38.847 | INFO | __main__:<module>:146 - Analysis date: 2023-10-12
92+
2023-12-07 11:19:22.380 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:02:43.532370
93+
2023-12-07 11:20:44.174 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:01:21.793091
94+
2023-12-07 11:24:16.872 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:03:32.693001
95+
2023-12-07 11:26:01.393 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:44.519706
96+
2023-12-07 11:26:03.532 | INFO | __main__:<module>:153 - execution time: 0:09:24.684958
97+
2023-12-07 11:26:03.533 | INFO | __main__:<module>:146 - Analysis date: 2023-10-13
98+
2023-12-07 11:27:49.898 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:46.365398
99+
2023-12-07 11:28:42.884 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:52.985060
100+
2023-12-07 11:31:37.102 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:02:54.217587
101+
2023-12-07 11:33:15.956 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:38.852931
102+
2023-12-07 11:33:18.118 | INFO | __main__:<module>:153 - execution time: 0:07:14.584642
103+
2023-12-07 11:33:18.118 | INFO | __main__:<module>:146 - Analysis date: 2023-10-14
104+
2023-12-07 11:34:30.701 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:12.582000
105+
2023-12-07 11:35:05.565 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:34.862761
106+
2023-12-07 11:36:45.351 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:01:39.783550
107+
2023-12-07 11:38:00.729 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:15.377564
108+
2023-12-07 11:38:01.906 | INFO | __main__:<module>:153 - execution time: 0:04:43.787174
109+
2023-12-07 11:38:01.907 | INFO | __main__:<module>:146 - Analysis date: 2023-10-15
110+
2023-12-07 11:39:11.878 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 0 to GCS: 0:01:09.970960
111+
2023-12-07 11:39:41.119 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 1 to GCS: 0:00:29.239334
112+
2023-12-07 11:41:06.654 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 2 to GCS: 0:01:25.534943
113+
2023-12-07 11:42:23.741 | INFO | __main__:loop_through_batches_and_download_vp:109 - exported batch 3 to GCS: 0:01:17.085643
114+
2023-12-07 11:42:25.330 | INFO | __main__:<module>:153 - execution time: 0:04:23.422319
115+
2023-12-07 11:42:49.427 | INFO | __main__:<module>:95 - Analysis date: 2023-10-12
116+
2023-12-07 11:42:56.013 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:06.585412
117+
2023-12-07 11:45:34.652 | INFO | __main__:<module>:110 - export concatenated vp: 0:02:38.638416
118+
2023-12-07 11:56:29.508 | INFO | __main__:<module>:95 - Analysis date: 2023-10-13
119+
2023-12-07 11:56:36.821 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:07.300166
120+
2023-12-07 11:58:57.072 | INFO | __main__:<module>:110 - export concatenated vp: 0:02:20.250823
121+
2023-12-07 12:09:32.138 | INFO | __main__:<module>:95 - Analysis date: 2023-10-14
122+
2023-12-07 12:09:37.434 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:05.295270
123+
2023-12-07 12:11:29.596 | INFO | __main__:<module>:110 - export concatenated vp: 0:01:52.162187
124+
2023-12-07 12:11:29.598 | INFO | __main__:<module>:113 - vp concat part 1 execution time: 0:01:57.459187
125+
2023-12-07 12:11:29.598 | INFO | __main__:<module>:95 - Analysis date: 2023-10-15
126+
2023-12-07 12:11:35.022 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:05.422421
127+
2023-12-07 12:13:10.177 | INFO | __main__:<module>:110 - export concatenated vp: 0:01:35.155384
128+
2023-12-07 12:13:10.177 | INFO | __main__:<module>:113 - vp concat part 1 execution time: 0:01:40.578317
129+
2023-12-07 12:15:22.803 | INFO | __main__:<module>:95 - Analysis date: 2023-10-09
130+
2023-12-07 12:15:31.251 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:08.447395
131+
2023-12-07 12:18:01.558 | INFO | __main__:<module>:110 - export concatenated vp: 0:02:30.307021
132+
2023-12-07 12:18:01.559 | INFO | __main__:<module>:113 - vp concat part 1 execution time: 0:02:38.755800
133+
2023-12-07 12:18:01.560 | INFO | __main__:<module>:95 - Analysis date: 2023-10-10
134+
2023-12-07 12:18:07.290 | INFO | __main__:<module>:103 - concat and filter batched data: 0:00:05.730005
135+
2023-12-07 12:20:38.518 | INFO | __main__:<module>:110 - export concatenated vp: 0:02:31.227383
136+
2023-12-07 12:20:38.518 | INFO | __main__:<module>:113 - vp concat part 1 execution time: 0:02:36.957871
137+
2023-12-07 12:22:01.630 | INFO | __main__:<module>:30 - Analysis date: 2023-10-09
138+
2023-12-07 12:24:46.894 | INFO | __main__:<module>:48 - remove batched parquets
139+
2023-12-07 12:24:46.895 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:02:45.263826
140+
2023-12-07 12:37:35.222 | INFO | __main__:<module>:30 - Analysis date: 2023-10-10
141+
2023-12-07 12:40:19.937 | INFO | __main__:<module>:48 - remove batched parquets
142+
2023-12-07 12:40:19.938 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:02:44.695982
143+
2023-12-07 13:11:53.101 | INFO | __main__:<module>:30 - Analysis date: 2023-10-12
144+
2023-12-07 13:14:57.800 | INFO | __main__:<module>:48 - remove batched parquets
145+
2023-12-07 13:14:57.801 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:03:04.670146
146+
2023-12-07 13:15:00.088 | INFO | __main__:<module>:30 - Analysis date: 2023-10-13
147+
2023-12-07 13:17:37.647 | INFO | __main__:<module>:48 - remove batched parquets
148+
2023-12-07 13:17:37.647 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:02:37.558741
149+
2023-12-07 13:17:39.640 | INFO | __main__:<module>:30 - Analysis date: 2023-10-14
150+
2023-12-07 13:19:22.692 | INFO | __main__:<module>:48 - remove batched parquets
151+
2023-12-07 13:19:22.693 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:01:43.051667
152+
2023-12-07 13:19:23.914 | INFO | __main__:<module>:30 - Analysis date: 2023-10-15
153+
2023-12-07 13:20:57.669 | INFO | __main__:<module>:48 - remove batched parquets
154+
2023-12-07 13:20:57.670 | INFO | __main__:<module>:51 - vp concat part 2 execution time: 0:01:33.755100

0 commit comments

Comments
 (0)