test sjoin postprocessing through recalculating speeds

cal-itp · Sep 22, 2023 · 009ed58 · 009ed58
1 parent 93a908f
commit 009ed58
Show file tree

Hide file tree

Showing 8 changed files with 390 additions and 106 deletions.
diff --git a/rt_segment_speeds/logs/sjoin_vp_segments.log b/rt_segment_speeds/logs/sjoin_vp_segments.log
@@ -42,3 +42,9 @@
 2023-09-14 13:06:15.758 | INFO     | __main__:<module>:223 - execution time: 0:12:03.599629
 2023-09-14 13:06:31.913 | INFO     | __main__:<module>:457 - Analysis date: 2023-09-13
 2023-09-14 13:14:08.656 | INFO     | __main__:<module>:465 - remove erroneous sjoin results: 0:07:36.742381
+2023-09-21 10:58:58.711 | INFO     | __main__:<module>:457 - Analysis date: 2023-09-13
+2023-09-21 11:05:58.377 | INFO     | __main__:<module>:465 - remove erroneous sjoin results: 0:06:59.626650
+2023-09-21 12:11:55.875 | INFO     | __main__:<module>:456 - Analysis date: 2023-08-15
+2023-09-21 12:19:13.216 | INFO     | __main__:<module>:464 - remove erroneous sjoin results: 0:07:17.337816
+2023-09-21 12:58:44.489 | INFO     | __main__:<module>:456 - Analysis date: 2023-07-12
+2023-09-21 13:07:33.610 | INFO     | __main__:<module>:464 - remove erroneous sjoin results: 0:08:49.120980
diff --git a/rt_segment_speeds/logs/speeds_by_segment_trip.log b/rt_segment_speeds/logs/speeds_by_segment_trip.log
@@ -40,3 +40,27 @@
 2023-08-24 14:51:47.828 | INFO     | __main__:linear_referencing_and_speed_by_segment:123 - calculate speeds: 0:00:00.006706
 2023-08-24 14:56:14.082 | INFO     | __main__:<module>:150 - speeds for stop segments: 0:04:32.610802
 2023-08-24 14:56:14.083 | INFO     | __main__:<module>:151 - execution time: 0:04:32.611824
+2023-09-21 11:53:11.705 | INFO     | __main__:<module>:367 - Analysis date: 2023-09-13
+2023-09-21 11:53:18.285 | INFO     | __main__:linear_referencing_vp_against_line:58 - set up merged vp with segments: 0:00:00.152710
+2023-09-21 11:53:18.322 | INFO     | __main__:linear_referencing_vp_against_line:76 - linear referencing: 0:00:00.037063
+2023-09-21 11:57:20.821 | INFO     | __main__:linear_referencing_and_speed_by_segment:300 - linear referencing: 0:04:09.111393
+2023-09-21 12:03:20.231 | INFO     | __main__:linear_referencing_and_speed_by_segment:311 - make wide and get initial speeds: 0:05:59.409953
+2023-09-21 12:05:33.476 | INFO     | __main__:linear_referencing_and_speed_by_segment:352 - recalculate speeds and get final: 0:02:13.244950
+2023-09-21 12:05:46.785 | INFO     | __main__:<module>:375 - speeds for stop segments: 0:12:35.079026
+2023-09-21 12:05:46.788 | INFO     | __main__:<module>:376 - execution time: 0:12:35.081785
+2023-09-21 12:28:12.385 | INFO     | __main__:<module>:367 - Analysis date: 2023-08-15
+2023-09-21 12:28:19.027 | INFO     | __main__:linear_referencing_vp_against_line:58 - set up merged vp with segments: 0:00:00.162699
+2023-09-21 12:28:19.063 | INFO     | __main__:linear_referencing_vp_against_line:76 - linear referencing: 0:00:00.036557
+2023-09-21 12:32:49.165 | INFO     | __main__:linear_referencing_and_speed_by_segment:300 - linear referencing: 0:04:36.761005
+2023-09-21 12:39:56.447 | INFO     | __main__:linear_referencing_and_speed_by_segment:311 - make wide and get initial speeds: 0:07:07.282236
+2023-09-21 12:42:14.396 | INFO     | __main__:linear_referencing_and_speed_by_segment:352 - recalculate speeds and get final: 0:02:17.948959
+2023-09-21 12:42:27.893 | INFO     | __main__:<module>:375 - speeds for stop segments: 0:14:15.493395
+2023-09-21 12:42:27.894 | INFO     | __main__:<module>:376 - execution time: 0:14:15.494456
+2023-09-21 13:16:38.177 | INFO     | __main__:<module>:367 - Analysis date: 2023-07-12
+2023-09-21 13:16:44.655 | INFO     | __main__:linear_referencing_vp_against_line:58 - set up merged vp with segments: 0:00:00.156084
+2023-09-21 13:16:44.692 | INFO     | __main__:linear_referencing_vp_against_line:76 - linear referencing: 0:00:00.037555
+2023-09-21 13:20:41.231 | INFO     | __main__:linear_referencing_and_speed_by_segment:300 - linear referencing: 0:04:03.035504
+2023-09-21 13:26:57.794 | INFO     | __main__:linear_referencing_and_speed_by_segment:311 - make wide and get initial speeds: 0:06:16.562615
+2023-09-21 13:29:08.771 | INFO     | __main__:linear_referencing_and_speed_by_segment:352 - recalculate speeds and get final: 0:02:10.977540
+2023-09-21 13:29:21.791 | INFO     | __main__:<module>:375 - speeds for stop segments: 0:12:43.599595
+2023-09-21 13:29:21.792 | INFO     | __main__:<module>:376 - execution time: 0:12:43.600710
diff --git a/rt_segment_speeds/logs/valid_vehicle_positions.log b/rt_segment_speeds/logs/valid_vehicle_positions.log
@@ -85,3 +85,21 @@
 2023-09-14 13:20:47.965 | INFO     | __main__:pare_down_vp_by_segment:123 - exported: 0:06:15.874961
 2023-09-14 13:20:47.966 | INFO     | __main__:<module>:148 - pare down vp by stop segments for all cases 0:06:17.237883
 2023-09-14 13:20:47.967 | INFO     | __main__:<module>:151 - execution time: 0:06:17.243094
+2023-09-21 11:06:27.840 | INFO     | __main__:<module>:134 - Analysis date: 2023-09-13
+2023-09-21 11:06:29.244 | INFO     | __main__:pare_down_vp_by_segment:103 - merge usable vp with sjoin results: 0:00:01.386494
+2023-09-21 11:06:29.323 | INFO     | __main__:pare_down_vp_by_segment:112 - keep enter/exit points: 0:00:00.079269
+2023-09-21 11:13:01.939 | INFO     | __main__:pare_down_vp_by_segment:123 - exported: 0:06:32.615887
+2023-09-21 11:13:01.940 | INFO     | __main__:<module>:148 - pare down vp by stop segments for all cases 0:06:34.083216
+2023-09-21 11:13:01.941 | INFO     | __main__:<module>:151 - execution time: 0:06:34.088864
+2023-09-21 12:19:30.995 | INFO     | __main__:<module>:134 - Analysis date: 2023-08-15
+2023-09-21 12:19:32.432 | INFO     | __main__:pare_down_vp_by_segment:103 - merge usable vp with sjoin results: 0:00:01.410420
+2023-09-21 12:19:32.535 | INFO     | __main__:pare_down_vp_by_segment:112 - keep enter/exit points: 0:00:00.103218
+2023-09-21 12:27:56.376 | INFO     | __main__:pare_down_vp_by_segment:123 - exported: 0:08:23.840718
+2023-09-21 12:27:56.378 | INFO     | __main__:<module>:148 - pare down vp by stop segments for all cases 0:08:25.356390
+2023-09-21 12:27:56.379 | INFO     | __main__:<module>:151 - execution time: 0:08:25.360301
+2023-09-21 13:07:51.526 | INFO     | __main__:<module>:134 - Analysis date: 2023-07-12
+2023-09-21 13:07:52.848 | INFO     | __main__:pare_down_vp_by_segment:103 - merge usable vp with sjoin results: 0:00:01.297768
+2023-09-21 13:07:52.939 | INFO     | __main__:pare_down_vp_by_segment:112 - keep enter/exit points: 0:00:00.091154
+2023-09-21 13:16:20.491 | INFO     | __main__:pare_down_vp_by_segment:123 - exported: 0:08:27.552189
+2023-09-21 13:16:20.493 | INFO     | __main__:<module>:148 - pare down vp by stop segments for all cases 0:08:28.942646
+2023-09-21 13:16:20.493 | INFO     | __main__:<module>:151 - execution time: 0:08:28.946383
diff --git a/rt_segment_speeds/scripts/A2_sjoin_postprocessing.py b/rt_segment_speeds/scripts/A2_sjoin_postprocessing.py
@@ -380,7 +380,6 @@ def remove_erroneous_sjoin_results(
     """
     USABLE_VP = dict_inputs["stage1"]
     INPUT_FILE_PREFIX = dict_inputs["stage2"]
-
     SEGMENT_IDENTIFIER_COLS = dict_inputs["segment_identifier_cols"]
     SEGMENT_TRIP_COLS = ["trip_instance_key"] + SEGMENT_IDENTIFIER_COLS
     GROUPING_COL = dict_inputs["grouping_col"]

diff --git a/rt_segment_speeds/scripts/A3_valid_vehicle_positions.py b/rt_segment_speeds/scripts/A3_valid_vehicle_positions.py
@@ -116,7 +116,7 @@ def pare_down_vp_by_segment(
                   .repartition(npartitions=3)
                  )
     vp_to_keep.to_parquet(
-        f"{SEGMENT_GCS}vp_pare_down/{EXPORT_FILE}_all_{analysis_date}",
+        f"{SEGMENT_GCS}{EXPORT_FILE}_{analysis_date}",
         overwrite=True
     )