From 9e222f2f6e7ce7c971d75651deb15addb2b14685 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Fri, 14 Nov 2025 22:30:24 -0800 Subject: [PATCH 1/7] Try url first then use local file path if url fails. --- src/data/create_products.py | 40 +++++++++++++++----------------- src/data/gulper.py | 46 ++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/src/data/create_products.py b/src/data/create_products.py index 8c83a7e4..aa5343a5 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -88,28 +88,26 @@ class CreateProducts: } def _open_ds(self): - if self.args.local: - local_nc = Path( - BASE_PATH, - self.args.auv_name, - MISSIONNETCDFS, - self.args.mission, - f"{self.args.auv_name}_{self.args.mission}_{FREQ}.nc", - ) + local_nc = Path( + BASE_PATH, + self.args.auv_name, + MISSIONNETCDFS, + self.args.mission, + f"{self.args.auv_name}_{self.args.mission}_{FREQ}.nc", + ) + # Requires mission to have been processed and archived to AUVCTD + dap_url = os.path.join( # noqa: PTH118 + AUVCTD_OPENDAP_BASE, + "surveys", + self.args.mission.split(".")[0], + "netcdf", + f"{self.args.auv_name}_{self.args.mission}_{FREQ}.nc", + ) + try: + self.ds = xr.open_dataset(dap_url) + except OSError: + self.logger.debug("%s not available yet", dap_url) self.ds = xr.open_dataset(local_nc) - else: - # Requires mission to have been processed and archived to AUVCTD - dap_url = os.path.join( # noqa: PTH118 - AUVCTD_OPENDAP_BASE, - "surveys", - self.args.mission.split(".")[0], - "netcdf", - f"{self.args.auv_name}_{self.args.mission}_{FREQ}.nc", - ) - try: - self.ds = xr.open_dataset(dap_url) - except OSError as err: - self.logger.error("Error opening %s: %s", dap_url, err) # noqa: TRY400 def _grid_dims(self) -> tuple: # From Matlab code in plot_sections.m: diff --git a/src/data/gulper.py b/src/data/gulper.py index ddffb14c..35069b93 100755 --- a/src/data/gulper.py +++ b/src/data/gulper.py @@ -32,27 +32,31 @@ def mission_start_esecs(self) -> float: return self.args.start_esecs # Get the first time record from mission's navigation.nc file - if self.args.local: - url = Path( - self.args.base_path, - self.args.auv_name, - MISSIONNETCDFS, - self.args.mission, - "navigation.nc", - ) - else: - # Relies on auv-python having processed the mission - url = os.path.join( # noqa: PTH118 - "http://dods.mbari.org/opendap/data/auvctd/", - MISSIONNETCDFS, - self.args.mission.split(".")[0], - self.args.mission.split(".")[0] + self.args.mission.split(".")[1], - self.args.mission, - "navigation.nc", - ) - self.logger.info("Reading mission start time from %s", url) - ds = xr.open_dataset(url) - return ds.time[0].to_numpy().astype("float64") / 1e9 + file_path = Path( + self.args.base_path, + self.args.auv_name, + MISSIONNETCDFS, + self.args.mission, + "navigation.nc", + ) + # Relies on auv-python having processed the mission + url = os.path.join( # noqa: PTH118 + "http://dods.mbari.org/opendap/data/auvctd/", + MISSIONNETCDFS, + self.args.mission.split(".")[0], + self.args.mission.split(".")[0] + self.args.mission.split(".")[1], + self.args.mission, + "navigation.nc", + ) + try: + self.logger.info("Reading mission start time from url = %s", url) + ds = xr.open_dataset(url, decode_times=False) + return ds.time[0].to_numpy().astype("float64") / 1e9 + except OSError: + self.logger.info("%s not available yet", url) + self.logger.info("Reading mission start time from file_path = %s", file_path) + ds = xr.open_dataset(file_path, decode_times=False) + return ds.time[0].to_numpy().astype("float64") def parse_gulpers(self, sec_delay: int = 1) -> dict: # noqa: C901, PLR0912, PLR0915 """Parse the Gulper times and bottle numbers from the auvctd syslog file. From 343e29517e79d6a0601dc6e397f2d74e73bb0322 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Fri, 14 Nov 2025 22:31:30 -0800 Subject: [PATCH 2/7] Add mission 2025.316.02, the one with GPS Week Rollover Bug to be fixed times. --- .vscode/launch.json | 7 +++++-- src/data/dorado_info.py | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 71db7357..3e0dc6fd 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -24,7 +24,9 @@ //"args": ["--auv_name", "dorado", "--mission", "2017.044.00", "--noinput", "-v"] //"args": ["--auv_name", "i2map", "--mission", "2019.157.02", "--noinput", "-v", "1"] //"args": ["--auv_name", "dorado", "--mission", "2010.265.00", "--noinput", "-v"] - "args": ["--auv_name", "dorado", "--mission", "2023.324.00", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"] + //"args": ["--auv_name", "dorado", "--mission", "2023.324.00", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"] + // Mission suffering from GPS Rollover bug. Add 1024 * 7 * 24 * 3600 = 619315200 seconds + "args": ["--auv_name", "dorado", "--mission", "2025.316.02", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs", "--add_seconds", "619315200" ] }, { "name": "1.1 - lopcToNetCDF", @@ -271,7 +273,8 @@ //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2008.010.10"] //"args": ["-v", "2", "--mission", "2004.029.03", "--noinput", "--no_cleanup"], //"args": ["-v", "1", "--mission", "2023.192.01", "--noinput", "--no_cleanup"], - "args": ["-v", "1", "--mission", "2010.151.04", "--noinput", "--no_cleanup", "--clobber"], + //"args": ["-v", "1", "--mission", "2010.151.04", "--noinput", "--no_cleanup", "--clobber"], + "args": ["-v", "1", "--mission", "2025.316.02", "--noinput", "--no_cleanup", "--add_seconds", "619315200"], }, { diff --git a/src/data/dorado_info.py b/src/data/dorado_info.py index 285fa02d..0307e843 100644 --- a/src/data/dorado_info.py +++ b/src/data/dorado_info.py @@ -2954,3 +2954,11 @@ " - ctdToUse = ctd2 " ), } +dorado_info["2025.316.02"] = { + "program": f"{MBTSLINE}", + "comment": ( + "Monterey Bay MBTS Mission - 31625G" + " ISUS, and LISST payloads removed, main vehicle computer NTP synced with GPS Week Rollover Bug, 1024*7*24*3600 seconds added to timestamps. " + " - ctdToUse = ctd2 " + ), +} From 9ca4c92961df588ba4b63275ead5a211d94491e7 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Fri, 14 Nov 2025 22:35:39 -0800 Subject: [PATCH 3/7] Add --add_seconds option and correct_times() to fix GPS Week Rollover Bug times in the log files. --- src/data/logs2netcdfs.py | 44 +++++++++++++++++++++++++++++++++++++--- src/data/process.py | 10 +++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/data/logs2netcdfs.py b/src/data/logs2netcdfs.py index 28da0359..38961815 100755 --- a/src/data/logs2netcdfs.py +++ b/src/data/logs2netcdfs.py @@ -552,8 +552,36 @@ def _create_variable( # noqa: PLR0913 self.logger.error("data seriously does not match shape") # noqa: TRY400 raise + def correct_times(self, log_data, add_seconds: int = 0): + """Correct timeTag variables in log_data by adding add_seconds + returning the corrected log_data""" + new_log_data = [] + for variable in log_data: + if variable.data_type == "timeTag": + self.logger.info( + "Adding to time values: %d seconds", + add_seconds, + ) + # Create a new log_record with corrected data + corrected_variable = log_record( + variable.data_type, + variable.short_name, + variable.long_name, + variable.units, + variable.instrument_name, + [tv + add_seconds for tv in variable.data], # ← New data list + ) + new_log_data.append(corrected_variable) + else: + # For non-timeTag variables + new_log_data.append(variable) # or create a copy if needed + return new_log_data + def write_variables(self, log_data, netcdf_filename): log_data = self._correct_dup_short_names(log_data) + if self.args.mission == "2025.316.02" and self.args.add_seconds: + # So far only this mission is known to suffer from GPS Week Rollover bug + log_data = self.correct_times(log_data, self.args.add_seconds) self.nc_file.createDimension(TIME, len(log_data[0].data)) for variable in log_data: self.logger.debug( @@ -682,6 +710,10 @@ def _process_log_file(self, log_filename, netcdf_filename, src_dir=None): self.nc_file.summary = SUMMARY_SOURCE.format(src_dir) if hasattr(self.args, "summary") and self.args.summary: self.nc_file.summary = self.args.summary + if self.args.add_seconds: + self.nc_file.summary += ( + f". Corrected timeTag variables by adding {self.args.add_seconds} seconds. " + ) monotonic = monotonic_increasing_time_indices(self.nc_file["time"][:]) if (~monotonic).any(): self.logger.info( @@ -930,6 +962,14 @@ def process_command_line(self): action="store", help="Directory for the vehicle's mission logs, e.g.: /Volumes/AUVCTD/missionlogs", ) + parser.add_argument( + # To use for mission 2025.316.02 which suffered from the GPS week rollover bug: + # 1024 * 7 * 24 * 3600 = 619315200 seconds to add to timeTag variables in the log_data + "--add_seconds", + type=int, + default=0, + help="Seconds to add to timeTag in log data", + ) parser.add_argument( "-v", "--verbose", @@ -965,10 +1005,8 @@ def process_command_line(self): else: raise argparse.ArgumentError( None, - "Must provide --src_dir with --auv_name & --mission", + "Must provide --vehicle_dir with --auv_name & --mission", ) - - auv_netcdf.download_process_logs(src_dir=Path()) elif auv_netcdf.args.start and auv_netcdf.args.end: auv_netcdf._deployments_between() else: diff --git a/src/data/process.py b/src/data/process.py index f0036d72..6f39e42c 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -174,6 +174,7 @@ def download_process(self, mission: str, src_dir: str) -> None: auv_netcdf.args.mission = mission auv_netcdf.args.use_portal = self.args.use_portal auv_netcdf.set_portal() + auv_netcdf.args.add_seconds = self.args.add_seconds auv_netcdf.args.verbose = self.args.verbose auv_netcdf.logger.setLevel(self._log_levels[self.args.verbose]) auv_netcdf.logger.addHandler(self.log_handler) @@ -793,6 +794,15 @@ def process_command_line(self): type=int, help="Number of core processors to use", ) + parser.add_argument( + "--add_seconds", + action="store", + type=int, + help=( + "Add seconds to time variables. Used to correct Dorado log files " + "saved with GPS Week Rollover Bug." + ), + ) parser.add_argument( "-v", "--verbose", From ac63b8ac996a8a0154b96b997eef31bdf8c0bcb9 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Fri, 14 Nov 2025 22:50:20 -0800 Subject: [PATCH 4/7] Fix tests - added ns.add_seconds to conftest.py. --- src/data/conftest.py | 1 + src/data/process.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/data/conftest.py b/src/data/conftest.py index d96f2290..54e464d9 100644 --- a/src/data/conftest.py +++ b/src/data/conftest.py @@ -102,6 +102,7 @@ def complete_dorado_processing(): ns.no_cleanup = True ns.skip_download_process = False ns.num_cores = 1 + ns.add_seconds = None ns.verbose = 1 proc.args = ns proc.process_missions(TEST_START_YEAR) diff --git a/src/data/process.py b/src/data/process.py index 6f39e42c..ed2c35e0 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -173,8 +173,8 @@ def download_process(self, mission: str, src_dir: str) -> None: auv_netcdf.args.auv_name = self.vehicle auv_netcdf.args.mission = mission auv_netcdf.args.use_portal = self.args.use_portal - auv_netcdf.set_portal() auv_netcdf.args.add_seconds = self.args.add_seconds + auv_netcdf.set_portal() auv_netcdf.args.verbose = self.args.verbose auv_netcdf.logger.setLevel(self._log_levels[self.args.verbose]) auv_netcdf.logger.addHandler(self.log_handler) From cd9107f73fe98d558a09c544ffc728dca4063021 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Sat, 15 Nov 2025 10:37:46 -0800 Subject: [PATCH 5/7] Add 'ns.add_seconds = None' to the i2map test too. --- src/data/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data/conftest.py b/src/data/conftest.py index 54e464d9..4f08da02 100644 --- a/src/data/conftest.py +++ b/src/data/conftest.py @@ -148,6 +148,7 @@ def complete_i2map_processing(): ns.skip_download_process = False ns.last_n_days = 0 ns.num_cores = 1 + ns.add_seconds = None ns.verbose = 1 proc.args = ns proc.process_missions(TEST_START_YEAR) From 1bc7a576a6163b9cf03ac4a8df24c2e197f3970b Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Sat, 15 Nov 2025 10:41:44 -0800 Subject: [PATCH 6/7] Update EXPECTED_SIZE_GITHUB --- src/data/test_process_dorado.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index 3eb1033c..bcb41ac1 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -31,7 +31,7 @@ def test_process_dorado(complete_dorado_processing): # but it will alert us if a code change unexpectedly changes the file size. # If code changes are expected to change the file size then we should # update the expected size here. - EXPECTED_SIZE_GITHUB = 621298 + EXPECTED_SIZE_GITHUB = 621286 EXPECTED_SIZE_ACT = 621298 EXPECTED_SIZE_LOCAL = 621286 if str(proc.args.base_path).startswith("/home/runner"): From 034499799c2aef66b097f074bf8c31c778c781fc Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Sat, 15 Nov 2025 10:45:19 -0800 Subject: [PATCH 7/7] Update values for CI. --- src/data/test_process_dorado.py | 2 +- src/data/test_process_i2map.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index bcb41ac1..90ec047b 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -50,7 +50,7 @@ def test_process_dorado(complete_dorado_processing): check_md5 = True if check_md5: # Check that the MD5 hash has not changed - EXPECTED_MD5_GITHUB = "6550bb8ed5919f21413f30dfffdcf116" + EXPECTED_MD5_GITHUB = "9f3f9e2e5abed08692ddb233dec0d0ac" EXPECTED_MD5_ACT = "bdb9473e5dedb694618f518b8cf0ca1e" EXPECTED_MD5_LOCAL = "6ecb2229b00835055619e982fe9d5023" if str(proc.args.base_path).startswith("/home/runner"): diff --git a/src/data/test_process_i2map.py b/src/data/test_process_i2map.py index cbd2c2c3..e2f6cb05 100644 --- a/src/data/test_process_i2map.py +++ b/src/data/test_process_i2map.py @@ -30,7 +30,7 @@ def test_process_i2map(complete_i2map_processing): # but it will alert us if a code change unexpectedly changes the file size. # If code changes are expected to change the file size then we should # update the expected size here. - EXPECTED_SIZE_GITHUB = 58839 + EXPECTED_SIZE_GITHUB = 58832 EXPECTED_SIZE_ACT = 58816 EXPECTED_SIZE_LOCAL = 58884 if str(proc.args.base_path).startswith("/home/runner"):