From 01d4dd61ddb8ef955ee0c2188f22da4c4980254f Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 12 Jun 2025 15:33:13 -0700 Subject: [PATCH 1/4] Correct typos in comment. --- src/data/test_process_dorado.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index 16ca9bcf..875a1a0f 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -13,7 +13,7 @@ def test_process_dorado(complete_dorado_processing): """Test that Dorado processing runs without error""" # complete_processing ia a fixture from the conftest.py module; # it is automatically loaded by pytest. We need this complete_processing - # to run withut error to trust that tthe production processing is working. + # to run without error to trust that the production processing is working. proc = complete_dorado_processing # Check that the _1S.nc file was created and is the correct size From 40ab5e253c3c1775b71566c3b091aa4703bfb0ea Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 12 Jun 2025 17:13:07 -0700 Subject: [PATCH 2/4] Improve setting path for Dorado389 bootstrapping and processing. One is not local and the other is local. --- src/data/process.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/data/process.py b/src/data/process.py index 32e9dbf0..f0036d72 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -134,10 +134,13 @@ def get_mission_dir(self, mission: str) -> str: self.logger.error("%s does not exist.", self.vehicle_dir) self.logger.info("Is %s mounted?", self.mount_dir) sys.exit(1) - if self.vehicle.lower() == "dorado": - year = mission.split(".")[0] - yearyd = "".join(mission.split(".")[:2]) - path = Path(self.vehicle_dir, year, yearyd, mission) + if self.vehicle.lower() == "dorado" or self.vehicle == "Dorado389": + if self.args.local: + path = Path(self.vehicle_dir, mission) + else: + year = mission.split(".")[0] + yearyd = "".join(mission.split(".")[:2]) + path = Path(self.vehicle_dir, year, yearyd, mission) elif self.vehicle.lower() == "i2map": year = int(mission.split(".")[0]) # Could construct the YYYY/MM/YYYYMMDD path on M3/Master From e9201b58eac1b519959fd99826dafe54ba32130e Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 12 Jun 2025 17:13:30 -0700 Subject: [PATCH 3/4] Remove warning about deprecated np.NaN. --- src/data/resample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/resample.py b/src/data/resample.py index be0dc2f5..c18e1705 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -1016,7 +1016,7 @@ def resample_variable( # noqa: PLR0913 instrs_to_pad[instr], ) dt_index = pd.date_range(mission_start, mission_end, freq=freq.lower()) - self.df_r[variable] = pd.Series(np.NaN, index=dt_index) + self.df_r[variable] = pd.Series(np.nan, index=dt_index) instr_data = ( self.df_o[f"{variable}_mf"] .shift(0.5, freq=freq.lower()) From de97f686cf9bf42a350b25432f15f87b49378997 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 12 Jun 2025 17:15:42 -0700 Subject: [PATCH 4/4] Display bootstrap_mission message if pytest fails. Also, a new process_Dorado389.py script is added for creating the local data requited for testing. --- .vscode/launch.json | 12 ++++++++++-- src/data/conftest.py | 10 ++++++++++ src/data/process_Dorado389.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100755 src/data/process_Dorado389.py diff --git a/.vscode/launch.json b/.vscode/launch.json index 10a857e9..a61ba927 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -235,7 +235,7 @@ //"args": ["-v", "1", "--mission", "2007.344.00", "--use_portal", "--clobber", "--noinput"] //"args": ["-v", "1", "--mission", "2009.055.05", "--clobber", "--noinput"] //"args": ["-v", "1", "--mission", "2010.172.01", "--local", "--clobber", "--noinput"] - //"args": ["-v", "1", "--start_year", "2010", "--end_year", "2010", "--start_yd", "81", "--end_yd", "84", "--clobber", "--noinput"] + "args": ["-v", "1", "--start_year", "2010", "--end_year", "2010", "--start_yd", "81", "--end_yd", "84", "--no_cleanup", "--num_cores", "1", "--noinput"] //"args": ["-v", "1", "--mission", "2010.286.02", "--clobber", "--noinput"] //"args": ["-v", "1", "--start_year", "2010", "--end_year", "2010", "--start_yd", "286", "--local", "--clobber", "--noinput"] //"args": ["-v", "1", "--start_year", "2004", "--end_year", "2004", "--start_yd", "167", "--clobber", "--noinput"] @@ -262,7 +262,7 @@ //"args": ["-v", "1", "--noinput", "--start_year", "2016", "--end_year", "2016", "--start_yd", "270", "--end_yd", "270", "--num_cores", "1", "--create_products", "--archive", "--archive_only_products"] //"args": ["-v", "1", "--noinput", "--num_cores", "1", "--mission", "2023.285.01"] //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2018.079.00"] - "args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2024.317.01", "--clobber"] + //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2024.317.01", "--clobber"] //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2016.090.01", "--resample", "--archive", "--flash_threshold", "3E10"] //"args": ["-v", "1", "--noinput", "--num_cores", "8"] //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2011.256.02", "--clobber"] @@ -271,5 +271,13 @@ //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2008.010.10"] }, + { + "name": "process_Dorado389", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/src/data/process_Dorado389.py", + "console": "integratedTerminal", + "args": ["-v", "1", "--noinput", "--no_cleanup", "--download", "--mission", "2011.256.02"] + }, ] } diff --git a/src/data/conftest.py b/src/data/conftest.py index b7a80546..d96f2290 100644 --- a/src/data/conftest.py +++ b/src/data/conftest.py @@ -11,6 +11,14 @@ from process import Processor from resample import FLASH_THRESHOLD, FREQ, MF_WIDTH +bootstrap_mission = """The working directory on a development machine must be bootstrapped +with some mission data. Process the mission used for testing with: + +uv run src/data/process_Dorado389.py --no_cleanup --download --mission 2011.256.02 -v + +This uses the legacy name "Dorado389" for the vehicle - the new name is "dorado". +""" + TEST_VEHICLE = "Dorado389" TEST_MISSION = "2011.256.02" # http://stoqs.mbari.org/p/DmHOaxI # Set TEST_VEHICLE_DIR to local path for testing @@ -32,6 +40,8 @@ @pytest.fixture(scope="session", autouse=False) def mission_data(): + if not Path(TEST_VEHICLE_DIR).exists(): + pytest.fail(f"\n\n{bootstrap_mission}\n") """Load a short recent mission to have some real data to work with""" cal_netcdf = Calibrate_NetCDF() ns = Namespace() diff --git a/src/data/process_Dorado389.py b/src/data/process_Dorado389.py new file mode 100755 index 00000000..990494f4 --- /dev/null +++ b/src/data/process_Dorado389.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +""" +This script is nearly identical to process_dorado.py, but it is used for +bootstrapping the testing infrastructure. It is used to process a short mission +using the legacy vehicle name "Dorado389" so that it's kept separate from the +new production name "dorado". +""" + +__author__ = "Mike McCann" +__copyright__ = "Copyright 2022, Monterey Bay Aquarium Research Institute" + +from process import Processor + + +class DoradoProcessor(Processor): + pass + + +if __name__ == "__main__": + VEHICLE = "Dorado389" + VEHICLE_DIR = "/Volumes/AUVCTD/missionlogs" + CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" + MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD" + START_YEAR = 2011 + + proc = DoradoProcessor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc.process_command_line() + proc.process_missions(START_YEAR)