From 1e605a8a83c3fd9a1e91194ad49437043d14d6bf Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 19 Nov 2025 14:46:49 -0800 Subject: [PATCH 1/7] Replace "vehicle" with "auv_name" so that auv_name is used consistently. --- src/data/process.py | 58 +++++++++++++++++------------------ src/data/process_Dorado389.py | 4 +-- src/data/process_dorado.py | 4 +-- src/data/process_i2map.py | 4 +-- src/data/process_lrauv.py | 4 +-- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/data/process.py b/src/data/process.py index 949b366f..e059137f 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -132,12 +132,12 @@ class Processor: logger.addHandler(_handler) _log_levels = (logging.WARN, logging.INFO, logging.DEBUG) - def __init__(self, vehicle, vehicle_dir, mount_dir, calibration_dir) -> None: + def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir) -> None: # Variables to be set by subclasses, e.g.: - # vehicle = "i2map" + # auv_name = "i2map" # vehicle_dir = "/Volumes/M3/master/i2MAP" # mount_dir = "smb://thalassa.shore.mbari.org/M3" - self.vehicle = vehicle + self.auv_name = auv_name self.vehicle_dir = vehicle_dir self.mount_dir = mount_dir self.calibration_dir = calibration_dir @@ -187,14 +187,14 @@ def get_mission_dir(self, mission: str) -> str: self.logger.error("%s does not exist.", self.vehicle_dir) self.logger.info("Is %s mounted?", self.mount_dir) sys.exit(1) - if self.vehicle.lower() == "dorado" or self.vehicle == "Dorado389": + if self.auv_name.lower() == "dorado" or self.auv_name == "Dorado389": if self.args.local: path = Path(self.vehicle_dir, mission) else: year = mission.split(".")[0] yearyd = "".join(mission.split(".")[:2]) path = Path(self.vehicle_dir, year, yearyd, mission) - elif self.vehicle.lower() == "i2map": + elif self.auv_name.lower() == "i2map": year = int(mission.split(".")[0]) # Could construct the YYYY/MM/YYYYMMDD path on M3/Master # but use the mission_list() method to find the mission dir instead @@ -205,8 +205,8 @@ def get_mission_dir(self, mission: str) -> str: self.logger.error("Cannot find %s in %s", mission, self.vehicle_dir) error_message = f"Cannot find {mission} in {self.vehicle_dir}" raise FileNotFoundError(error_message) - elif self.vehicle == "Dorado389": - # The Dorado389 vehicle is a special case used for testing locally and in CI + elif self.auv_name == "Dorado389": + # The Dorado389 auv_name is a special case used for testing locally and in CI path = self.vehicle_dir if not Path(path).exists(): self.logger.error("%s does not exist.", path) @@ -223,7 +223,7 @@ def download_process(self, mission: str, src_dir: str) -> None: auv_netcdf.args.noinput = self.args.noinput auv_netcdf.args.clobber = self.args.clobber auv_netcdf.args.noreprocess = self.args.noreprocess - auv_netcdf.args.auv_name = self.vehicle + auv_netcdf.args.auv_name = self.auv_name auv_netcdf.args.mission = mission auv_netcdf.args.use_portal = self.args.use_portal auv_netcdf.args.add_seconds = self.args.add_seconds @@ -238,7 +238,7 @@ def download_process(self, mission: str, src_dir: str) -> None: # Run lopcToNetCDF.py - mimic log message from logs2netcdfs.py lopc_bin = Path( self.args.base_path, - self.vehicle, + self.auv_name, MISSIONLOGS, mission, "lopc.bin", @@ -246,7 +246,7 @@ def download_process(self, mission: str, src_dir: str) -> None: try: file_size = Path(lopc_bin).stat().st_size except FileNotFoundError: - if "lopc" in EXPECTED_SENSORS[self.vehicle]: + if "lopc" in EXPECTED_SENSORS[self.auv_name]: self.logger.warning("No lopc.bin file for %s", mission) return self.logger.info("Processing file %s (%d bytes)", lopc_bin, file_size) @@ -255,7 +255,7 @@ def download_process(self, mission: str, src_dir: str) -> None: lopc_processor.args.bin_fileName = lopc_bin lopc_processor.args.netCDF_fileName = os.path.join( # noqa: PTH118 This is an arg, keep it a string self.args.base_path, - self.vehicle, + self.auv_name, MISSIONNETCDFS, mission, "lopc.nc", @@ -286,7 +286,7 @@ def calibrate(self, mission: str) -> None: cal_netcdf.args.noinput = self.args.noinput cal_netcdf.args.clobber = self.args.clobber cal_netcdf.args.noreprocess = self.args.noreprocess - cal_netcdf.args.auv_name = self.vehicle + cal_netcdf.args.auv_name = self.auv_name cal_netcdf.args.mission = mission cal_netcdf.args.plot = None cal_netcdf.calibration_dir = self.calibration_dir @@ -306,7 +306,7 @@ def align(self, mission: str = "", log_file: str = "") -> None: align_netcdf = Align_NetCDF() align_netcdf.args = argparse.Namespace() align_netcdf.args.base_path = self.args.base_path - align_netcdf.args.auv_name = self.vehicle + align_netcdf.args.auv_name = self.auv_name align_netcdf.args.mission = mission align_netcdf.args.log_file = self.args.log_file align_netcdf.args.plot = None @@ -320,7 +320,7 @@ def align(self, mission: str = "", log_file: str = "") -> None: align_netcdf.write_combined_netcdf(netcdf_dir, log_file=log_file) else: netcdf_dir = align_netcdf.process_cal() - align_netcdf.write_combined_netcdf(netcdf_dir, vehicle=self.vehicle) + align_netcdf.write_combined_netcdf(netcdf_dir, vehicle=self.auv_name) except (FileNotFoundError, EOFError) as e: align_netcdf.logger.error("%s %s", mission, e) # noqa: TRY400 error_message = f"{mission} {e}" @@ -332,7 +332,7 @@ def resample(self, mission: str = "") -> None: self.logger.info("Resampling steps for %s", mission) resamp = Resampler() resamp.args = argparse.Namespace() - resamp.args.auv_name = self.vehicle + resamp.args.auv_name = self.auv_name resamp.args.mission = mission resamp.args.log_file = self.args.log_file resamp.args.plot = None @@ -396,7 +396,7 @@ def archive( If log_file is provided, archive the processed data for LRAUV class vehicles.""" arch = Archiver(add_logger_handlers) arch.args = argparse.Namespace() - arch.args.auv_name = self.vehicle + arch.args.auv_name = self.auv_name arch.mount_dir = self.mount_dir arch.args.mission = mission arch.commandline = self.commandline @@ -441,7 +441,7 @@ def create_products(self, mission: str) -> None: cp = CreateProducts() cp.args = argparse.Namespace() cp.args.base_path = self.args.base_path - cp.args.auv_name = self.vehicle + cp.args.auv_name = self.auv_name cp.args.mission = mission cp.args.local = self.args.local cp.args.start_esecs = None @@ -459,7 +459,7 @@ def email(self, mission: str) -> None: self.logger.info("Sending notification email for %s", mission) email = Emailer() email.args = argparse.Namespace() - email.args.auv_name = self.vehicle + email.args.auv_name = self.auv_name email.args.mission = mission email.commandline = self.commandline email.args.clobber = self.args.clobber @@ -495,10 +495,10 @@ def cleanup(self, mission: str = None, log_file: str = None) -> None: ) try: shutil.rmtree( - Path(self.args.base_path, self.vehicle, MISSIONLOGS, mission), + Path(self.args.base_path, self.auv_name, MISSIONLOGS, mission), ) shutil.rmtree( - Path(self.args.base_path, self.vehicle, MISSIONNETCDFS, mission), + Path(self.args.base_path, self.auv_name, MISSIONNETCDFS, mission), ) self.logger.info("Done removing %s work files", mission) except FileNotFoundError as e: @@ -524,7 +524,7 @@ def cleanup(self, mission: str = None, log_file: str = None) -> None: def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C901, PLR0912, PLR0915 netcdfs_dir = Path( self.args.base_path, - self.vehicle, + self.auv_name, MISSIONNETCDFS, mission, ) @@ -535,7 +535,7 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 self.cleanup(mission) Path(netcdfs_dir).mkdir(parents=True, exist_ok=True) self.log_handler = logging.FileHandler( - Path(netcdfs_dir, f"{self.vehicle}_{mission}_{LOG_NAME}"), + Path(netcdfs_dir, f"{self.auv_name}_{mission}_{LOG_NAME}"), mode="w+", ) self.log_handler.setLevel(self._log_levels[self.args.verbose]) @@ -547,12 +547,12 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 self.logger.info("commandline = %s", self.commandline) try: program = "" - if self.vehicle.lower() == "dorado": + if self.auv_name.lower() == "dorado": program = dorado_info[mission]["program"] self.logger.info( 'dorado_info[mission]["comment"] = %s', dorado_info[mission]["comment"] ) - elif self.vehicle.lower() == "i2map": + elif self.auv_name.lower() == "i2map": program = "i2map" if program == TEST: error_message = ( @@ -660,10 +660,10 @@ def process_mission_exception_wrapper( if hasattr(self, "log_handler"): # If no log_handler then process_mission() failed, likely due to missing mount # Always archive the mission, especially the processing.log file - if self.vehicle == "Dorado389" and mission == "2011.256.02": + if self.auv_name == "Dorado389" and mission == "2011.256.02": self.logger.info( "Not archiving %s %s as it's likely CI testing", - self.vehicle, + self.auv_name, mission, ) if self.args.download_process: @@ -806,7 +806,7 @@ def process_log_files(self) -> None: if self.args.log_file: # log_file is string like: # brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4 - self.vehicle = self.args.log_file.split("/")[0].lower() + self.auv_name = self.args.log_file.split("/")[0].lower() self.process_log_file(self.args.log_file) def process_command_line(self): @@ -1028,12 +1028,12 @@ def process_command_line(self): if __name__ == "__main__": - VEHICLE = "i2map" + AUV_NAME = "i2map" VEHICLE_DIR = "/Volumes/M3/master/i2MAP" CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://thalassa.shore.mbari.org/M3" # Initialize for i2MAP processing, meant to be subclassed for other vehicles - proc = Processor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc = Processor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) proc.process_command_line() proc.process_missions() diff --git a/src/data/process_Dorado389.py b/src/data/process_Dorado389.py index 990494f4..eac7caf6 100755 --- a/src/data/process_Dorado389.py +++ b/src/data/process_Dorado389.py @@ -17,12 +17,12 @@ class DoradoProcessor(Processor): if __name__ == "__main__": - VEHICLE = "Dorado389" + AUV_NAME = "Dorado389" VEHICLE_DIR = "/Volumes/AUVCTD/missionlogs" CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD" START_YEAR = 2011 - proc = DoradoProcessor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) proc.process_command_line() proc.process_missions(START_YEAR) diff --git a/src/data/process_dorado.py b/src/data/process_dorado.py index aaee26db..a60c0e3b 100755 --- a/src/data/process_dorado.py +++ b/src/data/process_dorado.py @@ -30,12 +30,12 @@ class DoradoProcessor(Processor): if __name__ == "__main__": - VEHICLE = "dorado" + AUV_NAME = "dorado" VEHICLE_DIR = "/Volumes/AUVCTD/missionlogs" CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD" START_YEAR = 2003 - proc = DoradoProcessor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) proc.process_command_line() proc.process_missions(START_YEAR) diff --git a/src/data/process_i2map.py b/src/data/process_i2map.py index e2517558..c6ee2247 100755 --- a/src/data/process_i2map.py +++ b/src/data/process_i2map.py @@ -29,12 +29,12 @@ class I2mapProcessor(Processor): if __name__ == "__main__": - VEHICLE = "i2map" + AUV_NAME = "i2map" VEHICLE_DIR = "/Volumes/M3/master/i2MAP" CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://thalassa.shore.mbari.org/M3" START_YEAR = 2017 - proc = I2mapProcessor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc = I2mapProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) proc.process_command_line() proc.process_missions(START_YEAR) diff --git a/src/data/process_lrauv.py b/src/data/process_lrauv.py index 7a99f92b..b3db134e 100755 --- a/src/data/process_lrauv.py +++ b/src/data/process_lrauv.py @@ -30,7 +30,7 @@ class LRAUVProcessor(Processor): if __name__ == "__main__": - VEHICLE = "tethys" + AUV_NAME = "tethys" LRAUV_DIR = "/Volumes/LRAUV" # It's possible that we might need calibration files for some sensors # in the future, so point to a potential directory where they can be found. @@ -38,6 +38,6 @@ class LRAUVProcessor(Processor): MOUNT_DIR = "smb://atlas.shore.mbari.org/LRAUV" START_YEAR = 2012 - proc = LRAUVProcessor(VEHICLE, LRAUV_DIR, MOUNT_DIR, CALIBRATION_DIR) + proc = LRAUVProcessor(AUV_NAME, LRAUV_DIR, MOUNT_DIR, CALIBRATION_DIR) proc.process_command_line() proc.process_log_files() From a19a35124d4ab9e960c88be6e4019d0bc641d294 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 09:34:19 -0800 Subject: [PATCH 2/7] Factored all the options into _CONFIG_SCHEMA and use a config pattern for sharing. NGL, I used Claude/Sonnet 4 to help me do this. The pattern is mostly implemented in process.py and its subclasses. Changes made also in conftest.py to help decouple testing from routine code changes. --- src/data/conftest.py | 144 +++++++------- src/data/process.py | 351 ++++++++++++++++++++-------------- src/data/process_Dorado389.py | 8 +- src/data/process_dorado.py | 8 +- src/data/process_i2map.py | 8 +- src/data/process_lrauv.py | 9 +- 6 files changed, 303 insertions(+), 225 deletions(-) diff --git a/src/data/conftest.py b/src/data/conftest.py index fd181ce1..3486953f 100644 --- a/src/data/conftest.py +++ b/src/data/conftest.py @@ -11,6 +11,53 @@ from process import Processor from resample import FLASH_THRESHOLD, FREQ, MF_WIDTH + +def create_test_namespace(vehicle_overrides=None, processing_overrides=None): + """Create a standardized test namespace using Processor's CONFIG_SCHEMA. + + Args: + vehicle_overrides: Dict of vehicle-specific overrides (mission, auv_name, etc.) + processing_overrides: Dict of processing-specific overrides (verbose, clobber, etc.) + + Returns: + argparse.Namespace with all CONFIG_SCHEMA attributes properly set + """ + # Start with Processor's config schema defaults + config = dict(Processor._CONFIG_SCHEMA) + + # Apply common test defaults + test_defaults = { + "base_path": os.getenv("BASE_PATH", BASE_PATH), + "local": True, + "noinput": True, + "noreprocess": False, + "use_portal": False, + "freq": FREQ, + "mf_width": MF_WIDTH, + "flash_threshold": FLASH_THRESHOLD, + "clobber": False, + "no_cleanup": True, + "num_cores": 1, + "verbose": 1, + } + config.update(test_defaults) + + # Apply vehicle-specific overrides + if vehicle_overrides: + config.update(vehicle_overrides) + + # Apply processing-specific overrides + if processing_overrides: + config.update(processing_overrides) + + # Create namespace and set all attributes + ns = Namespace() + for key, value in config.items(): + setattr(ns, key, value) + + return ns + + bootstrap_mission = """The working directory on a development machine must be bootstrapped with some mission data. Process the mission used for testing with: @@ -72,40 +119,19 @@ def calibration(mission_data): @pytest.fixture(scope="session", autouse=False) def complete_dorado_processing(): """Load a short mission to have some real data to work with""" - proc = Processor(TEST_VEHICLE, TEST_VEHICLE_DIR, TEST_MOUNT_DIR, TEST_CALIBRATION_DIR) - ns = Namespace() - ns.base_path = os.getenv("BASE_PATH", BASE_PATH) - ns.auv_name = TEST_VEHICLE - ns.mission = TEST_MISSION - ns.start_year = TEST_START_YEAR - # There are several options that need to be set to run the full processing - ns.clobber = False - proc.commandline = "args set in conftest.py::complete_dorado_processing()" - ns.local = True - ns.noinput = True - ns.noreprocess = False - ns.use_portal = False - ns.freq = FREQ - ns.mf_width = MF_WIDTH - ns.flash_threshold = FLASH_THRESHOLD - # Set step flags to false to force all steps to run as the logic in - # process_mission() is not fully implemented. - ns.download_process = False - ns.calibrate = False - ns.align = False - ns.resample = False - ns.create_products = False - ns.archive = False - ns.archive_only_products = False - ns.email_to = None - ns.cleanup = False - ns.no_cleanup = True - ns.skip_download_process = False - ns.num_cores = 1 - ns.add_seconds = None - ns.log_file = None - ns.verbose = 1 - proc.args = ns + # Create namespace with vehicle-specific settings + vehicle_overrides = { + "auv_name": TEST_VEHICLE, + "mission": TEST_MISSION, + "start_year": TEST_START_YEAR, + } + + ns = create_test_namespace(vehicle_overrides=vehicle_overrides) + + # Create processor using new factory method + proc = Processor.from_args( + TEST_VEHICLE, TEST_VEHICLE_DIR, TEST_MOUNT_DIR, TEST_CALIBRATION_DIR, ns + ) proc.process_missions(TEST_START_YEAR) return proc @@ -113,45 +139,23 @@ def complete_dorado_processing(): @pytest.fixture(scope="session", autouse=False) def complete_i2map_processing(): """Load a short mission to have some real data to work with""" - proc = Processor( + # Create namespace with i2map-specific settings + vehicle_overrides = { + "auv_name": TEST_I2MAP_VEHICLE, + "mission": TEST_I2MAP_MISSION, + "start_year": TEST_I2MAP_START_YEAR, + "last_n_days": 0, # i2map-specific setting + } + + ns = create_test_namespace(vehicle_overrides=vehicle_overrides) + + # Create processor using new factory method + proc = Processor.from_args( TEST_I2MAP_VEHICLE, TEST_I2MAP_VEHICLE_DIR, TEST_I2MAP_MOUNT_DIR, TEST_I2MAP_CALIBRATION_DIR, + ns, ) - ns = Namespace() - ns.base_path = os.getenv("BASE_PATH", BASE_PATH) - ns.auv_name = TEST_I2MAP_VEHICLE - ns.mission = TEST_I2MAP_MISSION - ns.start_year = TEST_I2MAP_START_YEAR - # There are several options that need to be set to run the full processing - ns.clobber = False - proc.commandline = "args set in conftest.py::complete_i2map_processing()" - ns.local = True - ns.noinput = True - ns.noreprocess = False - ns.use_portal = False - ns.freq = FREQ - ns.mf_width = MF_WIDTH - ns.flash_threshold = FLASH_THRESHOLD - # Set step flags to false to force all steps to run as the logic in - # process_mission() is not fully implemented. - ns.download_process = False - ns.calibrate = False - ns.align = False - ns.resample = False - ns.create_products = False - ns.archive = False - ns.archive_only_products = False - ns.email_to = None - ns.cleanup = False - ns.no_cleanup = True - ns.skip_download_process = False - ns.last_n_days = 0 - ns.num_cores = 1 - ns.add_seconds = None - ns.log_file = None - ns.verbose = 1 - proc.args = ns - proc.process_missions(TEST_START_YEAR) + proc.process_missions(TEST_I2MAP_START_YEAR) return proc diff --git a/src/data/process.py b/src/data/process.py index e059137f..fde779ca 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -110,7 +110,7 @@ def wrapper(self, log_file: str): if hasattr(self, "log_handler"): # Cleanup and archiving logic self.archive(mission=None, log_file=log_file) - if not self.args.no_cleanup: + if not self.config.get("no_cleanup"): self.cleanup(log_file=log_file) self.logger.info( "log_file %s took %.1f seconds to process", log_file, time.time() - t_start @@ -132,7 +132,7 @@ class Processor: logger.addHandler(_handler) _log_levels = (logging.WARN, logging.INFO, logging.DEBUG) - def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir) -> None: + def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir, config=None) -> None: # noqa: PLR0913 # Variables to be set by subclasses, e.g.: # auv_name = "i2map" # vehicle_dir = "/Volumes/M3/master/i2MAP" @@ -141,6 +141,105 @@ def __init__(self, auv_name, vehicle_dir, mount_dir, calibration_dir) -> None: self.vehicle_dir = vehicle_dir self.mount_dir = mount_dir self.calibration_dir = calibration_dir + self.config = config or {} + + # Configuration schema with defaults - shared between from_args and common_config + _CONFIG_SCHEMA = { + # Core configuration + "base_path": BASE_PATH, + "local": False, + "noinput": False, + "clobber": False, + "noreprocess": False, + "use_portal": False, + "add_seconds": None, + "verbose": 0, + "freq": FREQ, + "mf_width": MF_WIDTH, + "flash_threshold": None, + "log_file": None, + # Processing control + "download_process": False, + "calibrate": False, + "align": False, + "resample": False, + "archive": False, + "create_products": False, + "email_to": None, + "cleanup": False, + "no_cleanup": False, + "skip_download_process": False, + "archive_only_products": False, + "num_cores": None, + # Filtering/processing params (only used in from_args, not common_config) + "start_year": None, + "end_year": None, + "start_yd": None, + "end_yd": None, + "last_n_days": None, + "mission": None, + } + + # Subset of config schema that should be passed to child processes + _CHILD_CONFIG_KEYS = { + "base_path", + "local", + "noinput", + "clobber", + "noreprocess", + "use_portal", + "add_seconds", + "verbose", + "freq", + "mf_width", + "flash_threshold", + "log_file", + "download_process", + "calibrate", + "align", + "resample", + "archive", + "create_products", + "email_to", + "cleanup", + "no_cleanup", + "skip_download_process", + "archive_only_products", + "num_cores", + } + + @property + def common_config(self): + """Get common configuration used by all child processes""" + return { + key: self.config.get(key, self._CONFIG_SCHEMA[key]) for key in self._CHILD_CONFIG_KEYS + } + + def _create_child_namespace(self, **overrides): + """Create args namespace for child processes with config overrides""" + config = {**self.common_config, **overrides} + + namespace = argparse.Namespace() + for key, value in config.items(): + setattr(namespace, key, value) + return namespace + + @classmethod + def from_args(cls, auv_name, vehicle_dir, mount_dir, calibration_dir, args): # noqa: PLR0913 + """Factory method to create Processor from argparse namespace""" + config = {} + for key, default_value in cls._CONFIG_SCHEMA.items(): + # Handle special cases for args that might not exist or have different names + if key == "add_seconds": + config[key] = getattr(args, "add_seconds", default_value) + else: + config[key] = getattr(args, key, default_value) + + instance = cls(auv_name, vehicle_dir, mount_dir, calibration_dir, config) + instance.args = args # Keep reference for compatibility + instance.commandline = " ".join(sys.argv) # Set commandline attribute + instance.logger.setLevel(instance._log_levels[args.verbose]) # Set logger level + return instance def mission_list(self, start_year: int, end_year: int) -> dict: """Return a dictionary of source directories keyed by mission name.""" @@ -156,11 +255,11 @@ def mission_list(self, start_year: int, end_year: int) -> dict: else: find_cmd = f'find {safe_vehicle_dir} -regex "{REGEX}"' self.logger.debug("Executing %s", find_cmd) - if self.args.last_n_days: + if self.config.get("last_n_days"): self.logger.info( - "Will be looking back %d days for new missions...", self.args.last_n_days + "Will be looking back %d days for new missions...", self.config["last_n_days"] ) - find_cmd += f" -mtime -{self.args.last_n_days}" + find_cmd += f" -mtime -{self.config['last_n_days']}" self.logger.info("Finding missions from %s to %s", start_year, end_year) # Can be time consuming - use to discover missions lines = subprocess.getoutput(f"{find_cmd} | sort").split("\n") # noqa: S605 @@ -188,7 +287,7 @@ def get_mission_dir(self, mission: str) -> str: self.logger.info("Is %s mounted?", self.mount_dir) sys.exit(1) if self.auv_name.lower() == "dorado" or self.auv_name == "Dorado389": - if self.args.local: + if self.config.get("local"): path = Path(self.vehicle_dir, mission) else: year = mission.split(".")[0] @@ -217,19 +316,9 @@ def get_mission_dir(self, mission: str) -> str: def download_process(self, mission: str, src_dir: str) -> None: self.logger.info("Download and processing steps for %s", mission) auv_netcdf = AUV_NetCDF() - auv_netcdf.args = argparse.Namespace() - auv_netcdf.args.base_path = self.args.base_path - auv_netcdf.args.local = self.args.local - auv_netcdf.args.noinput = self.args.noinput - auv_netcdf.args.clobber = self.args.clobber - auv_netcdf.args.noreprocess = self.args.noreprocess - auv_netcdf.args.auv_name = self.auv_name - auv_netcdf.args.mission = mission - auv_netcdf.args.use_portal = self.args.use_portal - auv_netcdf.args.add_seconds = self.args.add_seconds + auv_netcdf.args = self._create_child_namespace(auv_name=self.auv_name, mission=mission) auv_netcdf.set_portal() - auv_netcdf.args.verbose = self.args.verbose - auv_netcdf.logger.setLevel(self._log_levels[self.args.verbose]) + auv_netcdf.logger.setLevel(self._log_levels[self.config["verbose"]]) auv_netcdf.logger.addHandler(self.log_handler) auv_netcdf.commandline = self.commandline auv_netcdf.download_process_logs(src_dir=src_dir) @@ -237,7 +326,7 @@ def download_process(self, mission: str, src_dir: str) -> None: # Run lopcToNetCDF.py - mimic log message from logs2netcdfs.py lopc_bin = Path( - self.args.base_path, + self.config["base_path"], self.auv_name, MISSIONLOGS, mission, @@ -251,25 +340,24 @@ def download_process(self, mission: str, src_dir: str) -> None: return self.logger.info("Processing file %s (%d bytes)", lopc_bin, file_size) lopc_processor = LOPC_Processor() - lopc_processor.args = argparse.Namespace() - lopc_processor.args.bin_fileName = lopc_bin - lopc_processor.args.netCDF_fileName = os.path.join( # noqa: PTH118 This is an arg, keep it a string - self.args.base_path, - self.auv_name, - MISSIONNETCDFS, - mission, - "lopc.nc", + lopc_processor.args = self._create_child_namespace( + bin_fileName=lopc_bin, + netCDF_fileName=os.path.join( # noqa: PTH118 This is an arg, keep it a string + self.config["base_path"], + self.auv_name, + MISSIONNETCDFS, + mission, + "lopc.nc", + ), + text_fileName="", + trans_AIcrit=0.4, + LargeCopepod_AIcrit=0.6, + LargeCopepod_ESDmin=1100.0, + LargeCopepod_ESDmax=1700.0, + debugLevel=0, + force=self.config["clobber"], ) - lopc_processor.args.text_fileName = "" - lopc_processor.args.trans_AIcrit = 0.4 - lopc_processor.args.LargeCopepod_AIcrit = 0.6 - lopc_processor.args.LargeCopepod_ESDmin = 1100.0 - lopc_processor.args.LargeCopepod_ESDmax = 1700.0 - lopc_processor.args.verbose = self.args.verbose - lopc_processor.args.debugLevel = 0 - lopc_processor.args.force = self.args.clobber - lopc_processor.args.noinput = self.args.noinput - lopc_processor.logger.setLevel(self._log_levels[self.args.verbose]) + lopc_processor.logger.setLevel(self._log_levels[self.config["verbose"]]) lopc_processor.logger.addHandler(self.log_handler) try: lopc_processor.main() @@ -280,18 +368,11 @@ def download_process(self, mission: str, src_dir: str) -> None: def calibrate(self, mission: str) -> None: self.logger.info("Calibration steps for %s", mission) cal_netcdf = Calibrate_NetCDF() - cal_netcdf.args = argparse.Namespace() - cal_netcdf.args.base_path = self.args.base_path - cal_netcdf.args.local = self.args.local - cal_netcdf.args.noinput = self.args.noinput - cal_netcdf.args.clobber = self.args.clobber - cal_netcdf.args.noreprocess = self.args.noreprocess - cal_netcdf.args.auv_name = self.auv_name - cal_netcdf.args.mission = mission - cal_netcdf.args.plot = None + cal_netcdf.args = self._create_child_namespace( + auv_name=self.auv_name, mission=mission, plot=None + ) cal_netcdf.calibration_dir = self.calibration_dir - cal_netcdf.args.verbose = self.args.verbose - cal_netcdf.logger.setLevel(self._log_levels[self.args.verbose]) + cal_netcdf.logger.setLevel(self._log_levels[self.config["verbose"]]) cal_netcdf.logger.addHandler(self.log_handler) cal_netcdf.commandline = self.commandline try: @@ -304,14 +385,10 @@ def calibrate(self, mission: str) -> None: def align(self, mission: str = "", log_file: str = "") -> None: self.logger.info("Alignment steps for %s", mission) align_netcdf = Align_NetCDF() - align_netcdf.args = argparse.Namespace() - align_netcdf.args.base_path = self.args.base_path - align_netcdf.args.auv_name = self.auv_name - align_netcdf.args.mission = mission - align_netcdf.args.log_file = self.args.log_file - align_netcdf.args.plot = None - align_netcdf.args.verbose = self.args.verbose - align_netcdf.logger.setLevel(self._log_levels[self.args.verbose]) + align_netcdf.args = self._create_child_namespace( + auv_name=self.auv_name, mission=mission, plot=None + ) + align_netcdf.logger.setLevel(self._log_levels[self.config["verbose"]]) align_netcdf.logger.addHandler(self.log_handler) align_netcdf.commandline = self.commandline try: @@ -331,17 +408,11 @@ def align(self, mission: str = "", log_file: str = "") -> None: def resample(self, mission: str = "") -> None: self.logger.info("Resampling steps for %s", mission) resamp = Resampler() - resamp.args = argparse.Namespace() - resamp.args.auv_name = self.auv_name - resamp.args.mission = mission - resamp.args.log_file = self.args.log_file - resamp.args.plot = None - resamp.args.freq = self.args.freq - resamp.args.mf_width = self.args.mf_width - resamp.args.flash_threshold = self.args.flash_threshold + resamp.args = self._create_child_namespace( + auv_name=self.auv_name, mission=mission, plot=None + ) resamp.commandline = self.commandline - resamp.args.verbose = self.args.verbose - resamp.logger.setLevel(self._log_levels[self.args.verbose]) + resamp.logger.setLevel(self._log_levels[self.config["verbose"]]) resamp.logger.addHandler(self.log_handler) file_name = f"{resamp.args.auv_name}_{resamp.args.mission}_align.nc" if resamp.args.log_file: @@ -349,16 +420,16 @@ def resample(self, mission: str = "") -> None: nc_file = Path(netcdfs_dir, f"{Path(resamp.args.log_file).stem}_align.nc") else: nc_file = Path( - self.args.base_path, + self.config["base_path"], resamp.args.auv_name, MISSIONNETCDFS, resamp.args.mission, file_name, ) - if self.args.flash_threshold and self.args.resample: + if self.config["flash_threshold"] and self.config["resample"]: self.logger.info( "Executing only resample step to produce netCDF file with flash_threshold = %s", - f"{self.args.flash_threshold:.0e}", + f"{self.config['flash_threshold']:.0e}", ) dap_file_str = os.path.join( # noqa: PTH118 AUVCTD_OPENDAP_BASE.replace("opendap/", ""), @@ -395,18 +466,10 @@ def archive( If mission is provided, archive the processed data for Dorado class vehicles. If log_file is provided, archive the processed data for LRAUV class vehicles.""" arch = Archiver(add_logger_handlers) - arch.args = argparse.Namespace() - arch.args.auv_name = self.auv_name + arch.args = self._create_child_namespace(auv_name=self.auv_name, mission=mission) arch.mount_dir = self.mount_dir - arch.args.mission = mission arch.commandline = self.commandline - arch.args.create_products = self.args.create_products - arch.args.archive_only_products = self.args.archive_only_products - arch.args.clobber = self.args.clobber - arch.args.resample = self.args.resample - arch.args.flash_threshold = self.args.flash_threshold - arch.args.verbose = self.args.verbose - arch.logger.setLevel(self._log_levels[self.args.verbose]) + arch.logger.setLevel(self._log_levels[self.config["verbose"]]) if add_logger_handlers: arch.logger.addHandler(self.log_handler) if mission: @@ -428,25 +491,21 @@ def archive( arch.args.mission, ) else: - arch.copy_to_AUVTCD(nc_file_base, self.args.freq) + arch.copy_to_AUVTCD(nc_file_base, self.config["freq"]) elif log_file: # LRAUV class vehicle archiving self.logger.info("Archiving steps for %s", log_file) - arch.copy_to_LRAUV(log_file, freq=self.args.freq) + arch.copy_to_LRAUV(log_file, freq=self.config["freq"]) else: arch.logger.error("Either mission or log_file must be provided for archiving.") arch.logger.removeHandler(self.log_handler) def create_products(self, mission: str) -> None: cp = CreateProducts() - cp.args = argparse.Namespace() - cp.args.base_path = self.args.base_path - cp.args.auv_name = self.auv_name - cp.args.mission = mission - cp.args.local = self.args.local - cp.args.start_esecs = None - cp.args.verbose = self.args.verbose - cp.logger.setLevel(self._log_levels[self.args.verbose]) + cp.args = self._create_child_namespace( + auv_name=self.auv_name, mission=mission, start_esecs=None + ) + cp.logger.setLevel(self._log_levels[self.config["verbose"]]) cp.logger.addHandler(self.log_handler) # cp.plot_biolume() @@ -458,13 +517,9 @@ def create_products(self, mission: str) -> None: def email(self, mission: str) -> None: self.logger.info("Sending notification email for %s", mission) email = Emailer() - email.args = argparse.Namespace() - email.args.auv_name = self.auv_name - email.args.mission = mission + email.args = self._create_child_namespace(auv_name=self.auv_name, mission=mission) email.commandline = self.commandline - email.args.clobber = self.args.clobber - email.args.verbose = self.args.verbose - email.logger.setLevel(self._log_levels[self.args.verbose]) + email.logger.setLevel(self._log_levels[self.config["verbose"]]) email.logger.addHandler(self.log_handler) def _remove_empty_parents(self, path: Path, stop_at: Path) -> None: @@ -495,10 +550,10 @@ def cleanup(self, mission: str = None, log_file: str = None) -> None: ) try: shutil.rmtree( - Path(self.args.base_path, self.auv_name, MISSIONLOGS, mission), + Path(self.config["base_path"], self.auv_name, MISSIONLOGS, mission), ) shutil.rmtree( - Path(self.args.base_path, self.auv_name, MISSIONNETCDFS, mission), + Path(self.config["base_path"], self.auv_name, MISSIONNETCDFS, mission), ) self.logger.info("Done removing %s work files", mission) except FileNotFoundError as e: @@ -523,13 +578,13 @@ def cleanup(self, mission: str = None, log_file: str = None) -> None: def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C901, PLR0912, PLR0915 netcdfs_dir = Path( - self.args.base_path, + self.config["base_path"], self.auv_name, MISSIONNETCDFS, mission, ) - if self.args.clobber and ( - self.args.noinput + if self.config["clobber"] and ( + self.config["noinput"] or input("Do you want to remove all work files? [y/N] ").lower() == "y" ): self.cleanup(mission) @@ -538,7 +593,7 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 Path(netcdfs_dir, f"{self.auv_name}_{mission}_{LOG_NAME}"), mode="w+", ) - self.log_handler.setLevel(self._log_levels[self.args.verbose]) + self.log_handler.setLevel(self._log_levels[self.config["verbose"]]) self.log_handler.setFormatter(AUV_NetCDF._formatter) self.logger.info( "=====================================================================================================================", @@ -574,30 +629,30 @@ def process_mission(self, mission: str, src_dir: str = "") -> None: # noqa: C90 except KeyError: error_message = f"{mission} not in dorado_info" raise MissingDoradoInfo(error_message) from None - if self.args.download_process: + if self.config["download_process"]: self.download_process(mission, src_dir) - elif self.args.calibrate: + elif self.config["calibrate"]: self.calibrate(mission) - elif self.args.align: + elif self.config["align"]: self.align(mission) - elif self.args.resample: + elif self.config["resample"]: self.resample(mission) - elif self.args.resample and self.args.archive: + elif self.config["resample"] and self.config["archive"]: self.resample(mission) self.archive(mission, add_logger_handlers=False) - elif self.args.create_products and self.args.archive: + elif self.config["create_products"] and self.config["archive"]: self.create_products(mission) self.archive(mission, add_logger_handlers=False) - elif self.args.create_products: + elif self.config["create_products"]: self.create_products(mission) - elif self.args.archive: + elif self.config["archive"]: self.archive(mission) - elif self.args.email_to: + elif self.config["email_to"]: self.email(mission) - elif self.args.cleanup: + elif self.config["cleanup"]: self.cleanup(mission) else: - if not self.args.skip_download_process: + if not self.config["skip_download_process"]: self.download_process(mission, src_dir) self.calibrate(mission) self.align(mission) @@ -621,12 +676,12 @@ def process_mission_job(self, mission: str, src_dir: str = "") -> None: except (TestMission, FailedMission) as e: self.logger.info(str(e)) finally: - if self.args.download_process: + if self.config["download_process"]: self.logger.info("Not archiving %s as --download_process is set", mission) else: # Still need to archive the mission, especially the processing.log file self.archive(mission) - if not self.args.no_cleanup: + if not self.config["no_cleanup"]: self.cleanup(mission) self.logger.info( "Mission %s took %.1f seconds to process", @@ -666,11 +721,11 @@ def process_mission_exception_wrapper( self.auv_name, mission, ) - if self.args.download_process: + if self.config["download_process"]: self.logger.info("Not archiving %s as --download_process is set", mission) else: self.archive(mission) - if not self.args.no_cleanup: + if not self.config["no_cleanup"]: self.cleanup(mission) self.logger.info( "Mission %s took %.1f seconds to process", @@ -679,34 +734,34 @@ def process_mission_exception_wrapper( ) self.logger.removeHandler(self.log_handler) - def process_missions(self, start_year: int) -> None: - if not self.args.start_year: - self.args.start_year = start_year - if self.args.mission: + def process_missions(self, start_year: int = None) -> None: + if not self.config.get("start_year"): + self.config["start_year"] = start_year + if self.config.get("mission"): # mission is string like: 2021.062.01 and is assumed to exist self.process_mission_exception_wrapper( - self.args.mission, - src_dir=self.get_mission_dir(self.args.mission), + self.config["mission"], + src_dir=self.get_mission_dir(self.config["mission"]), ) - elif self.args.start_year and self.args.end_year: + elif self.config.get("start_year") and self.config.get("end_year"): missions = self.mission_list( - start_year=self.args.start_year, - end_year=self.args.end_year, + start_year=self.config["start_year"], + end_year=self.config["end_year"], ) - if self.args.start_year == self.args.end_year: + if self.config["start_year"] == self.config["end_year"]: # Subselect missions by year day, has effect if --start_yd & --end_yd # are specified and --start_year & --end_year are the same missions = { mission: missions[mission] for mission in missions if ( - int(mission.split(".")[1]) >= self.args.start_yd - and int(mission.split(".")[1]) <= self.args.end_yd + int(mission.split(".")[1]) >= self.config["start_yd"] + and int(mission.split(".")[1]) <= self.config["end_yd"] ) } # https://pythonspeed.com/articles/python-multiprocessing/ - Swimming with sharks! - ncores = self.args.num_cores if self.args.num_cores else multiprocessing.cpu_count() + ncores = self.config.get("num_cores") or multiprocessing.cpu_count() missions = dict(sorted(missions.items())) if ncores > 1: self.logger.info( @@ -749,11 +804,9 @@ def process_missions(self, start_year: int) -> None: def extract(self, log_file: str) -> None: self.logger.info("Extracting log file: %s", log_file) extract = Extract() - extract.args = argparse.Namespace() - extract.args.verbose = self.args.verbose - extract.args.log_file = self.args.log_file + extract.args = self._create_child_namespace() extract.commandline = self.commandline - extract.logger.setLevel(self._log_levels[self.args.verbose]) + extract.logger.setLevel(self._log_levels[self.config["verbose"]]) extract.logger.addHandler(self.log_handler) url = os.path.join(BASE_LRAUV_WEB, log_file) # noqa: PTH118 @@ -769,12 +822,9 @@ def combine(self, log_file: str) -> None: "Adds nudge positions and more layers of quality control." ) combine = Combine_NetCDF() - combine.args = argparse.Namespace() - combine.args.plot = None - combine.args.verbose = self.args.verbose - combine.args.log_file = self.args.log_file + combine.args = self._create_child_namespace(plot=None) combine.commandline = self.commandline - combine.logger.setLevel(self._log_levels[self.args.verbose]) + combine.logger.setLevel(self._log_levels[self.config["verbose"]]) combine.logger.addHandler(self.log_handler) combine.combine_groups() @@ -787,7 +837,7 @@ def process_log_file(self, log_file: str) -> None: self.log_handler = logging.FileHandler( Path(netcdfs_dir, f"{Path(log_file).stem}_processing.log"), mode="w+" ) - self.log_handler.setLevel(self._log_levels[self.args.verbose]) + self.log_handler.setLevel(self._log_levels[self.config["verbose"]]) self.log_handler.setFormatter(AUV_NetCDF._formatter) self.logger.info( "=====================================================================================================================", @@ -803,11 +853,11 @@ def process_log_file(self, log_file: str) -> None: self.logger.info("Finished processing log file: %s", log_file) def process_log_files(self) -> None: - if self.args.log_file: + if self.config.get("log_file"): # log_file is string like: # brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4 - self.auv_name = self.args.log_file.split("/")[0].lower() - self.process_log_file(self.args.log_file) + self.auv_name = self.config["log_file"].split("/")[0].lower() + self.process_log_file(self.config["log_file"]) def process_command_line(self): parser = argparse.ArgumentParser( @@ -1025,6 +1075,7 @@ def process_command_line(self): self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) + return self.args if __name__ == "__main__": @@ -1033,7 +1084,15 @@ def process_command_line(self): CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://thalassa.shore.mbari.org/M3" - # Initialize for i2MAP processing, meant to be subclassed for other vehicles - proc = Processor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) - proc.process_command_line() - proc.process_missions() + # Parse command line and initialize with config pattern + temp_proc = Processor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + args = temp_proc.process_command_line() + + # Create configured processor instance + proc = Processor.from_args(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR, args) + + # Process based on arguments + if args.log_file: + proc.process_log_files() + else: + proc.process_missions(2020) diff --git a/src/data/process_Dorado389.py b/src/data/process_Dorado389.py index eac7caf6..9f3abdef 100755 --- a/src/data/process_Dorado389.py +++ b/src/data/process_Dorado389.py @@ -23,6 +23,10 @@ class DoradoProcessor(Processor): MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD" START_YEAR = 2011 - proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) - proc.process_command_line() + # Parse command line and initialize with config pattern + temp_proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + args = temp_proc.process_command_line() + + # Create configured processor instance + proc = DoradoProcessor.from_args(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR, args) proc.process_missions(START_YEAR) diff --git a/src/data/process_dorado.py b/src/data/process_dorado.py index a60c0e3b..890ed4f8 100755 --- a/src/data/process_dorado.py +++ b/src/data/process_dorado.py @@ -36,6 +36,10 @@ class DoradoProcessor(Processor): MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD" START_YEAR = 2003 - proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) - proc.process_command_line() + # Parse command line and initialize with config pattern + temp_proc = DoradoProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + args = temp_proc.process_command_line() + + # Create configured processor instance + proc = DoradoProcessor.from_args(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR, args) proc.process_missions(START_YEAR) diff --git a/src/data/process_i2map.py b/src/data/process_i2map.py index c6ee2247..fe7a065d 100755 --- a/src/data/process_i2map.py +++ b/src/data/process_i2map.py @@ -35,6 +35,10 @@ class I2mapProcessor(Processor): MOUNT_DIR = "smb://thalassa.shore.mbari.org/M3" START_YEAR = 2017 - proc = I2mapProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) - proc.process_command_line() + # Parse command line and initialize with config pattern + temp_proc = I2mapProcessor(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR) + args = temp_proc.process_command_line() + + # Create configured processor instance + proc = I2mapProcessor.from_args(AUV_NAME, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR, args) proc.process_missions(START_YEAR) diff --git a/src/data/process_lrauv.py b/src/data/process_lrauv.py index b3db134e..20986179 100755 --- a/src/data/process_lrauv.py +++ b/src/data/process_lrauv.py @@ -36,8 +36,11 @@ class LRAUVProcessor(Processor): # in the future, so point to a potential directory where they can be found. CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files" MOUNT_DIR = "smb://atlas.shore.mbari.org/LRAUV" - START_YEAR = 2012 - proc = LRAUVProcessor(AUV_NAME, LRAUV_DIR, MOUNT_DIR, CALIBRATION_DIR) - proc.process_command_line() + # Parse command line and initialize with config pattern + temp_proc = LRAUVProcessor(AUV_NAME, LRAUV_DIR, MOUNT_DIR, CALIBRATION_DIR) + args = temp_proc.process_command_line() + + # Create configured processor instance + proc = LRAUVProcessor.from_args(AUV_NAME, LRAUV_DIR, MOUNT_DIR, CALIBRATION_DIR, args) proc.process_log_files() From 50b46a9d94d30f3895725dfe6764129eaa94fa4d Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 17:48:36 -0800 Subject: [PATCH 3/7] Add common_args.py and use to reduce arguement parsing code replication. --- src/data/__init__.py | 0 src/data/align.py | 64 ++-------- src/data/archive.py | 59 ++------- src/data/calibrate.py | 58 ++------- src/data/combine.py | 40 ++---- src/data/common_args.py | 235 ++++++++++++++++++++++++++++++++++++ src/data/create_products.py | 53 ++------ src/data/emailer.py | 52 ++------ src/data/logs2netcdfs.py | 83 ++----------- src/data/nc42netcdfs.py | 37 +----- src/data/resample.py | 76 ++---------- 11 files changed, 328 insertions(+), 429 deletions(-) create mode 100644 src/data/__init__.py create mode 100644 src/data/common_args.py diff --git a/src/data/__init__.py b/src/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/data/align.py b/src/data/align.py index 81cf28fe..21295e27 100755 --- a/src/data/align.py +++ b/src/data/align.py @@ -12,14 +12,12 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2021, Monterey Bay Aquarium Research Institute" -import argparse -import json +import json # noqa: I001 import logging import os import re import sys import time -from argparse import RawTextHelpFormatter from datetime import UTC, datetime from pathlib import Path from socket import gethostname @@ -27,17 +25,12 @@ import git import numpy as np import pandas as pd +from scipy.interpolate import interp1d import xarray as xr -from logs2netcdfs import ( - BASE_PATH, - MISSIONNETCDFS, - SUMMARY_SOURCE, - TIME, - TIME60HZ, - AUV_NetCDF, -) + +from common_args import get_standard_lrauv_parser +from logs2netcdfs import AUV_NetCDF, MISSIONNETCDFS, SUMMARY_SOURCE, TIME, TIME60HZ from nc42netcdfs import BASE_LRAUV_PATH -from scipy.interpolate import interp1d class InvalidCalFile(Exception): @@ -679,6 +672,7 @@ def write_netcdf(self, netcdfs_dir, vehicle: str = "", name: str = "") -> None: ) def process_command_line(self): + """Process command line arguments using shared parser infrastructure.""" examples = "Examples:" + "\n\n" examples += " Align calibrated data for some missions:\n" examples += " " + sys.argv[0] + " --mission 2020.064.10\n" @@ -691,57 +685,19 @@ def process_command_line(self): + "202509140809_202509150109.nc4\n" ) - parser = argparse.ArgumentParser( - formatter_class=RawTextHelpFormatter, + # Use shared LRAUV parser since align handles both Dorado and LRAUV + parser = get_standard_lrauv_parser( description=__doc__, epilog=examples, ) - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help=f"Base directory for missionlogs and missionnetcdfs, default: {BASE_PATH}", - ) - parser.add_argument( - "--auv_name", - action="store", - default="Dorado389", - help="Dorado389 (default), i2MAP, or Multibeam", - ) - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ) - parser.add_argument( - "--log_file", - action="store", - help=( - "Path to the log file of original LRAUV data, e.g.: " - "brizo/missionlogs/2025/20250903_20250909/" - "20250905T072042/202509050720_202509051653.nc4" - ), - ) + # Add align-specific arguments parser.add_argument( "--plot", action="store_true", help="Create intermediate plots to validate data operations.", ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) + self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) diff --git a/src/data/archive.py b/src/data/archive.py index 0593923a..69d43f31 100755 --- a/src/data/archive.py +++ b/src/data/archive.py @@ -9,19 +9,22 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2022, Monterey Bay Aquarium Research Institute" -import argparse -import logging +import logging # noqa: I001 import os import shutil import sys import time from pathlib import Path +from common_args import DEFAULT_BASE_PATH, get_standard_dorado_parser from create_products import MISSIONIMAGES, MISSIONODVS -from logs2netcdfs import BASE_PATH, LOG_FILES, MISSIONNETCDFS, AUV_NetCDF +from logs2netcdfs import AUV_NetCDF, LOG_FILES, MISSIONNETCDFS from nc42netcdfs import BASE_LRAUV_PATH, GROUP from resample import FREQ +# Define BASE_PATH for backward compatibility +BASE_PATH = DEFAULT_BASE_PATH + LOG_NAME = "processing.log" AUVCTD_VOL = "/Volumes/AUVCTD" LRAUV_VOL = "/Volumes/LRAUV" @@ -226,33 +229,13 @@ def copy_to_LRAUV(self, log_file: str, freq: str = FREQ) -> None: # noqa: C901, ) def process_command_line(self): - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + """Process command line arguments using shared parser infrastructure.""" + # Use shared parser with archive-specific additions + parser = get_standard_dorado_parser( description=__doc__, ) - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help="Base directory for missionlogs and missionnetcdfs, default: auv_data", - ) - parser.add_argument( - "--auv_name", - action="store", - default="Dorado389", - help="Dorado389 (default), i2map, or Multibeam", - ) - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ) - parser.add_argument( - "--freq", - action="store", - default=FREQ, - help="Resample freq", - ) + + # Add archive-specific arguments parser.add_argument( "--M3", action="store_true", @@ -263,11 +246,6 @@ def process_command_line(self): action="store_true", help="Copy reampled netCDF file(s) to appropriate place on AUVCTD", ) - parser.add_argument( - "--clobber", - action="store_true", - help="Remove existing netCDF files before copying to the AUVCTD directory", - ) parser.add_argument( "--archive_only_products", action="store_true", @@ -278,20 +256,7 @@ def process_command_line(self): action="store_true", help="Create products from the resampled netCDF file(s)", ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) + self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) diff --git a/src/data/calibrate.py b/src/data/calibrate.py index c9e735f0..f06c1b69 100755 --- a/src/data/calibrate.py +++ b/src/data/calibrate.py @@ -27,15 +27,13 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2020, Monterey Bay Aquarium Research Institute" -import argparse # noqa: I001 -import logging +import logging # noqa: I001 import os import shlex import shutil import subprocess import sys import time -from argparse import RawTextHelpFormatter from collections import OrderedDict from datetime import UTC, datetime from pathlib import Path @@ -46,16 +44,17 @@ import defusedxml.ElementTree as ET # noqa: N817 import matplotlib.pyplot as plt import numpy as np +import pandas as pd +import pyproj import xarray as xr +from scipy import signal from scipy.interpolate import interp1d -from seawater import eos80 -import pandas as pd -import pyproj from AUV import monotonic_increasing_time_indices, nudge_positions +from common_args import get_standard_dorado_parser from hs2_proc import compute_backscatter, hs2_calc_bb, hs2_read_cal_file -from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, TIME, TIME60HZ, AUV_NetCDF -from scipy import signal +from logs2netcdfs import AUV_NetCDF, MISSIONLOGS, MISSIONNETCDFS, TIME, TIME60HZ +from seawater import eos80 AVG_SALINITY = 33.6 # Typical value for upper 100m of Monterey Bay @@ -3282,39 +3281,19 @@ def process_logs(self, vehicle: str = "", name: str = "", process_gps: bool = Tr return netcdfs_dir def process_command_line(self): + """Process command line arguments using shared parser infrastructure.""" examples = "Examples:" + "\n\n" examples += " Calibrate original data for some missions:\n" examples += " " + sys.argv[0] + " --mission 2020.064.10\n" examples += " " + sys.argv[0] + " --auv_name i2map --mission 2020.055.01\n" - parser = argparse.ArgumentParser( - formatter_class=RawTextHelpFormatter, + # Use shared parser with calibrate-specific additions + parser = get_standard_dorado_parser( description=__doc__, epilog=examples, ) - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help=f"Base directory for missionlogs and missionnetcdfs, default: {BASE_PATH}", - ) - parser.add_argument( - "--auv_name", - action="store", - default="Dorado389", - help="Dorado389 (default), i2MAP, or Multibeam", - ) - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ) - parser.add_argument( - "--noinput", - action="store_true", - help="Execute without asking for a response, e.g. to not ask to re-download file", - ) + # Add calibrate-specific arguments parser.add_argument( "--plot", action="store", @@ -3322,24 +3301,9 @@ def process_command_line(self): " to validate data operations. Use first to plot " " points, e.g. first2000. Program blocks upon show.", ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) - self.commandline = " ".join(sys.argv) diff --git a/src/data/combine.py b/src/data/combine.py index a1f40b70..dc05ecec 100755 --- a/src/data/combine.py +++ b/src/data/combine.py @@ -38,22 +38,21 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2025, Monterey Bay Aquarium Research Institute" -import argparse # noqa: I001 -import json +import json # noqa: I001 import logging import sys import time -from argparse import RawTextHelpFormatter from datetime import UTC from pathlib import Path from socket import gethostname from typing import NamedTuple + import cf_xarray # Needed for the .cf accessor # noqa: F401 import numpy as np -import xarray as xr - import pandas as pd +import xarray as xr from AUV import monotonic_increasing_time_indices, nudge_positions +from common_args import get_standard_lrauv_parser from logs2netcdfs import AUV_NetCDF, TIME, TIME60HZ from nc42netcdfs import BASE_LRAUV_PATH, GROUP @@ -717,6 +716,7 @@ def write_netcdf(self) -> None: return netcdfs_dir def process_command_line(self): + """Process command line arguments using shared parser infrastructure.""" examples = "Examples:" + "\n\n" examples += " Combine original data from Group files for an LRAUV log file:\n" examples += ( @@ -727,43 +727,21 @@ def process_command_line(self): + "202509140809_202509150109.nc4\n" ) - parser = argparse.ArgumentParser( - formatter_class=RawTextHelpFormatter, + # Use shared parser with combine-specific additions + parser = get_standard_lrauv_parser( description=__doc__, epilog=examples, ) - parser.add_argument( - "--log_file", - action="store", - help=( - "Path to the log file of original LRAUV data, e.g.: " - "brizo/missionlogs/2025/20250903_20250909/" - "20250905T072042/202509050720_202509051653.nc4" - ), - ) + + # Add combine-specific arguments parser.add_argument( "--plot", action="store_true", help="Create intermediate plot(s) to help validate processing", ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) - self.commandline = " ".join(sys.argv) diff --git a/src/data/common_args.py b/src/data/common_args.py new file mode 100644 index 00000000..e79b6198 --- /dev/null +++ b/src/data/common_args.py @@ -0,0 +1,235 @@ +""" +Shared argument parser infrastructure for AUV data processing modules. + +Provides common argument parsers to eliminate duplication across modules +and ensure consistent command-line interfaces. +""" + +import argparse +from pathlib import Path + +# Define constants locally to avoid circular imports +DEFAULT_BASE_PATH = Path(__file__).parent.joinpath("../../data/auv_data").resolve() +DEFAULT_FREQ = "1S" # 1 Hz resampling frequency +DEFAULT_MF_WIDTH = 3 # Median filter width + + +class CommonArgumentParser: + """Shared argument parser factory for all AUV processing modules.""" + + @staticmethod + def get_core_parser(): + """Get parser with core arguments used across all modules. + + Returns: + argparse.ArgumentParser: Parser configured with add_help=False for parent use + """ + parser = argparse.ArgumentParser(add_help=False) + + # Core processing arguments - used by almost all modules + parser.add_argument( + "--base_path", + action="store", + default=DEFAULT_BASE_PATH, + help=f"Base directory for missionlogs and missionnetcdfs, default: {DEFAULT_BASE_PATH}", + ) + parser.add_argument( + "--auv_name", + action="store", + default="Dorado389", + help="AUV name: Dorado389 (default), i2map, or multibeam", + ) + parser.add_argument( + "--mission", + action="store", + help="Mission directory, e.g.: 2020.064.10", + ) + parser.add_argument( + "--noinput", + action="store_true", + help="Execute without asking for responses, e.g. to not ask to re-download file", + ) + parser.add_argument( + "--verbose", + type=int, + choices=range(3), + default=0, + help="Verbosity level: 0=WARN (default), 1=INFO, 2=DEBUG", + ) + + return parser + + @staticmethod + def get_processing_parser(): + """Get parser with common processing control arguments. + + Returns: + argparse.ArgumentParser: Parser configured with add_help=False for parent use + """ + parser = argparse.ArgumentParser(add_help=False) + + # Processing control arguments + parser.add_argument( + "--local", + action="store_true", + help="Specify if files are local in the MISSION directory", + ) + parser.add_argument( + "--clobber", + action="store_true", + help="Overwrite existing output files", + ) + parser.add_argument( + "--noreprocess", + action="store_true", + help="Don't re-process existing output files", + ) + + return parser + + @staticmethod + def get_dorado_parser(): + """Get parser with Dorado-specific arguments. + + Returns: + argparse.ArgumentParser: Parser configured with add_help=False for parent use + """ + parser = argparse.ArgumentParser(add_help=False) + + # Dorado-specific arguments + parser.add_argument( + "--add_seconds", + type=int, + help="Add seconds for GPS Week Rollover Bug", + ) + parser.add_argument( + "--use_portal", + action="store_true", + help="Download via portal instead of mount", + ) + parser.add_argument( + "--freq", + type=str, + default=DEFAULT_FREQ, + help=f"Resampling frequency in Hz, default: {DEFAULT_FREQ}", + ) + parser.add_argument( + "--mf_width", + type=int, + default=DEFAULT_MF_WIDTH, + help=f"Median filter width for smoothing, default: {DEFAULT_MF_WIDTH}", + ) + + return parser + + @staticmethod + def get_lrauv_parser(): + """Get parser with LRAUV-specific arguments. + + Returns: + argparse.ArgumentParser: Parser configured with add_help=False for parent use + """ + parser = argparse.ArgumentParser(add_help=False) + + # LRAUV-specific arguments + parser.add_argument( + "--log_file", + action="store", + help=( + "Path to the log file of original LRAUV data, e.g.: " + "brizo/missionlogs/2025/20250903_20250909/" + "20250905T072042/202509050720_202509051653.nc4" + ), + ) + + return parser + + @staticmethod + def get_time_range_parser(): + """Get parser with time range filtering arguments. + + Returns: + argparse.ArgumentParser: Parser configured with add_help=False for parent use + """ + parser = argparse.ArgumentParser(add_help=False) + + # Time range filtering arguments + parser.add_argument( + "--start_year", + type=int, + help="Start year for mission filtering", + ) + parser.add_argument( + "--end_year", + type=int, + help="End year for mission filtering", + ) + parser.add_argument( + "--start_yd", + type=int, + help="Start year day for mission filtering", + ) + parser.add_argument( + "--end_yd", + type=int, + help="End year day for mission filtering", + ) + parser.add_argument( + "--last_n_days", + type=int, + help="Process only the last N days of data", + ) + + return parser + + @classmethod + def create_parser(cls, module_name, parents=None, **kwargs): + """Create a parser with standard formatting and common parents. + + Args: + module_name: Name of the module (for help text) + parents: List of parent parsers to include + **kwargs: Additional arguments for ArgumentParser + + Returns: + argparse.ArgumentParser: Configured parser + """ + default_kwargs = { + "formatter_class": argparse.RawTextHelpFormatter, + "parents": parents or [], + } + default_kwargs.update(kwargs) + + return argparse.ArgumentParser(**default_kwargs) + + +# Convenience functions for common parser combinations +def get_standard_dorado_parser(**kwargs): + """Get parser with standard Dorado arguments (core + processing + dorado).""" + parents = [ + CommonArgumentParser.get_core_parser(), + CommonArgumentParser.get_processing_parser(), + CommonArgumentParser.get_dorado_parser(), + ] + return CommonArgumentParser.create_parser("dorado", parents=parents, **kwargs) + + +def get_standard_lrauv_parser(**kwargs): + """Get parser with standard LRAUV arguments (core + processing + lrauv).""" + parents = [ + CommonArgumentParser.get_core_parser(), + CommonArgumentParser.get_processing_parser(), + CommonArgumentParser.get_lrauv_parser(), + ] + return CommonArgumentParser.create_parser("lrauv", parents=parents, **kwargs) + + +def get_mission_processing_parser(**kwargs): + """Get parser with mission processing arguments (includes time range).""" + parents = [ + CommonArgumentParser.get_core_parser(), + CommonArgumentParser.get_processing_parser(), + CommonArgumentParser.get_dorado_parser(), + CommonArgumentParser.get_time_range_parser(), + ] + return CommonArgumentParser.create_parser("mission_processing", parents=parents, **kwargs) diff --git a/src/data/create_products.py b/src/data/create_products.py index aa5343a5..54dbdece 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -7,7 +7,7 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2023, Monterey Bay Aquarium Research Institute" -import argparse +import argparse # noqa: I001 import contextlib import logging import os @@ -22,11 +22,16 @@ import numpy as np import pyproj import xarray as xr + +from common_args import DEFAULT_BASE_PATH, get_standard_dorado_parser from gulper import Gulper -from logs2netcdfs import BASE_PATH, MISSIONNETCDFS, AUV_NetCDF +from logs2netcdfs import AUV_NetCDF, MISSIONNETCDFS from resample import AUVCTD_OPENDAP_BASE, FREQ from scipy.interpolate import griddata +# Define BASE_PATH for backward compatibility +BASE_PATH = DEFAULT_BASE_PATH + MISSIONODVS = "missionodvs" MISSIONIMAGES = "missionimages" @@ -524,51 +529,19 @@ def gulper_odv(self, sec_bnds: int = 1) -> str: # noqa: C901, PLR0912, PLR0915 ) def process_command_line(self): - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + """Process command line arguments using shared parser infrastructure.""" + # Use shared parser with create_products-specific additions + parser = get_standard_dorado_parser( description=__doc__, ) - ( - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help=f"Base directory for missionlogs and missionnetcdfs, default: {BASE_PATH}", - ), - ) - parser.add_argument( - "--auv_name", - action="store", - default="dorado", - help="dorado (default), i2map", - ) - ( - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ), - ) + + # Add create_products-specific arguments parser.add_argument( "--start_esecs", help="Start time of mission in epoch seconds, optional for gulper time lookup", type=float, ) - parser.add_argument("--local", help="Read local files", action="store_true") - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) + self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) diff --git a/src/data/emailer.py b/src/data/emailer.py index 1459760b..4ff7b571 100755 --- a/src/data/emailer.py +++ b/src/data/emailer.py @@ -7,14 +7,17 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2023, Monterey Bay Aquarium Research Institute" -import argparse -import logging +import logging # noqa: I001 import platform import sys import time from pathlib import Path -from logs2netcdfs import BASE_PATH, MISSIONNETCDFS, AUV_NetCDF +from common_args import DEFAULT_BASE_PATH, get_standard_dorado_parser +from logs2netcdfs import AUV_NetCDF, MISSIONNETCDFS + +# Define BASE_PATH for backward compatibility +BASE_PATH = DEFAULT_BASE_PATH NOTIFICATION_EMAIL = "auvctd@listserver.mbari.org" TEMPLATE = """ @@ -90,31 +93,13 @@ def compose_message(self) -> str: ) def process_command_line(self): - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + """Process command line arguments using shared parser infrastructure.""" + # Use shared parser with emailer-specific additions + parser = get_standard_dorado_parser( description=__doc__, ) - ( - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help="Base directory for missionlogs and missionnetcdfs, default: auv_data", - ), - ) - parser.add_argument( - "--auv_name", - action="store", - default="Dorado389", - help="Dorado389 (default), i2map, or Multibeam", - ) - ( - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ), - ) + + # Add emailer-specific arguments parser.add_argument( "--email_to", action="store", @@ -124,20 +109,7 @@ def process_command_line(self): f"default: {NOTIFICATION_EMAIL}" ), ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) + self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) diff --git a/src/data/logs2netcdfs.py b/src/data/logs2netcdfs.py index 444ca816..c931bcb8 100755 --- a/src/data/logs2netcdfs.py +++ b/src/data/logs2netcdfs.py @@ -9,7 +9,7 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2020, Monterey Bay Aquarium Research Institute" -import argparse +import argparse # noqa: I001 import asyncio import concurrent import logging @@ -27,8 +27,10 @@ import requests from aiohttp import ClientSession from aiohttp.client_exceptions import ClientConnectorError -from AUV import monotonic_increasing_time_indices from netCDF4 import Dataset + +from AUV import monotonic_increasing_time_indices +from common_args import get_standard_dorado_parser from readauvlog import log_record LOG_FILES = ( @@ -883,42 +885,19 @@ def set_portal(self) -> None: self.deployments_url = Path(self.args.portal, "deployments") def process_command_line(self): + """Process command line arguments using shared parser infrastructure.""" examples = "Examples:" + "\n\n" examples += " Write to local missionnetcdfs direcory:\n" examples += " " + sys.argv[0] + " --mission 2020.064.10\n" examples += " " + sys.argv[0] + " --auv_name i2map --mission 2020.055.01\n" - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + # Use shared parser with logs2netcdfs-specific additions + parser = get_standard_dorado_parser( description=__doc__, epilog=examples, ) - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help="Base directory for missionlogs and missionnetcdfs, default: auv_data", - ) - parser.add_argument( - "--auv_name", - action="store", - help=( - "Dorado389, i2map, or multibeam. Will be saved in " - "directory with this name no matter its portal entry" - ), - ) - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ) - parser.add_argument( - "--local", - action="store_true", - help="Specify if files are local in the MISSION directory", - ) - + # Add logs2netcdfs-specific arguments parser.add_argument( "--title", action="store", @@ -929,22 +908,6 @@ def process_command_line(self): action="store", help="Additional information about the dataset", ) - - parser.add_argument( - "--noinput", - action="store_true", - help="Execute without asking for a response, e.g. to not ask to re-download file", - ) - parser.add_argument( - "--clobber", - action="store_true", - help="Use with --noinput to overwrite existing downloaded log files", - ) - parser.add_argument( - "--noreprocess", - action="store_true", - help="Use with --noinput to not re-process existing downloaded log files", - ) parser.add_argument( "--start", action="store", @@ -972,41 +935,11 @@ def process_command_line(self): " service, e.g.:" " http://stoqs.mbari.org:8080/auvdata/v1", ) - parser.add_argument( - "--use_portal", - action="store_true", - help=( - "Download data using portal (much faster than copy over" - " remote connection), otherwise copy from mount point" - ), - ) parser.add_argument( "--vehicle_dir", action="store", help="Directory for the vehicle's mission logs, e.g.: /Volumes/AUVCTD/missionlogs", ) - parser.add_argument( - # To use for mission 2025.316.02 which suffered from the GPS week rollover bug: - # 1024 * 7 * 24 * 3600 = 619315200 seconds to add to timeTag variables in the log_data - "--add_seconds", - type=int, - default=0, - help="Seconds to add to timeTag in log data", - ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) diff --git a/src/data/nc42netcdfs.py b/src/data/nc42netcdfs.py index f4f5f51d..da65f9ec 100755 --- a/src/data/nc42netcdfs.py +++ b/src/data/nc42netcdfs.py @@ -8,7 +8,6 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2025, Monterey Bay Aquarium Research Institute" -import argparse import logging import os import sys @@ -20,6 +19,7 @@ import netCDF4 import numpy as np import pooch +from common_args import get_standard_lrauv_parser # Conditional imports for plotting (only when needed) try: @@ -1032,6 +1032,7 @@ def global_metadata(self, log_file: str, group_name: str): return metadata def process_command_line(self): + """Process command line arguments using shared parser infrastructure.""" examples = "Examples:" + "\n\n" examples += " Write to local missionnetcdfs direcory:\n" examples += " " + sys.argv[0] + " --mission 2020.064.10\n" @@ -1044,11 +1045,13 @@ def process_command_line(self): + "202509140809_202509150109.nc4 --plot_time /latitude_time\n" ) - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + # Use shared parser with nc42netcdfs-specific additions + parser = get_standard_lrauv_parser( description=__doc__, epilog=examples, ) + + # Add nc42netcdfs-specific arguments parser.add_argument( "--filter_monotonic_time", action="store_true", @@ -1071,20 +1074,6 @@ def process_command_line(self): action="store", help="Convert a range of missions wth end time in YYYYMMDD format", ) - parser.add_argument( - "--auv_name", - action="store", - help="Name of the AUV and the directory name for its data, e.g.: tethys, ahi, pontus", - ) - parser.add_argument( - "--log_file", - action="store", - help=( - "Path to the log file for the mission, e.g.: " - "brizo/missionlogs/2025/20250903_20250909/" - "20250905T072042/202509050720_202509051653.nc4" - ), - ) parser.add_argument( "--known_hash", action="store", @@ -1110,20 +1099,6 @@ def process_command_line(self): "Format for is /Group/variable_name." ), ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) diff --git a/src/data/resample.py b/src/data/resample.py index bf1b9623..0fb6a7a0 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -9,8 +9,7 @@ __author__ = "Mike McCann" __copyright__ = "Copyright 2021, Monterey Bay Aquarium Research Institute" -import argparse -import logging +import logging # noqa: I001 import re import sys import time @@ -25,12 +24,14 @@ import numpy as np import pandas as pd import xarray as xr -from dorado_info import dorado_info -from logs2netcdfs import BASE_PATH, MISSIONNETCDFS, SUMMARY_SOURCE, TIME, AUV_NetCDF -from nc42netcdfs import BASE_LRAUV_PATH from pysolar.solar import get_altitude from scipy import signal +from common_args import get_standard_lrauv_parser +from dorado_info import dorado_info +from logs2netcdfs import AUV_NetCDF, BASE_PATH, MISSIONNETCDFS, SUMMARY_SOURCE, TIME +from nc42netcdfs import BASE_LRAUV_PATH + MF_WIDTH = 3 FREQ = "1S" PLOT_SECONDS = 300 @@ -1320,40 +1321,13 @@ def resample_mission( # noqa: C901, PLR0912, PLR0915, PLR0913 self.logger.info("Saved resampled mission to %s", out_fn) def process_command_line(self): - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, + """Process command line arguments using shared parser infrastructure.""" + # Use shared parser with resample-specific additions + parser = get_standard_lrauv_parser( description=__doc__, ) - ( - parser.add_argument( - "--base_path", - action="store", - default=BASE_PATH, - help="Base directory for missionlogs and missionnetcdfs, default: auv_data", - ), - ) - parser.add_argument( - "--auv_name", - action="store", - default="Dorado389", - help="Dorado389 (default), i2MAP, or Multibeam", - ) - ( - parser.add_argument( - "--mission", - action="store", - help="Mission directory, e.g.: 2020.064.10", - ), - ) - parser.add_argument( - "--log_file", - action="store", - help=( - "Path to the log file of original LRAUV data, e.g.: " - "brizo/missionlogs/2025/20250903_20250909/" - "20250905T072042/202509050720_202509051653.nc4" - ), - ) + + # Add resample-specific arguments parser.add_argument("--plot", action="store_true", help="Plot data") parser.add_argument( "--plot_seconds", @@ -1362,19 +1336,6 @@ def process_command_line(self): type=float, help="Plot seconds of data", ) - parser.add_argument( - "--mf_width", - action="store", - default=MF_WIDTH, - type=int, - help="Median filter width", - ) - parser.add_argument( - "--freq", - action="store", - default=FREQ, - help="Resample freq", - ) parser.add_argument( "--flash_threshold", action="store", @@ -1384,20 +1345,7 @@ def process_command_line(self): "and append to output filename" ), ) - parser.add_argument( - "-v", - "--verbose", - type=int, - choices=range(3), - action="store", - default=0, - const=1, - nargs="?", - help="verbosity level: " - + ", ".join( - [f"{i}: {v}" for i, v in enumerate(("WARN", "INFO", "DEBUG"))], - ), - ) + self.args = parser.parse_args() self.logger.setLevel(self._log_levels[self.args.verbose]) self.commandline = " ".join(sys.argv) From 83078408b8806b62ab674a831b5a5d9417ee0ede Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 17:56:57 -0800 Subject: [PATCH 4/7] Fix linting with exceptions for non-pep8 module names. --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f6684c65..cb55b1da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,3 +110,10 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "src/data/dorado_info.py" = ["E501"] +# Legacy module names that don't follow PEP 8 naming convention +"src/data/AUV.py" = ["N999"] +"src/data/BLFilter.py" = ["N999"] +"src/data/lopcMEP.py" = ["N999"] +"src/data/lopcToNetCDF.py" = ["N999"] +"src/data/process_Dorado389.py" = ["N999"] +"src/data/usblToNetCDF.py" = ["N999"] From 9ff09a4dac0287b6f36df4d8622f98892af67c12 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 18:04:28 -0800 Subject: [PATCH 5/7] Fix pytest: Add the current directory to Python path so modules can import each other. --- src/data/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/data/conftest.py b/src/data/conftest.py index 3486953f..054dba6a 100644 --- a/src/data/conftest.py +++ b/src/data/conftest.py @@ -1,10 +1,16 @@ # noqa: INP001 import logging import os +import sys from argparse import Namespace from pathlib import Path import pytest + +# Add the current directory to Python path so modules can import each other +# This preserves the original import behavior while allowing package structure +sys.path.insert(0, str(Path(__file__).parent)) + from calibrate import Calibrate_NetCDF from hs2_proc import hs2_read_cal_file from logs2netcdfs import BASE_PATH, MISSIONLOGS From 15d93bb8d08aa4ec958866df275e1efe1a2e4476 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 18:11:24 -0800 Subject: [PATCH 6/7] Update values for CI to pass in Actions and act. --- src/data/test_process_dorado.py | 6 +++--- src/data/test_process_i2map.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index d368b183..423feab3 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -32,7 +32,7 @@ def test_process_dorado(complete_dorado_processing): # If code changes are expected to change the file size then we should # update the expected size here. EXPECTED_SIZE_GITHUB = 621404 - EXPECTED_SIZE_ACT = 621298 + EXPECTED_SIZE_ACT = 621402 EXPECTED_SIZE_LOCAL = 621452 if str(proc.args.base_path).startswith("/home/runner"): # The size is different in GitHub Actions, maybe due to different metadata @@ -50,8 +50,8 @@ def test_process_dorado(complete_dorado_processing): check_md5 = True if check_md5: # Check that the MD5 hash has not changed - EXPECTED_MD5_GITHUB = "3bab0300e575c1d752a35f49e49e340e" - EXPECTED_MD5_ACT = "bdb9473e5dedb694618f518b8cf0ca1e" + EXPECTED_MD5_GITHUB = "631c25971f0e3b4f83f981389a179917" + EXPECTED_MD5_ACT = "bb1d539284bee531a00c4d4d99580bf0" EXPECTED_MD5_LOCAL = "9137be5a2ed840cfca94a723285355ec" if str(proc.args.base_path).startswith("/home/runner"): # The MD5 hash is different in GitHub Actions, maybe due to different metadata diff --git a/src/data/test_process_i2map.py b/src/data/test_process_i2map.py index 66508695..e7a9b553 100644 --- a/src/data/test_process_i2map.py +++ b/src/data/test_process_i2map.py @@ -31,7 +31,7 @@ def test_process_i2map(complete_i2map_processing): # If code changes are expected to change the file size then we should # update the expected size here. EXPECTED_SIZE_GITHUB = 58942 - EXPECTED_SIZE_ACT = 58816 + EXPECTED_SIZE_ACT = 58912 EXPECTED_SIZE_LOCAL = 59042 if str(proc.args.base_path).startswith("/home/runner"): # The size is different in GitHub Actions, maybe due to different metadata From 8b22fe2bbf1a5a10fc901b6b851ee4df4ebf28a9 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Thu, 20 Nov 2025 18:13:28 -0800 Subject: [PATCH 7/7] One more update in value for Actions. --- src/data/test_process_dorado.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index 423feab3..1f00d2c5 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -50,7 +50,7 @@ def test_process_dorado(complete_dorado_processing): check_md5 = True if check_md5: # Check that the MD5 hash has not changed - EXPECTED_MD5_GITHUB = "631c25971f0e3b4f83f981389a179917" + EXPECTED_MD5_GITHUB = "3bab0300e575c1d752a35f49e49e340e" EXPECTED_MD5_ACT = "bb1d539284bee531a00c4d4d99580bf0" EXPECTED_MD5_LOCAL = "9137be5a2ed840cfca94a723285355ec" if str(proc.args.base_path).startswith("/home/runner"):