Fixes for problems found installing on a new workstation

.vscode/launch.json

@@ -235,7 +235,7 @@
             //"args": ["-v", "1", "--mission", "2007.344.00", "--use_portal", "--clobber", "--noinput"]
             //"args": ["-v", "1", "--mission", "2009.055.05", "--clobber", "--noinput"]
             //"args": ["-v", "1", "--mission", "2010.172.01", "--local", "--clobber", "--noinput"]
-            //"args": ["-v", "1", "--start_year", "2010",  "--end_year", "2010", "--start_yd", "81", "--end_yd", "84", "--clobber", "--noinput"]
+            "args": ["-v", "1", "--start_year", "2010",  "--end_year", "2010", "--start_yd", "81", "--end_yd", "84", "--no_cleanup", "--num_cores", "1", "--noinput"]
             //"args": ["-v", "1", "--mission", "2010.286.02", "--clobber", "--noinput"]
             //"args": ["-v", "1", "--start_year", "2010",  "--end_year", "2010", "--start_yd", "286", "--local", "--clobber", "--noinput"]
             //"args": ["-v", "1", "--start_year", "2004",  "--end_year", "2004", "--start_yd", "167", "--clobber", "--noinput"]
@@ -262,7 +262,7 @@
             //"args": ["-v", "1", "--noinput",  "--start_year", "2016", "--end_year", "2016", "--start_yd", "270", "--end_yd", "270", "--num_cores", "1", "--create_products", "--archive", "--archive_only_products"]
             //"args": ["-v", "1", "--noinput", "--num_cores", "1", "--mission", "2023.285.01"]
             //"args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2018.079.00"]
-            "args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2024.317.01", "--clobber"]
+            //"args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2024.317.01", "--clobber"]
             //"args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2016.090.01", "--resample", "--archive", "--flash_threshold", "3E10"]
             //"args": ["-v", "1", "--noinput", "--num_cores", "8"]
             //"args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2011.256.02", "--clobber"]
@@ -271,5 +271,13 @@
             //"args": ["-v", "1", "--noinput", "--no_cleanup",  "--mission", "2008.010.10"]
 
         },
+        {
+            "name": "process_Dorado389",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/data/process_Dorado389.py",
+            "console": "integratedTerminal",
+            "args": ["-v", "1", "--noinput", "--no_cleanup", "--download", "--mission", "2011.256.02"]
+        },
     ]
 }

src/data/conftest.py

@@ -11,6 +11,14 @@
 from process import Processor
 from resample import FLASH_THRESHOLD, FREQ, MF_WIDTH
 
+bootstrap_mission = """The working directory on a development machine must be bootstrapped
+with some mission data. Process the mission used for testing with:
+
+uv run src/data/process_Dorado389.py --no_cleanup --download --mission 2011.256.02 -v
+
+This uses the legacy name "Dorado389" for the vehicle - the new name is "dorado".
+"""
+
 TEST_VEHICLE = "Dorado389"
 TEST_MISSION = "2011.256.02"  # http://stoqs.mbari.org/p/DmHOaxI
 # Set TEST_VEHICLE_DIR to local path for testing
@@ -32,6 +40,8 @@
 
 @pytest.fixture(scope="session", autouse=False)
 def mission_data():
+    if not Path(TEST_VEHICLE_DIR).exists():
+        pytest.fail(f"\n\n{bootstrap_mission}\n")
     """Load a short recent mission to have some real data to work with"""
     cal_netcdf = Calibrate_NetCDF()
     ns = Namespace()

src/data/process.py

@@ -134,10 +134,13 @@ def get_mission_dir(self, mission: str) -> str:
             self.logger.error("%s does not exist.", self.vehicle_dir)
             self.logger.info("Is %s mounted?", self.mount_dir)
             sys.exit(1)
-        if self.vehicle.lower() == "dorado":
-            year = mission.split(".")[0]
-            yearyd = "".join(mission.split(".")[:2])
-            path = Path(self.vehicle_dir, year, yearyd, mission)
+        if self.vehicle.lower() == "dorado" or self.vehicle == "Dorado389":
+            if self.args.local:
+                path = Path(self.vehicle_dir, mission)
+            else:
+                year = mission.split(".")[0]
+                yearyd = "".join(mission.split(".")[:2])
+                path = Path(self.vehicle_dir, year, yearyd, mission)
         elif self.vehicle.lower() == "i2map":
             year = int(mission.split(".")[0])
             # Could construct the YYYY/MM/YYYYMMDD path on M3/Master

src/data/process_Dorado389.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""
+This script is nearly identical to process_dorado.py, but it is used for
+bootstrapping the testing infrastructure. It is used to process a short mission
+using the legacy vehicle name "Dorado389" so that it's kept separate from the
+new production name "dorado".
+"""
+
+__author__ = "Mike McCann"
+__copyright__ = "Copyright 2022, Monterey Bay Aquarium Research Institute"
+
+from process import Processor
+
+
+class DoradoProcessor(Processor):
+    pass
+
+
+if __name__ == "__main__":
+    VEHICLE = "Dorado389"
+    VEHICLE_DIR = "/Volumes/AUVCTD/missionlogs"
+    CALIBRATION_DIR = "/Volumes/DMO/MDUC_CORE_CTD_200103/Calibration Files"
+    MOUNT_DIR = "smb://atlas.shore.mbari.org/AUVCTD"
+    START_YEAR = 2011
+
+    proc = DoradoProcessor(VEHICLE, VEHICLE_DIR, MOUNT_DIR, CALIBRATION_DIR)
+    proc.process_command_line()
+    proc.process_missions(START_YEAR)

src/data/resample.py

@@ -1016,7 +1016,7 @@ def resample_variable(  # noqa: PLR0913
                     instrs_to_pad[instr],
                 )
                 dt_index = pd.date_range(mission_start, mission_end, freq=freq.lower())
-                self.df_r[variable] = pd.Series(np.NaN, index=dt_index)
+                self.df_r[variable] = pd.Series(np.nan, index=dt_index)
                 instr_data = (
                     self.df_o[f"{variable}_mf"]
                     .shift(0.5, freq=freq.lower())

src/data/test_process_dorado.py

@@ -13,7 +13,7 @@ def test_process_dorado(complete_dorado_processing):
     """Test that Dorado processing runs without error"""
     # complete_processing ia a fixture from the conftest.py module;
     # it is automatically loaded by pytest.  We need this complete_processing
-    # to run withut error to trust that tthe production processing is working.
+    # to run without error to trust that the production processing is working.
     proc = complete_dorado_processing
 
     # Check that the _1S.nc file was created and is the correct size