Merge long-running LRAUV development branch into main

.vscode/launch.json

@@ -14,7 +14,7 @@
             //"args": ["--auv_name", "i2map", "--mission", "2020.055.01", "--noinput", "--local", "-v", "2", "--clobber"]
             //"args": ["--auv_name", "Dorado389", "--mission", "2020.245.00", "--noinput", "-v", "2", "--portal", "http://stoqs.mbari.org:8080/auvdata/v1", "--clobber"]
             //"args": ["--auv_name", "Dorado389", "--mission", "2020.245.00", "--noinput", "-v"]
-            //"args": ["--auv_name", "Dorado389", "--mission", "2017.297.00", "--local",  "-v", "2"]
+            //"args": ["--auv_name", "dorado", "--mission", "2017.297.00",  "-v", "1", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"]
             //"args": ["--auv_name", "Dorado389", "--start", "20190701", "--end", "20191230", "-v", "2"]
             //"args": ["--auv_name", "i2map", "--mission", "2021.062.01", "--noinput",  "-v", "1"]
             //"args": ["--auv_name", "dorado", "--mission", "2021.109.00", "--noinput", "-v"]
@@ -41,13 +41,32 @@
             "args": ["-v", "1", "-d", "0", "-i", "data/auv_data/dorado/missionlogs/2009.055.05/lopc.bin", "-n", "data/auv_data/dorado/missionnetcdfs/2009.055.05/lopc.nc", "-f", "--LargeCopepod_AIcrit", "0.3"]
         },
         {
-            "name": "1.1 - correct_log_times.py --mission 2017.284.00 --auv_name Dorado389",
+            "name": "1.2 - correct_log_times.py --mission 2017.284.00 --auv_name Dorado389",
             "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/src/data/correct_log_times.py",
             "console": "integratedTerminal",
             "args": ["--auv_name", "Dorado389", "--mission", "2017.284.00", "-v", "2"]
         },
+        {
+            "name": "1.3 - nc42netcdfs",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/data/nc42netcdfs.py",
+            "console": "integratedTerminal",
+            // A small log_file that has a reasonable amount of data, and known_hash to verify download
+            //"args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250908_20250912/20250911T201546/202509112015_202509112115.nc4", "--known_hash", "d1235ead55023bea05e9841465d54a45dfab007a283320322e28b84438fb8a85"]
+            // brizo 20250914T080941 has bad latitude and longitude values and lots of bad Universal latitude_time and longitude_time values
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--plot_time", "/longitude_time"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--plot_time", "/latitude_time"]
+            // brizo 20250916T230652 has several ESP Samples from stoqs_lrauv_sep2025
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--plot_time", "/longitude_time"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109_cleaned_by_quinn.nc4", "--plot_time", "/longitude_time"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109_cleaned_by_quinn_latlon.nc4", "--plot_time", "/longitude_time"]
+            // Conflicting sizes for nudged_time and data - fixed by filtering GPS fixes to be monotonically increasing
+            "args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--plot_time", "/longitude_time"]
+        },
         {
             "name": "2.0 - calibrate.py",
             "type": "debugpy",
@@ -85,7 +104,8 @@
             //"args": ["--auv_name", "dorado", "--mission", "2018.079.00", "-v", "1"]
             //"args": ["--auv_name", "i2map", "--mission", "2018.348.01", "-v", "2"]
             //"args": ["--auv_name", "dorado", "--mission", "2023.324.00", "-v", "1", "--plot", "first10000"]
-            "args": ["--auv_name", "dorado", "--mission", "2022.201.00", "-v", "1", "--plot", "first10000"]
+            //"args": ["--auv_name", "dorado", "--mission", "2022.201.00", "-v", "1", "--plot", "first10000"]
+            "args": ["--auv_name", "dorado", "--mission", "2025.316.02", "-v", "1"]
         },
         {
             "name": "2.1 - Test hs2_proc.py (its unit tests)",
@@ -94,6 +114,21 @@
             "program": "${workspaceFolder}/src/data/hs2_proc.py",
             "console": "integratedTerminal",
         },
+
+        {
+            "name": "2.2 - combine.py",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/data/combine.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--plot"]
+            //"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120909T010636/201209090106_201209091521.nc4", "--plot"]
+            // Conflicting sizes for nudged_time and data - fixed by filtering GPS fixes to be monotonically increasing
+            "args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--plot"]
+
+        },
         {
             "name": "3.0 - align.py",
             "type": "debugpy",
@@ -113,7 +148,19 @@
             //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2004.236.00"],
             //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2008.289.03"],
             //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2023.192.01"],
-            "args": ["-v", "1", "--auv_name", "dorado", "--mission", "2024.317.01"],
+            //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2024.317.01"],
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
+            "args": ["-v", "1", "--auv_name", "dorado", "--mission", "2025.316.02"],
+        },
+        {
+            "name": "3.1 - align.py for LRAUV --log_file",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/data/align.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"],
+            "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
         },
         {
             "name": "4.0 - resample.py",
@@ -136,13 +183,17 @@
             //"args": ["--auv_name", "dorado", "--mission", "2017.044.00", "-v", "1"]
             //"args": ["--auv_name", "dorado", "--mission", "2021.102.02", "-v", "1"]
             //"args": ["--auv_name", "dorado", "--mission", "2004.236.00", "-v", "1"]
-            "args": ["--auv_name", "dorado", "--mission", "2023.192.01", "-v", "1"]
+            //"args": ["--auv_name", "dorado", "--mission", "2023.192.01", "-v", "1"]
             //"args": ["--auv_name", "i2map", "--mission", "2019.157.02", "-v", "2", "--plot", "--plot_seconds", "82000"],
             //"args": ["--auv_name", "dorado", "--mission", "2021.102.02", "-v", "1", "--flash_threshold", "1.5e10"],
             //"args": ["--auv_name", "dorado", "--mission", "2024.317.01", "-v", "1"],
             //"args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"],
             //"args": ["--auv_name", "dorado", "--mission", "2020.337.00", "-v", "1"],
             //"args": ["--auv_name", "dorado", "--mission", "2023.123.00", "-v", "1"],
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
+            //"args": ["--auv_name", "dorado", "--mission", "2025.316.02", "-v", "1"],
+            //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250604_20250616/20250608T020852/202506080209_202506081934.nc4"],
+            "args": ["-v", "1", "--log_file", "ahi/missionlogs/2025/20250414_20250418/20250414T205440/202504142054_202504150400.nc4"],
         },
         {
             "name": "5.0 - archive.py",
@@ -285,5 +336,33 @@
             "console": "integratedTerminal",
             "args": ["-v", "1", "--noinput", "--no_cleanup", "--download", "--mission", "2011.256.02"]
         },
+        {
+            "name": "process_lrauv",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/src/data/process_lrauv.py",
+            "console": "integratedTerminal",
+            // Lots bad time values in brizo 20250914T080941 due to memory corruption on the vehicle
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--clobber"]
+            //"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--clobber", "--no_cleanup"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--no_cleanup"]
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--no_cleanup", "--clobber"]
+            // Has different universals time coodinates for longitude/latitude and depth
+            //"args": ["-v", "1", "--auv_name", "tethys", "--start", "20120901T000000", "--end", "20121101T000000", "--noinput", "--no_cleanup"]
+            // Conflicting sizes for nudged_time and data - fixed by filtering GPS fixes to be monotonically increasing
+            //"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--no_cleanup"
+            //"args": ["-v", "1", "--auv_name", "brizo", "--start", "20250915T000000", "--end", "20250917T000000", "--noinput", "--num_cores", "1", "--no_cleanup"]
+            // No nudged latitude and longitude variables - fixed as of 26 Nov 2025
+            //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250915T015535/202509150155_202509151602.nc4", "--no_cleanup"]
+            // Plankitvore deployment for CeNCOOS Syncro - whole month of April 2025
+            //"args": ["-v", "1", "--auv_name", "ahi", "--start", "20250401T000000", "--end", "20250502T000000", "--noinput", "--num_cores", "1", "--no_cleanup"]
+            // Fails with ValueError: different number of dimensions on data and dims: 2 vs 1 for wetlabsubat_digitized_raw_ad_counts variable
+            //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250604_20250616/20250608T020852/202506080209_202506081934.nc4", "--no_cleanup"]
+            // Full month of June 2025 for Pontus with WetLabsUBAT Group data
+            //"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250702T000000", "--noinput", "--num_cores", "1", "--no_cleanup"]
+            "args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250702T000000", "--noinput", "--num_cores", "1", "--no_cleanup", "--clobber"]
+        },
+
     ]
 }

WORKFLOW.md → DORADO_WORKFLOW.md

@@ -1,6 +1,6 @@
-## Data Workflow
+## Dorado Data Workflow
 
-The sequence of steps to process data is as follows:
+The sequence of steps to process Dorado data is as follows:
 
   logs2netcdfs.py → calibrate.py → align.py → resample.py → archive.py → plot.py
 
@@ -70,6 +70,6 @@ on the local file system's work directory is as follows:
 
     archive.py
         Copy the netCDF files to the archive directory. The archive directory
-        is initally in the AUVCTD share on atlas which is shared with the
+        is initially in the AUVCTD share on atlas which is shared with the
         data from the Dorado Gulper vehicle, but can also be on the M3 share
         on thalassa near the original log data.

LRAUV_WORKFLOW.md

@@ -0,0 +1,73 @@
+## LRAUV Data Workflow
+
+The sequence of steps to process LRAUV data is as follows:
+
+  nc42netcdfs.py → combine.py → align.py → resample.py → archive.py → plot.py
+
+Details of each step are described in the respective scripts and in the
+description of output netCDF files below. The output file directory structure
+on the local file system's work directory is as follows:
+
+    ├── data
+    │   ├── lrauv_data
+    │   │   ├── <auv_name>           <- e.g.: ahi, brizo, pontus, tethys, ...
+    │   │   │   ├── missionlogs/year/dlist_dir
+    │   │   │   │   ├── <log_dir>    <- e.g.: ahi/missionlogs/2025/20250908_20250912/20250911T201546/202509112015_202509112115.nc4
+    │   │   │   │   │   ├── <nc4>    <- .nc4 file containing original data - created by unserialize
+    │   │   │   │   │   ├── <nc>     <- .nc files, one for each group from the .nc4 file
+    |   |   |   |   |   |                data identical to original in NetCDF4 format,
+    |   |   |   |   |   |                but in more interoperable NetCDF3 format 
+    |   |   |   |   |   |                - created by nc42netcdfs.py
+    │   │   │   │   │   ├── <_combined>     <- A single NetCDF3 .nc file containing all the
+    |   |   |   |   |   |                   varibles from the .nc files along with nudged
+    |   |   |   |   |   |                   latitudes and longitudes - created by combine.py
+    │   │   │   │   │   ├── <_align> <- .nc file with all measurement variables
+    |   |   |   |   |   |               having associated coordinate variables
+    |   |   |   |   |   |               at original instrument sampling rate
+    |   |   |   |   |   |                - created by align.py
+    │   │   │   │   │   ├── <_nS>    <- .nc file with all measurement variables
+                                        resampled to a common time grid at n
+                                        Second intervals - created by resample.py
+
+    nc42netcdfs.py
+        Extract the groups and the variables we want from the groups into 
+        individual .nc files. These data are saved using NetCDF4 format as
+        there are many unlimited dimensions that are not allowed in NetCDF3.
+        The data in the .nc files are identical to what is in the .nc4 groups.
+
+    combine.py
+        Combine all group data into a single NetCDF file with consolidated
+        time coordinates. When GPS fix data is available, this step includes
+        nudging the underwater portions of the navigation positions to the
+        GPS fixes done at the surface. GPS fixes are filtered to ensure
+        monotonically increasing timestamps before nudging. Some minimal QC
+        is done in this step, namely removal of non-monotonic times. The
+        nudged coordinates are added as separate variables (nudged_longitude,
+        nudged_latitude) with their own time dimension. For missions without
+        GPS data, the combine step completes successfully but without nudged
+        coordinates.
+
+    align.py
+        Interpolate nudged lat/lon variables to the original sampling
+        intervals for each instrument's record variables. This step requires
+        nudged coordinates from combine.py and will fail with an informative
+        error if they are not present (as in missions without GPS data).
+        This format is analogous to the .nc4 files produced by the LRAUV
+        unserialize process. These are the best files to use for the highest
+        temporal resolution of the data. Unlike the .nc4 files, align.py's
+        output files use a naming convention rather than netCDF4 groups for
+        each instrument.
+
+    resample.py
+        Produce a netCDF file with all of the instrument's record variables
+        resampled to the same temporal interval. The coordinate variables are
+        also resampled to the same temporal interval and named with standard
+        depth, latitude, and longitude names. These are the best files to
+        use for loading data into STOQS and for analyses requiring all the
+        data to be on the same spatial temporal grid.
+
+    archive.py
+        Copy the netCDF files to the archive directory. The archive directory
+        is initially in the AUVCTD share on atlas which is shared with the
+        data from the Dorado Gulper vehicle, but can also be on the M3 share
+        on thalassa near the original log data.

README.md

@@ -59,8 +59,9 @@ print out the usage information for each of the processing scripts:
     uv run src/data/archive.py --help  
     uv run src/data/process_i2map.py --help  
     uv run src/data/process_dorado.py --help  
+    uv run src/data/process_lrauv.py --help  
 
-See [WORKFLOW.md](WORKFLOW.md) for more details on the data processing workflow.
+See [DORADO_WORKFLOW.md](DORADO_WORKFLOW.md) and [LRAUV_WORKFLOW.md](LRAUV_WORKFLOW.md) for more details on the data processing workflows.
 
 ### Jupyter Notebooks ###
 To run the Jupyter Notebooks, start Jupyter Lab at the command line with:
@@ -103,11 +104,11 @@ First time use with Docker on a server using a service account:
 * git clone git@github.com:mbari-org/auv-python.git
 * cd auv-python
 * Create a .env file in `/opt/auv-python` with the following contents:   
-    `M3_VOL=<mount_location>`   
-    `AUVCTD_VOL=<mount_location>`   
-    `CALIBRATION_VOL=<mount_location>`   
-    `WORK_VOL=<auv-python_home>/data`   
-
+    `M3_VOL=<mount_location>`
+    `AUVCTD_VOL=<mount_location>`
+    `CALIBRATION_VOL=<mount_location>`
+    `WORK_VOL=<auv-python_home>/data`
+    `HOST_NAME=<name_of_host>`
 After installation and when logging into the server again mission data can be processed thusly:
 * Setting up environment and printing help message:   
     `sudo -u docker_user -i`  
@@ -118,6 +119,10 @@ After installation and when logging into the server again mission data can be pr
     `docker compose run --rm auvpython src/data/process_i2map.py --help`   
 * To actually process a mission and have the processed data copied to the archive use the `-v` and `--clobber` options, e.g.:   
     `docker compose run --rm auvpython src/data/process_dorado.py --mission 2025.139.04 -v --clobber --noinput`   
+* To process LRAUV data for a specific vehicle and time range:   
+    `docker compose run --rm auvpython src/data/process_lrauv.py --auv_name tethys --start 20250401T000000 --end 20250502T000000 -v --noinput`   
+* To process a specific LRAUV log file:   
+    `docker compose run --rm auvpython src/data/process_lrauv.py --log_file tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4 -v --noinput`   
 
 
 --