Skip to content

Metadata fixups and misc. fixes for problems encountered by more end-to-end testing. #71

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 14 commits into from
Nov 26, 2025
Merged

Metadata fixups and misc. fixes for problems encountered by more end-to-end testing. #71

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
4c8593b
Add _align_root_group_coordinates() and fix --plot_time.
MBARIMike Nov 25, 2025
2e0be4b
Add ctdneilbrown for early tethys log_files, other resample() fixes.
MBARIMike Nov 25, 2025
add6b51
Set args for running combine.py individually.
MBARIMike Nov 25, 2025
a3e3487
Metadata fixups: Add deployment name, specific summary info, processi…
MBARIMike Nov 25, 2025
6325a01
Skip writing dynamic metadata if running from pytest, add arguments n…
MBARIMike Nov 25, 2025
95133f7
Remove ending period from summary text.
MBARIMike Nov 25, 2025
060a8b3
Skip metadata if running from pytest, remove summary note about furth…
MBARIMike Nov 25, 2025
f65e6c3
Use seg_count rather than i for the segment numbers.
MBARIMike Nov 26, 2025
a9ccd0d
Remove trailing period from summary, which later gets a period.
MBARIMike Nov 26, 2025
a411bd3
Set instance attributes from parsed arguments, use OrderedDict for su…
MBARIMike Nov 26, 2025
9a42001
Remove summary notes not needed after align step.
MBARIMike Nov 26, 2025
445e144
Use (simply) the summary attribute from the align file.
MBARIMike Nov 26, 2025
e4309b0
Testing for metadata fixups.
MBARIMike Nov 26, 2025
9ecc3d8
Update EXPECTED_SIZE_s.
MBARIMike Nov 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 19 additions & 10 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
//"args": ["--auv_name", "i2map", "--mission", "2020.055.01", "--noinput", "--local", "-v", "2", "--clobber"]
//"args": ["--auv_name", "Dorado389", "--mission", "2020.245.00", "--noinput", "-v", "2", "--portal", "http://stoqs.mbari.org:8080/auvdata/v1", "--clobber"]
//"args": ["--auv_name", "Dorado389", "--mission", "2020.245.00", "--noinput", "-v"]
"args": ["--auv_name", "dorado", "--mission", "2017.297.00", "-v", "2", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"]
//"args": ["--auv_name", "dorado", "--mission", "2017.297.00", "-v", "1", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"]
//"args": ["--auv_name", "Dorado389", "--start", "20190701", "--end", "20191230", "-v", "2"]
//"args": ["--auv_name", "i2map", "--mission", "2021.062.01", "--noinput", "-v", "1"]
//"args": ["--auv_name", "dorado", "--mission", "2021.109.00", "--noinput", "-v"]
Expand All @@ -26,7 +26,7 @@
//"args": ["--auv_name", "dorado", "--mission", "2010.265.00", "--noinput", "-v"]
//"args": ["--auv_name", "dorado", "--mission", "2023.324.00", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs"]
// Mission suffering from GPS Rollover bug. Add 1024 * 7 * 24 * 3600 = 619315200 seconds
//"args": ["--auv_name", "dorado", "--mission", "2025.316.02", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs", "--add_seconds", "619315200" ]
"args": ["--auv_name", "dorado", "--mission", "2025.316.02", "--noinput", "-v", "--vehicle_dir", "/Volumes/AUVCTD/missionlogs", "--add_seconds", "619315200" ]
},
{
"name": "1.1 - lopcToNetCDF",
Expand Down Expand Up @@ -61,9 +61,11 @@
//"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--plot_time", "/longitude_time"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--plot_time", "/latitude_time"]
// brizo 20250916T230652 has several ESP Samples from stoqs_lrauv_sep2025
"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--plot_time", "/longitude_time"]
//"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--plot_time", "/longitude_time"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109_cleaned_by_quinn.nc4", "--plot_time", "/longitude_time"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109_cleaned_by_quinn_latlon.nc4", "--plot_time", "/longitude_time"]
// Conflicting sizes for nudged_time and data
"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--plot_time", "/longitude_time"]
},
{
"name": "2.0 - calibrate.py",
Expand All @@ -84,7 +86,7 @@
//"args": ["--auv_name", "dorado", "--mission", "2010.181.00", "--plot", "first1000", "-v", "1"]
// OverflowError: time values outside range of 64 bit signed integers in calibrate.py:413
//"args": ["--auv_name", "dorado", "--mission", "2017.304.00", "--plot", "first1000", "-v", "1"]
"args": ["--auv_name", "dorado", "--mission", "2017.297.00", "-v", "1"]
//"args": ["--auv_name", "dorado", "--mission", "2017.297.00", "-v", "1"]
//"args": ["--auv_name", "i2map", "--mission", "2022.094.01", "-v", "2"]
//"args": ["--auv_name", "i2map", "--mission", "2018.025.00", "-v", "2"]
//"args": ["--auv_name", "dorado", "--mission", "2017.248.01", "-v", "1"]
Expand All @@ -103,6 +105,7 @@
//"args": ["--auv_name", "i2map", "--mission", "2018.348.01", "-v", "2"]
//"args": ["--auv_name", "dorado", "--mission", "2023.324.00", "-v", "1", "--plot", "first10000"]
//"args": ["--auv_name", "dorado", "--mission", "2022.201.00", "-v", "1", "--plot", "first10000"]
"args": ["--auv_name", "dorado", "--mission", "2025.316.02", "-v", "1"]
},
{
"name": "2.1 - Test hs2_proc.py (its unit tests)",
Expand All @@ -122,6 +125,9 @@
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--plot"]
//"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120909T010636/201209090106_201209091521.nc4", "--plot"]
// Conflicting sizes for nudged_time and data
//"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--plot"]

},
{
"name": "3.0 - align.py",
Expand All @@ -143,7 +149,8 @@
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2008.289.03"],
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2023.192.01"],
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2024.317.01"],
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2025.316.02"],
},
{
"name": "3.1 - align.py for LRAUV --log_file",
Expand Down Expand Up @@ -183,7 +190,8 @@
//"args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"],
//"args": ["--auv_name", "dorado", "--mission", "2020.337.00", "-v", "1"],
//"args": ["--auv_name", "dorado", "--mission", "2023.123.00", "-v", "1"],
"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
//"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4"]
"args": ["--auv_name", "dorado", "--mission", "2025.316.02", "-v", "1"],
},
{
"name": "5.0 - archive.py",
Expand Down Expand Up @@ -301,7 +309,7 @@
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--start_year", "2007", "--end_year", "2007", "--create_products", "--num_cores", "1", "--archive_only_products"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--start_year", "2007", "--end_year", "2007", "--start_yd", "171", "--end_yd", "171", "--num_cores", "1", "--create_products", "--archive_only_products"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2008.261.01", "--create_products", "--archive", "--archive_only_products"]
"args": ["-v", "1", "--noinput", "--no_cleanup", "--start_year", "2011", "--end_year", "2011", "--start_yd", "158", "--end_yd", "164", "--num_cores", "1"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--start_year", "2011", "--end_year", "2011", "--start_yd", "158", "--end_yd", "164", "--num_cores", "1"]
//"args": ["-v", "1", "--noinput", "--start_year", "2016", "--end_year", "2016", "--start_yd", "270", "--end_yd", "270", "--num_cores", "1", "--create_products", "--archive", "--archive_only_products"]
//"args": ["-v", "1", "--noinput", "--num_cores", "1", "--mission", "2023.285.01"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2018.079.00"]
Expand All @@ -315,7 +323,7 @@
//"args": ["-v", "2", "--mission", "2004.029.03", "--noinput", "--no_cleanup"],
//"args": ["-v", "1", "--mission", "2023.192.01", "--noinput", "--no_cleanup"],
//"args": ["-v", "1", "--mission", "2010.151.04", "--noinput", "--no_cleanup", "--clobber"],
//"args": ["-v", "1", "--mission", "2025.316.02", "--noinput", "--no_cleanup", "--add_seconds", "619315200"],
"args": ["-v", "1", "--mission", "2025.316.02", "--noinput", "--no_cleanup", "--add_seconds", "619315200"],

},
{
Expand All @@ -332,13 +340,14 @@
"request": "launch",
"program": "${workspaceFolder}/src/data/process_lrauv.py",
"console": "integratedTerminal",
// Lots bad time values in brizo 20250914T080941 due to memory corruption on the vehicle
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4"]
//"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--clobber"]
//"args": ["-v", "2", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250914T080941/202509140809_202509150109.nc4", "--clobber", "--no_cleanup"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--no_cleanup"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--no_cleanup", "--clobber"]
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250916T230652/202509162306_202509180305.nc4", "--no_cleanup", "--clobber"]
// Has different universals time coodinates for longitude/latitude and depth
"args": ["-v", "1", "--auv_name", "tethys", "--start", "20120901T000000", "--end", "20121101T000000", "--noinput", "--no_cleanup"]
//"args": ["-v", "1", "--auv_name", "tethys", "--start", "20120901T000000", "--end", "20121101T000000", "--noinput", "--no_cleanup"]
// Conflicting sizes for nudged_time and data
//"args": ["-v", "1", "--log_file", "tethys/missionlogs/2012/20120908_20120920/20120917T025522/201209170255_201209171110.nc4", "--no_cleanup"
//"args": ["-v", "1", "--auv_name", "brizo", "--start", "20250915T000000", "--end", "20250917T000000", "--noinput", "--num_cores", "1", "--no_cleanup"]
Expand Down
4 changes: 2 additions & 2 deletions src/data/AUV.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,10 +207,11 @@ def nudge_positions( # noqa: C901, PLR0912, PLR0913, PLR0915

if len(segi) > MIN_SEGMENT_LENGTH:
logger.info(
f"{i:5d}: {end_sec_diff:12.3f} {end_lon_diff:12.7f}" # noqa: G004
f"{seg_count:5d}: {end_sec_diff:12.3f} {end_lon_diff:12.7f}" # noqa: G004
f" {end_lat_diff:12.7f} {len(segi):-9d} {seg_min:9.2f}"
f" {u_drift:14.3f} {v_drift:14.3f} {lat.cf['T'].data[segi][-1]}",
)
seg_count += 1

# Start with zero adjustment at beginning and linearly ramp up to the diff at the end
lon_nudge = np.interp(
Expand Down Expand Up @@ -253,7 +254,6 @@ def nudge_positions( # noqa: C901, PLR0912, PLR0913, PLR0915
lon_nudged_array = np.append(lon_nudged_array, lon[segi] + lon_nudge)
lat_nudged_array = np.append(lat_nudged_array, lat[segi] + lat_nudge)
dt_nudged = np.append(dt_nudged, lon.cf["T"].data[segi])
seg_count += 1

# Any dead reckoned points after last GPS fix
segi = np.where(lat.cf["T"].data > lat_fix.cf["T"].data[-1])[0]
Expand Down
61 changes: 45 additions & 16 deletions src/data/align.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from common_args import get_standard_lrauv_parser
from logs2netcdfs import AUV_NetCDF, MISSIONNETCDFS, SUMMARY_SOURCE, TIME, TIME60HZ
from nc42netcdfs import BASE_LRAUV_PATH
from utils import get_deployment_name


class InvalidCalFile(Exception):
Expand Down Expand Up @@ -83,6 +84,11 @@ def global_metadata(self) -> dict: # noqa: PLR0915
"""Use instance variables to return a dictionary of
metadata specific for the data that are written
"""
# Skip dynamic metadata during testing to ensure reproducible results
if "pytest" in sys.modules:
self.logger.debug("Skipping dynamic metadata generation (running under pytest)")
return {}

auv_name = self.auv_name
mission = self.mission
log_file = self.log_file
Expand Down Expand Up @@ -140,30 +146,53 @@ def global_metadata(self) -> dict: # noqa: PLR0915
f" host {actual_hostname} using git commit {gitcommit} from"
f" software at 'https://github.com/mbari-org/auv-python'"
)
metadata["summary"] = (
"Observational oceanographic data obtained from an Autonomous"
" Underwater Vehicle mission with measurements at"
" original sampling intervals. The data have been calibrated"
" and the coordinate variables aligned using MBARI's auv-python"
" software."
metadata["summary"] = self.calibrated_nc.attrs.get(
"summary",
(
"Observational oceanographic data obtained from an Autonomous"
" Underwater Vehicle mission with measurements at"
" original sampling intervals. The data have been calibrated"
" and the coordinate variables aligned using MBARI's auv-python"
" software."
),
)
elif log_file:
metadata["title"] = (
f"Combined and aligned LRAUV instrument data from log file {Path(log_file)}"
# Remove notes not needed after align step
metadata["summary"] = metadata["summary"].replace(
" These data have been processed from the original lopc.bin file produced by the LOPC instrument.", # noqa: E501
"",
)
metadata["summary"] = metadata["summary"].replace(
" The data in this file are to be considered as simple time series data only and are as close to the original data as possible.", # noqa: E501
"",
)
metadata["summary"] = metadata["summary"].replace(
" Further processing is required to turn the data into a time series of profiles.",
"",
)
elif log_file:
# Build title with optional deployment name
title = f"Combined and aligned LRAUV instrument data from log file {Path(log_file)}"
deployment_name = get_deployment_name(log_file, BASE_LRAUV_PATH, self.logger)
if deployment_name:
title += f" - Deployment: {deployment_name}"
metadata["title"] = title

from_data = "combined data"
metadata["source"] = (
f"MBARI Long Range AUV data produced from {from_data}"
f" with execution of '{self.commandline}' at {iso_now} on"
f" host {actual_hostname} using git commit {gitcommit} from"
f" software at 'https://github.com/mbari-org/auv-python'"
)
metadata["summary"] = (
"Observational oceanographic data obtained from an Autonomous"
" Underwater Vehicle mission with measurements at"
" original sampling intervals. The position variables have been"
" corrected to GPS positions and aligned with the data variables"
" using MBARI's auv-python software."
metadata["summary"] = self.combined_nc.attrs.get(
"summary",
(
"Observational oceanographic data obtained from an Autonomous"
" Underwater Vehicle mission with measurements at"
" original sampling intervals. The position variables have been"
" corrected to GPS positions and aligned with the data variables"
" using MBARI's auv-python software."
),
)
# Append location of original data files to summary
if self.auv_name and self.mission:
Expand Down Expand Up @@ -678,7 +707,7 @@ def write_combined_netcdf(self, netcdfs_dir: Path) -> None:
self.logger.debug("Removing existing file %s", out_fn)
out_fn.unlink()
self.aligned_nc.to_netcdf(out_fn)
self.logger.info(
self.logger.debug(
"Data variables written: %s",
", ".join(sorted(self.aligned_nc.variables)),
)
Expand Down
Loading