Skip to content

Handle LRAUV log files without GPS data by creating nudged coordinates from dead-reckoned positions #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 1 commit into from
Dec 10, 2025
Merged

Handle LRAUV log files without GPS data by creating nudged coordinates from dead-reckoned positions #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,9 @@
// Fails in nudge_positions, maybe bad GPS data?
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250911T073640/202509110736_202509120809.nc4", "--plot"]
// Nudged coordinate is way out of reasonable range - segment 15
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"]
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"]
// No GPS data for a log_file that has an ESP Sample
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--plot"]


},
Expand Down Expand Up @@ -373,7 +375,7 @@
// ValueError: Dimension mismatch: wetlabsubat_digitized_raw_ad_counts_time has 33154 elements but wetlabsubat_hv_step_calibration_coefficient_time has 33155 elements
//"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T041517/202506260415_202506261400.nc4", "--no_cleanup"]
// ValueError: coords is not dict-like, but it has 1 items, which does not match the 2 dimensions of the data
"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"]
//"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"]
// Full month of June 2025 for Pontus with WetLabsUBAT Group data
//"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--no_cleanup"]
//"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--noinput", "--num_cores", "1", "--no_cleanup", "--clobber"]
Expand All @@ -389,6 +391,8 @@
//"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250903_20250903/20250903T175939/202509031759_202509032026.nc4", "--no_cleanup"]
// For loading stoqs_lrauv_sep2025 - which has really bad nav in _2S_scieng.nc files
//"args": ["-v", "1", "--auv_name", "brizo", "--start", "20250901T000000", "--end", "20251001T000000", "--no_cleanup"]
// No GPS data for a log_file that has an ESP Sample
"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--no_cleanup"]
},

]
Expand Down
138 changes: 84 additions & 54 deletions src/data/combine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,60 +868,80 @@ def _initial_coordinate_qc(self) -> None:
)

def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None:
"""Add nudged longitude and latitude variables to the combined dataset."""
"""Add nudged longitude and latitude variables to the combined dataset.

If GPS fixes are available, positions are nudged to GPS. Otherwise,
dead-reckoned positions are used with appropriate warnings in metadata.
"""
self._initial_coordinate_qc()

# Check if GPS fix variables exist
if (
"nal9602_longitude_fix" not in self.combined_nc
or "nal9602_latitude_fix" not in self.combined_nc
):
has_gps = (
"nal9602_longitude_fix" in self.combined_nc
and "nal9602_latitude_fix" in self.combined_nc
)

if not has_gps:
self.logger.warning(
"No GPS fix variables found in combined dataset - "
"skipping nudged coordinate creation"
"using uncorrected dead-reckoned positions for nudged coordinates"
)
return

# Ensure GPS fixes have monotonically increasing timestamps
gps_lon = self.combined_nc["nal9602_longitude_fix"]
gps_lat = self.combined_nc["nal9602_latitude_fix"]
gps_time_coord = gps_lon.coords[gps_lon.dims[0]]
# Use dead-reckoned positions directly
nudged_longitude = self.combined_nc["universals_longitude"].to_numpy()
nudged_latitude = self.combined_nc["universals_latitude"].to_numpy()
segment_count = 0
gps_corrected = False
else:
# Ensure GPS fixes have monotonically increasing timestamps
gps_lon = self.combined_nc["nal9602_longitude_fix"]
gps_lat = self.combined_nc["nal9602_latitude_fix"]
gps_time_coord = gps_lon.coords[gps_lon.dims[0]]

# Convert to pandas index which handles datetime comparisons properly
gps_time_index = gps_time_coord.to_index()
gps_monotonic = monotonic_increasing_time_indices(gps_time_index)
if not np.all(gps_monotonic):
monotonic_count = np.sum(gps_monotonic)
self.logger.warning(
"Filtered GPS fixes from %d to %d to ensure monotonically "
"increasing timestamps",
len(gps_lon),
monotonic_count,
)
gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic})
gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic})

# Convert to pandas index which handles datetime comparisons properly
gps_time_index = gps_time_coord.to_index()
gps_monotonic = monotonic_increasing_time_indices(gps_time_index)
if not np.all(gps_monotonic):
monotonic_count = np.sum(gps_monotonic)
self.logger.warning(
"Filtered GPS fixes from %d to %d to ensure monotonically increasing timestamps",
len(gps_lon),
monotonic_count,
try:
nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions(
nav_longitude=self.combined_nc["universals_longitude"],
nav_latitude=self.combined_nc["universals_latitude"],
gps_longitude=gps_lon,
gps_latitude=gps_lat,
logger=self.logger,
auv_name="",
mission="",
log_file=self.log_file,
max_sec_diff_at_end=max_sec_diff_at_end,
create_plots=self.plot,
)
gps_corrected = True
except ValueError:
self.logger.exception("Nudging positions failed - using uncorrected positions")
nudged_longitude = self.combined_nc["universals_longitude"].to_numpy()
nudged_latitude = self.combined_nc["universals_latitude"].to_numpy()
segment_count = 0
gps_corrected = False

if gps_corrected:
self.logger.info(
"nudge_positions created %d segments with segment_minsum = %f",
segment_count,
segment_minsum,
)
gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic})
gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic})

try:
nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions(
nav_longitude=self.combined_nc["universals_longitude"],
nav_latitude=self.combined_nc["universals_latitude"],
gps_longitude=gps_lon,
gps_latitude=gps_lat,
logger=self.logger,
auv_name="",
mission="",
log_file=self.log_file,
max_sec_diff_at_end=max_sec_diff_at_end,
create_plots=self.plot,
else:
self.logger.warning(
"Nudged coordinates are uncorrected dead-reckoned positions - use with caution"
)
except ValueError as e:
self.logger.error("Nudging positions failed: %s", e) # noqa: TRY400
return

self.logger.info(
"nudge_positions created %d segments with segment_minsum = %f",
segment_count,
segment_minsum,
)

# Calculate total underwater time and store for metadata
time_coord = self.combined_nc[self.variable_time_coord_mapping["universals_longitude"]]
Expand All @@ -931,6 +951,20 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None:
self.nudge_segment_count = segment_count
self.nudge_total_minutes = total_seconds / 60.0

# Create comment based on whether GPS correction was applied
if gps_corrected:
lon_comment = (
f"Dead reckoned positions from {segment_count} underwater segments "
f"nudged to GPS positions"
)
lat_comment = lon_comment
else:
lon_comment = (
"WARNING: Uncorrected dead-reckoned positions. No GPS fixes available. "
"These positions have not been adjusted for drift and should be used with caution."
)
lat_comment = lon_comment

self.combined_nc["nudged_longitude"] = xr.DataArray(
nudged_longitude,
coords=[
Expand All @@ -945,10 +979,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None:
"long_name": "Nudged Longitude",
"standard_name": "longitude",
"units": "degrees_east",
"comment": (
f"Dead reckoned positions from {segment_count} underwater segments "
f"nudged to GPS positions"
),
"comment": lon_comment,
"gps_corrected": "true" if gps_corrected else "false",
}
self.combined_nc["nudged_latitude"] = xr.DataArray(
nudged_latitude,
Expand All @@ -962,10 +994,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None:
"long_name": "Nudged Latitude",
"standard_name": "latitude",
"units": "degrees_north",
"comment": (
f"Dead reckoned positions from {segment_count} underwater segments "
f"nudged to GPS positions"
),
"comment": lat_comment,
"gps_corrected": "true" if gps_corrected else "false",
}

def combine_groups(self) -> None:
Expand All @@ -975,7 +1005,7 @@ def combine_groups(self) -> None:
self.summary_fields = set()
self.combined_nc = xr.Dataset()

for group_file in group_files:
for group_file in sorted(group_files):
self.logger.info("-" * 110)
self.logger.info("Group file: %s", group_file.name)
# Open group file without decoding to have np.allclose work properly
Expand Down