From 2ac917916aafd4c30174facf65f5aec9a88382dc Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Wed, 10 Dec 2025 09:45:51 -0800 Subject: [PATCH] Handle LRAUV log files without GPS data by creating nudged coordinates from dead-reckoned positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modified combine.py to always create nudged_longitude and nudged_latitude variables, even when GPS fixes are unavailable. This allows important log files to proceed through the full processing pipeline (combine → align → resample) to produce _1S.nc files. Key changes to _add_nudged_coordinates(): - Check for GPS availability before attempting nudging - Fall back to uncorrected universals_longitude/latitude when GPS is missing or nudging fails - Add clear warnings in both console logs and metadata attributes - Include new 'gps_corrected' attribute ("true"/"false") for programmatic detection When GPS correction cannot be applied: - Console: WARNING messages alert user during processing - Metadata: Variable 'comment' includes explicit warning about uncorrected positions - Metadata: 'gps_corrected' attribute set to "false" for downstream tools to detect This ensures that scientifically valuable log files (e.g., those with ESP samples) can be processed even without GPS data, while clearly marking the position data quality limitations for end users. Tested with brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4 which has no GPS data but successfully processed through to _1S.nc file. --- .vscode/launch.json | 8 ++- src/data/combine.py | 138 +++++++++++++++++++++++++++----------------- 2 files changed, 90 insertions(+), 56 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 6f95cea..76e8fa9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -136,7 +136,9 @@ // Fails in nudge_positions, maybe bad GPS data? //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250911T073640/202509110736_202509120809.nc4", "--plot"] // Nudged coordinate is way out of reasonable range - segment 15 - "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"] + //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"] + // No GPS data for a log_file that has an ESP Sample + "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--plot"] }, @@ -373,7 +375,7 @@ // ValueError: Dimension mismatch: wetlabsubat_digitized_raw_ad_counts_time has 33154 elements but wetlabsubat_hv_step_calibration_coefficient_time has 33155 elements //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T041517/202506260415_202506261400.nc4", "--no_cleanup"] // ValueError: coords is not dict-like, but it has 1 items, which does not match the 2 dimensions of the data - "args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"] + //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"] // Full month of June 2025 for Pontus with WetLabsUBAT Group data //"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--no_cleanup"] //"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--noinput", "--num_cores", "1", "--no_cleanup", "--clobber"] @@ -389,6 +391,8 @@ //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250903_20250903/20250903T175939/202509031759_202509032026.nc4", "--no_cleanup"] // For loading stoqs_lrauv_sep2025 - which has really bad nav in _2S_scieng.nc files //"args": ["-v", "1", "--auv_name", "brizo", "--start", "20250901T000000", "--end", "20251001T000000", "--no_cleanup"] + // No GPS data for a log_file that has an ESP Sample + "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--no_cleanup"] }, ] diff --git a/src/data/combine.py b/src/data/combine.py index 663c712..791e26b 100755 --- a/src/data/combine.py +++ b/src/data/combine.py @@ -868,60 +868,80 @@ def _initial_coordinate_qc(self) -> None: ) def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: - """Add nudged longitude and latitude variables to the combined dataset.""" + """Add nudged longitude and latitude variables to the combined dataset. + + If GPS fixes are available, positions are nudged to GPS. Otherwise, + dead-reckoned positions are used with appropriate warnings in metadata. + """ self._initial_coordinate_qc() # Check if GPS fix variables exist - if ( - "nal9602_longitude_fix" not in self.combined_nc - or "nal9602_latitude_fix" not in self.combined_nc - ): + has_gps = ( + "nal9602_longitude_fix" in self.combined_nc + and "nal9602_latitude_fix" in self.combined_nc + ) + + if not has_gps: self.logger.warning( "No GPS fix variables found in combined dataset - " - "skipping nudged coordinate creation" + "using uncorrected dead-reckoned positions for nudged coordinates" ) - return - - # Ensure GPS fixes have monotonically increasing timestamps - gps_lon = self.combined_nc["nal9602_longitude_fix"] - gps_lat = self.combined_nc["nal9602_latitude_fix"] - gps_time_coord = gps_lon.coords[gps_lon.dims[0]] + # Use dead-reckoned positions directly + nudged_longitude = self.combined_nc["universals_longitude"].to_numpy() + nudged_latitude = self.combined_nc["universals_latitude"].to_numpy() + segment_count = 0 + gps_corrected = False + else: + # Ensure GPS fixes have monotonically increasing timestamps + gps_lon = self.combined_nc["nal9602_longitude_fix"] + gps_lat = self.combined_nc["nal9602_latitude_fix"] + gps_time_coord = gps_lon.coords[gps_lon.dims[0]] + + # Convert to pandas index which handles datetime comparisons properly + gps_time_index = gps_time_coord.to_index() + gps_monotonic = monotonic_increasing_time_indices(gps_time_index) + if not np.all(gps_monotonic): + monotonic_count = np.sum(gps_monotonic) + self.logger.warning( + "Filtered GPS fixes from %d to %d to ensure monotonically " + "increasing timestamps", + len(gps_lon), + monotonic_count, + ) + gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic}) + gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic}) - # Convert to pandas index which handles datetime comparisons properly - gps_time_index = gps_time_coord.to_index() - gps_monotonic = monotonic_increasing_time_indices(gps_time_index) - if not np.all(gps_monotonic): - monotonic_count = np.sum(gps_monotonic) - self.logger.warning( - "Filtered GPS fixes from %d to %d to ensure monotonically increasing timestamps", - len(gps_lon), - monotonic_count, + try: + nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions( + nav_longitude=self.combined_nc["universals_longitude"], + nav_latitude=self.combined_nc["universals_latitude"], + gps_longitude=gps_lon, + gps_latitude=gps_lat, + logger=self.logger, + auv_name="", + mission="", + log_file=self.log_file, + max_sec_diff_at_end=max_sec_diff_at_end, + create_plots=self.plot, + ) + gps_corrected = True + except ValueError: + self.logger.exception("Nudging positions failed - using uncorrected positions") + nudged_longitude = self.combined_nc["universals_longitude"].to_numpy() + nudged_latitude = self.combined_nc["universals_latitude"].to_numpy() + segment_count = 0 + gps_corrected = False + + if gps_corrected: + self.logger.info( + "nudge_positions created %d segments with segment_minsum = %f", + segment_count, + segment_minsum, ) - gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic}) - gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic}) - - try: - nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions( - nav_longitude=self.combined_nc["universals_longitude"], - nav_latitude=self.combined_nc["universals_latitude"], - gps_longitude=gps_lon, - gps_latitude=gps_lat, - logger=self.logger, - auv_name="", - mission="", - log_file=self.log_file, - max_sec_diff_at_end=max_sec_diff_at_end, - create_plots=self.plot, + else: + self.logger.warning( + "Nudged coordinates are uncorrected dead-reckoned positions - use with caution" ) - except ValueError as e: - self.logger.error("Nudging positions failed: %s", e) # noqa: TRY400 - return - - self.logger.info( - "nudge_positions created %d segments with segment_minsum = %f", - segment_count, - segment_minsum, - ) # Calculate total underwater time and store for metadata time_coord = self.combined_nc[self.variable_time_coord_mapping["universals_longitude"]] @@ -931,6 +951,20 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: self.nudge_segment_count = segment_count self.nudge_total_minutes = total_seconds / 60.0 + # Create comment based on whether GPS correction was applied + if gps_corrected: + lon_comment = ( + f"Dead reckoned positions from {segment_count} underwater segments " + f"nudged to GPS positions" + ) + lat_comment = lon_comment + else: + lon_comment = ( + "WARNING: Uncorrected dead-reckoned positions. No GPS fixes available. " + "These positions have not been adjusted for drift and should be used with caution." + ) + lat_comment = lon_comment + self.combined_nc["nudged_longitude"] = xr.DataArray( nudged_longitude, coords=[ @@ -945,10 +979,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: "long_name": "Nudged Longitude", "standard_name": "longitude", "units": "degrees_east", - "comment": ( - f"Dead reckoned positions from {segment_count} underwater segments " - f"nudged to GPS positions" - ), + "comment": lon_comment, + "gps_corrected": "true" if gps_corrected else "false", } self.combined_nc["nudged_latitude"] = xr.DataArray( nudged_latitude, @@ -962,10 +994,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: "long_name": "Nudged Latitude", "standard_name": "latitude", "units": "degrees_north", - "comment": ( - f"Dead reckoned positions from {segment_count} underwater segments " - f"nudged to GPS positions" - ), + "comment": lat_comment, + "gps_corrected": "true" if gps_corrected else "false", } def combine_groups(self) -> None: @@ -975,7 +1005,7 @@ def combine_groups(self) -> None: self.summary_fields = set() self.combined_nc = xr.Dataset() - for group_file in group_files: + for group_file in sorted(group_files): self.logger.info("-" * 110) self.logger.info("Group file: %s", group_file.name) # Open group file without decoding to have np.allclose work properly