diff --git a/.vscode/launch.json b/.vscode/launch.json index 6f95cea..76e8fa9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -136,7 +136,9 @@ // Fails in nudge_positions, maybe bad GPS data? //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250911T073640/202509110736_202509120809.nc4", "--plot"] // Nudged coordinate is way out of reasonable range - segment 15 - "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"] + //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250909_20250915/20250913T080940/202509130809_202509140809.nc4", "--plot"] + // No GPS data for a log_file that has an ESP Sample + "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--plot"] }, @@ -373,7 +375,7 @@ // ValueError: Dimension mismatch: wetlabsubat_digitized_raw_ad_counts_time has 33154 elements but wetlabsubat_hv_step_calibration_coefficient_time has 33155 elements //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T041517/202506260415_202506261400.nc4", "--no_cleanup"] // ValueError: coords is not dict-like, but it has 1 items, which does not match the 2 dimensions of the data - "args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"] + //"args": ["-v", "1", "--log_file", "pontus/missionlogs/2025/20250623_20250707/20250626T140000/202506261400_202506262031.nc4", "--no_cleanup"] // Full month of June 2025 for Pontus with WetLabsUBAT Group data //"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--no_cleanup"] //"args": ["-v", "1", "--auv_name", "pontus", "--start", "20250601T000000", "--end", "20250721T000000", "--noinput", "--num_cores", "1", "--no_cleanup", "--clobber"] @@ -389,6 +391,8 @@ //"args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250903_20250903/20250903T175939/202509031759_202509032026.nc4", "--no_cleanup"] // For loading stoqs_lrauv_sep2025 - which has really bad nav in _2S_scieng.nc files //"args": ["-v", "1", "--auv_name", "brizo", "--start", "20250901T000000", "--end", "20251001T000000", "--no_cleanup"] + // No GPS data for a log_file that has an ESP Sample + "args": ["-v", "1", "--log_file", "brizo/missionlogs/2025/20250916_20250922/20250920T070029/202509200700_202509201900.nc4", "--no_cleanup"] }, ] diff --git a/src/data/combine.py b/src/data/combine.py index 663c712..791e26b 100755 --- a/src/data/combine.py +++ b/src/data/combine.py @@ -868,60 +868,80 @@ def _initial_coordinate_qc(self) -> None: ) def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: - """Add nudged longitude and latitude variables to the combined dataset.""" + """Add nudged longitude and latitude variables to the combined dataset. + + If GPS fixes are available, positions are nudged to GPS. Otherwise, + dead-reckoned positions are used with appropriate warnings in metadata. + """ self._initial_coordinate_qc() # Check if GPS fix variables exist - if ( - "nal9602_longitude_fix" not in self.combined_nc - or "nal9602_latitude_fix" not in self.combined_nc - ): + has_gps = ( + "nal9602_longitude_fix" in self.combined_nc + and "nal9602_latitude_fix" in self.combined_nc + ) + + if not has_gps: self.logger.warning( "No GPS fix variables found in combined dataset - " - "skipping nudged coordinate creation" + "using uncorrected dead-reckoned positions for nudged coordinates" ) - return - - # Ensure GPS fixes have monotonically increasing timestamps - gps_lon = self.combined_nc["nal9602_longitude_fix"] - gps_lat = self.combined_nc["nal9602_latitude_fix"] - gps_time_coord = gps_lon.coords[gps_lon.dims[0]] + # Use dead-reckoned positions directly + nudged_longitude = self.combined_nc["universals_longitude"].to_numpy() + nudged_latitude = self.combined_nc["universals_latitude"].to_numpy() + segment_count = 0 + gps_corrected = False + else: + # Ensure GPS fixes have monotonically increasing timestamps + gps_lon = self.combined_nc["nal9602_longitude_fix"] + gps_lat = self.combined_nc["nal9602_latitude_fix"] + gps_time_coord = gps_lon.coords[gps_lon.dims[0]] + + # Convert to pandas index which handles datetime comparisons properly + gps_time_index = gps_time_coord.to_index() + gps_monotonic = monotonic_increasing_time_indices(gps_time_index) + if not np.all(gps_monotonic): + monotonic_count = np.sum(gps_monotonic) + self.logger.warning( + "Filtered GPS fixes from %d to %d to ensure monotonically " + "increasing timestamps", + len(gps_lon), + monotonic_count, + ) + gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic}) + gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic}) - # Convert to pandas index which handles datetime comparisons properly - gps_time_index = gps_time_coord.to_index() - gps_monotonic = monotonic_increasing_time_indices(gps_time_index) - if not np.all(gps_monotonic): - monotonic_count = np.sum(gps_monotonic) - self.logger.warning( - "Filtered GPS fixes from %d to %d to ensure monotonically increasing timestamps", - len(gps_lon), - monotonic_count, + try: + nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions( + nav_longitude=self.combined_nc["universals_longitude"], + nav_latitude=self.combined_nc["universals_latitude"], + gps_longitude=gps_lon, + gps_latitude=gps_lat, + logger=self.logger, + auv_name="", + mission="", + log_file=self.log_file, + max_sec_diff_at_end=max_sec_diff_at_end, + create_plots=self.plot, + ) + gps_corrected = True + except ValueError: + self.logger.exception("Nudging positions failed - using uncorrected positions") + nudged_longitude = self.combined_nc["universals_longitude"].to_numpy() + nudged_latitude = self.combined_nc["universals_latitude"].to_numpy() + segment_count = 0 + gps_corrected = False + + if gps_corrected: + self.logger.info( + "nudge_positions created %d segments with segment_minsum = %f", + segment_count, + segment_minsum, ) - gps_lon = gps_lon.isel({gps_lon.dims[0]: gps_monotonic}) - gps_lat = gps_lat.isel({gps_lat.dims[0]: gps_monotonic}) - - try: - nudged_longitude, nudged_latitude, segment_count, segment_minsum = nudge_positions( - nav_longitude=self.combined_nc["universals_longitude"], - nav_latitude=self.combined_nc["universals_latitude"], - gps_longitude=gps_lon, - gps_latitude=gps_lat, - logger=self.logger, - auv_name="", - mission="", - log_file=self.log_file, - max_sec_diff_at_end=max_sec_diff_at_end, - create_plots=self.plot, + else: + self.logger.warning( + "Nudged coordinates are uncorrected dead-reckoned positions - use with caution" ) - except ValueError as e: - self.logger.error("Nudging positions failed: %s", e) # noqa: TRY400 - return - - self.logger.info( - "nudge_positions created %d segments with segment_minsum = %f", - segment_count, - segment_minsum, - ) # Calculate total underwater time and store for metadata time_coord = self.combined_nc[self.variable_time_coord_mapping["universals_longitude"]] @@ -931,6 +951,20 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: self.nudge_segment_count = segment_count self.nudge_total_minutes = total_seconds / 60.0 + # Create comment based on whether GPS correction was applied + if gps_corrected: + lon_comment = ( + f"Dead reckoned positions from {segment_count} underwater segments " + f"nudged to GPS positions" + ) + lat_comment = lon_comment + else: + lon_comment = ( + "WARNING: Uncorrected dead-reckoned positions. No GPS fixes available. " + "These positions have not been adjusted for drift and should be used with caution." + ) + lat_comment = lon_comment + self.combined_nc["nudged_longitude"] = xr.DataArray( nudged_longitude, coords=[ @@ -945,10 +979,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: "long_name": "Nudged Longitude", "standard_name": "longitude", "units": "degrees_east", - "comment": ( - f"Dead reckoned positions from {segment_count} underwater segments " - f"nudged to GPS positions" - ), + "comment": lon_comment, + "gps_corrected": "true" if gps_corrected else "false", } self.combined_nc["nudged_latitude"] = xr.DataArray( nudged_latitude, @@ -962,10 +994,8 @@ def _add_nudged_coordinates(self, max_sec_diff_at_end: int = 10) -> None: "long_name": "Nudged Latitude", "standard_name": "latitude", "units": "degrees_north", - "comment": ( - f"Dead reckoned positions from {segment_count} underwater segments " - f"nudged to GPS positions" - ), + "comment": lat_comment, + "gps_corrected": "true" if gps_corrected else "false", } def combine_groups(self) -> None: @@ -975,7 +1005,7 @@ def combine_groups(self) -> None: self.summary_fields = set() self.combined_nc = xr.Dataset() - for group_file in group_files: + for group_file in sorted(group_files): self.logger.info("-" * 110) self.logger.info("Group file: %s", group_file.name) # Open group file without decoding to have np.allclose work properly