From f525ebbecfe2a96a4788b2d25021a4444fe7ef02 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Mon, 24 Mar 2025 09:32:42 -0700 Subject: [PATCH] Improve select_nighttime_bl_raw() to return all nighttime data. --- .vscode/launch.json | 9 +++-- src/data/resample.py | 85 +++++++++++++++++++------------------------- 2 files changed, 42 insertions(+), 52 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 270fd31a..61943baa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -138,7 +138,8 @@ //"args": ["--auv_name", "i2map", "--mission", "2019.157.02", "-v", "2", "--plot", "--plot_seconds", "82000"], //"args": ["--auv_name", "dorado", "--mission", "2021.102.02", "-v", "1", "--flash_threshold", "1.5e10"], //"args": ["--auv_name", "dorado", "--mission", "2024.317.01", "-v", "1"], - "args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"], + //"args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"], + "args": ["--auv_name", "dorado", "--mission", "2020.337.00", "-v", "1"], }, { "name": "5.0 - archive.py", @@ -151,7 +152,8 @@ //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2023.192.01", "--AUVCTD"], //"args": ["-v", "1", "--auv_name", "i2map", "--mission", "2022.061.01", "--M3", "--AUVCTD"], //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2024.317.01", "--AUVCTD"], - "args": ["-v", "1", "--auv_name", "dorado", "--mission", "2022.201.00", "--AUVCTD"], + //"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2022.201.00", "--AUVCTD"], + "args": ["-v", "1", "--auv_name", "dorado", "--mission", "2020.337.00", "--AUVCTD", "--clobber"], }, { "name": "6.0 - create_products.py", @@ -224,7 +226,7 @@ //"args": ["-v", "1", "--last_n_days", "30", "--start_year", "2022", "--end_year", "2022", "--resample", "--noinput"] //"args": ["-v", "1", "--last_n_days", "30", "--start_year", "2022", "--end_year", "2022", "--archive", "--noinput"] //"args": ["-v", "1", "--start_year", "2021", "--end_year", "2022", "--noinput"] - "args": ["-v", "1", "--mission", "2022.201.00", "--clobber", "--noinput", "--no_cleanup"] + //"args": ["-v", "1", "--mission", "2022.201.00", "--clobber", "--noinput", "--no_cleanup"] //"args": ["-v", "1", "--mission", "2009.084.00", "--clobber", "--noinput"] //"args": ["-v", "1", "--mission", "2022.243.00", "--calibrate", "--clobber", "--noinput"] //"args": ["-v", "1", "--start_year", "2004", "--end_year", "2004", "--start_yd", "168", "--use_portal", "--clobber", "--noinput"] @@ -263,6 +265,7 @@ //"args": ["-v", "1", "--noinput", "--num_cores", "8"] //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2011.256.02", "--clobber"] //"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2010.341.00", "--download_process", "--local"] + "args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2020.337.00", "--local"] }, ] } diff --git a/src/data/resample.py b/src/data/resample.py index 188f6867..162a5028 100755 --- a/src/data/resample.py +++ b/src/data/resample.py @@ -353,17 +353,22 @@ def save_coordinates( def select_nighttime_bl_raw( self, stride: int = 3000, - ) -> tuple[pd.Series, datetime, datetime]: - # Select only the nighttime biolume_raw data, - # one hour past sunset to one before sunrise - # Default stride of 3000 give 10 minute resolution - # from 5 hz navigation data + ) -> tuple[pd.Series, list[datetime], list[datetime]]: + """ + Select nighttime biolume_raw data for multiple nights in a mission. + Default stride of 3000 gives 10-minute resolution from 5 Hz navigation data. + + Returns: + nighttime_bl_raw: A pandas Series containing nighttime biolume_raw data. + sunsets: A list of sunset times for each night. + sunrises: A list of sunrise times for each night. + """ lat = float(self.ds["navigation_latitude"].median()) lon = float(self.ds["navigation_longitude"].median()) self.logger.debug("Getting sun altitudes for nighttime selection") sun_alts = [] for ts in self.ds["navigation_time"].to_numpy()[::stride]: - # About 10 minute resolution from 5 hz navigation data + # About 10-minute resolution from 5 Hz navigation data sun_alts.append( # noqa: PERF401 get_altitude( lat, @@ -371,63 +376,44 @@ def select_nighttime_bl_raw( datetime.fromtimestamp(ts.astype(int) / 1.0e9, tz=timezone.utc), ), ) + # Find sunset and sunrise - where sun altitude changes sign sign_changes = np.where(np.diff(np.sign(sun_alts)))[0] ss_sr_times = ( self.ds["navigation_time"].isel({"navigation_time": sign_changes * stride}).to_numpy() ) - self.logger.debug("Sunset sunrise times %s", ss_sr_times) - sunset = None - sunrise = None - if np.sign(sun_alts[0]) == 1: - # Sun is up at start of mission - if len(ss_sr_times) == 2: # noqa: PLR2004 - sunset, sunrise = ss_sr_times - sunset += pd.to_timedelta(1, "h") - sunrise -= pd.to_timedelta(1, "h") - elif len(ss_sr_times) == 1: - sunset = ss_sr_times[0] - sunset += pd.to_timedelta(1, "h") - sunrise = self.ds["biolume_time60hz"].to_numpy()[-1] - self.logger.warning( - "Could not find sunrise time, using last time in dataset: %s", sunrise - ) - else: - self.logger.info("Sun is up at start, but no sunset in this mission") - if np.sign(sun_alts[0]) == -1: - try: - self.logger.warning( - "Sun is not up at start of mission: %s: alt = %s", - ss_sr_times[0], - sun_alts[0], - ) - except IndexError: - # Likely no values in ss_sr_times[] - self.logger.warning("Sun is not up at start of mission") + self.logger.debug("Sunset and sunrise times: %s", ss_sr_times) + + sunsets = [] + sunrises = [] + nighttime_bl_raw = pd.Series(dtype="float64") + + # Iterate over sunset and sunrise pairs + for i in range(0, len(ss_sr_times) - 1, 2): + sunset = ss_sr_times[i] + pd.to_timedelta(1, "h") # 1 hour past sunset + sunrise = ss_sr_times[i + 1] - pd.to_timedelta(1, "h") # 1 hour before sunrise + sunsets.append(sunset) + sunrises.append(sunrise) - if sunset is None and sunrise is None: - self.logger.info( - "No sunset during this mission. No biolume_raw data will be extracted.", - ) - nighttime_bl_raw = pd.Series(dtype="float64") - else: self.logger.info( "Extracting biolume_raw data between sunset %s and sunrise %s", sunset, sunrise, ) - nighttime_bl_raw = ( - ( - self.ds["biolume_raw"].where( - (self.ds["biolume_time60hz"] > sunset) - & (self.ds["biolume_time60hz"] < sunrise), - ) + nighttime_data = ( + self.ds["biolume_raw"] + .where( + (self.ds["biolume_time60hz"] > sunset) + & (self.ds["biolume_time60hz"] < sunrise), ) .to_pandas() .dropna() ) + nighttime_bl_raw = pd.concat([nighttime_bl_raw, nighttime_data]) - return nighttime_bl_raw, sunset, sunrise + if not sunsets or not sunrises: + self.logger.info("No sunset or sunrise found during this mission.") + return nighttime_bl_raw, sunsets, sunrises def add_profile(self, depth_threshold: float = 15) -> None: # Find depth vertices value using scipy's find_peaks algorithm @@ -624,7 +610,7 @@ def add_biolume_proxies( # noqa: PLR0913, PLR0915 self.df_r["biolume_bg_biolume"].attrs["units"] = "photons/liter" self.df_r["biolume_bg_biolume"].attrs["comment"] = zero_note - nighttime_bl_raw, sunset, sunrise = self.select_nighttime_bl_raw() + nighttime_bl_raw, sunsets, sunrises = self.select_nighttime_bl_raw() if nighttime_bl_raw.empty: self.logger.info( "No nighttime_bl_raw data to compute adinos, diatoms, hdinos proxies", @@ -639,7 +625,8 @@ def add_biolume_proxies( # noqa: PLR0913, PLR0915 fluo = ( self.resampled_nc["hs2_fl700"] .where( - (self.resampled_nc["time"] > sunset) & (self.resampled_nc["time"] < sunrise), + (self.resampled_nc["time"] > min(sunsets)) + & (self.resampled_nc["time"] < max(sunrises)), ) .to_pandas() .resample(freq)