Skip to content

Improve select_nighttime_bl_raw() to return all nighttime data. #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MBARIMike merged 1 commit into from
Mar 24, 2025
Merged

Improve select_nighttime_bl_raw() to return all nighttime data. #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions .vscode/launch.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@
//"args": ["--auv_name", "i2map", "--mission", "2019.157.02", "-v", "2", "--plot", "--plot_seconds", "82000"],
//"args": ["--auv_name", "dorado", "--mission", "2021.102.02", "-v", "1", "--flash_threshold", "1.5e10"],
//"args": ["--auv_name", "dorado", "--mission", "2024.317.01", "-v", "1"],
"args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"],
//"args": ["--auv_name", "dorado", "--mission", "2010.341.00", "-v", "1", "--plot", "--plot_seconds", "82000"],
"args": ["--auv_name", "dorado", "--mission", "2020.337.00", "-v", "1"],
},
{
"name": "5.0 - archive.py",
Expand All @@ -151,7 +152,8 @@
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2023.192.01", "--AUVCTD"],
//"args": ["-v", "1", "--auv_name", "i2map", "--mission", "2022.061.01", "--M3", "--AUVCTD"],
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2024.317.01", "--AUVCTD"],
"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2022.201.00", "--AUVCTD"],
//"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2022.201.00", "--AUVCTD"],
"args": ["-v", "1", "--auv_name", "dorado", "--mission", "2020.337.00", "--AUVCTD", "--clobber"],
},
{
"name": "6.0 - create_products.py",
Expand Down Expand Up @@ -224,7 +226,7 @@
//"args": ["-v", "1", "--last_n_days", "30", "--start_year", "2022", "--end_year", "2022", "--resample", "--noinput"]
//"args": ["-v", "1", "--last_n_days", "30", "--start_year", "2022", "--end_year", "2022", "--archive", "--noinput"]
//"args": ["-v", "1", "--start_year", "2021", "--end_year", "2022", "--noinput"]
"args": ["-v", "1", "--mission", "2022.201.00", "--clobber", "--noinput", "--no_cleanup"]
//"args": ["-v", "1", "--mission", "2022.201.00", "--clobber", "--noinput", "--no_cleanup"]
//"args": ["-v", "1", "--mission", "2009.084.00", "--clobber", "--noinput"]
//"args": ["-v", "1", "--mission", "2022.243.00", "--calibrate", "--clobber", "--noinput"]
//"args": ["-v", "1", "--start_year", "2004", "--end_year", "2004", "--start_yd", "168", "--use_portal", "--clobber", "--noinput"]
Expand Down Expand Up @@ -263,6 +265,7 @@
//"args": ["-v", "1", "--noinput", "--num_cores", "8"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2011.256.02", "--clobber"]
//"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2010.341.00", "--download_process", "--local"]
"args": ["-v", "1", "--noinput", "--no_cleanup", "--mission", "2020.337.00", "--local"]
},
]
}
85 changes: 36 additions & 49 deletions src/data/resample.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,81 +353,67 @@ def save_coordinates(
def select_nighttime_bl_raw(
self,
stride: int = 3000,
) -> tuple[pd.Series, datetime, datetime]:
# Select only the nighttime biolume_raw data,
# one hour past sunset to one before sunrise
# Default stride of 3000 give 10 minute resolution
# from 5 hz navigation data
) -> tuple[pd.Series, list[datetime], list[datetime]]:
"""
Select nighttime biolume_raw data for multiple nights in a mission.
Default stride of 3000 gives 10-minute resolution from 5 Hz navigation data.

Returns:
nighttime_bl_raw: A pandas Series containing nighttime biolume_raw data.
sunsets: A list of sunset times for each night.
sunrises: A list of sunrise times for each night.
"""
lat = float(self.ds["navigation_latitude"].median())
lon = float(self.ds["navigation_longitude"].median())
self.logger.debug("Getting sun altitudes for nighttime selection")
sun_alts = []
for ts in self.ds["navigation_time"].to_numpy()[::stride]:
# About 10 minute resolution from 5 hz navigation data
# About 10-minute resolution from 5 Hz navigation data
sun_alts.append( # noqa: PERF401
get_altitude(
lat,
lon,
datetime.fromtimestamp(ts.astype(int) / 1.0e9, tz=timezone.utc),
),
)

# Find sunset and sunrise - where sun altitude changes sign
sign_changes = np.where(np.diff(np.sign(sun_alts)))[0]
ss_sr_times = (
self.ds["navigation_time"].isel({"navigation_time": sign_changes * stride}).to_numpy()
)
self.logger.debug("Sunset sunrise times %s", ss_sr_times)
sunset = None
sunrise = None
if np.sign(sun_alts[0]) == 1:
# Sun is up at start of mission
if len(ss_sr_times) == 2: # noqa: PLR2004
sunset, sunrise = ss_sr_times
sunset += pd.to_timedelta(1, "h")
sunrise -= pd.to_timedelta(1, "h")
elif len(ss_sr_times) == 1:
sunset = ss_sr_times[0]
sunset += pd.to_timedelta(1, "h")
sunrise = self.ds["biolume_time60hz"].to_numpy()[-1]
self.logger.warning(
"Could not find sunrise time, using last time in dataset: %s", sunrise
)
else:
self.logger.info("Sun is up at start, but no sunset in this mission")
if np.sign(sun_alts[0]) == -1:
try:
self.logger.warning(
"Sun is not up at start of mission: %s: alt = %s",
ss_sr_times[0],
sun_alts[0],
)
except IndexError:
# Likely no values in ss_sr_times[]
self.logger.warning("Sun is not up at start of mission")
self.logger.debug("Sunset and sunrise times: %s", ss_sr_times)

sunsets = []
sunrises = []
nighttime_bl_raw = pd.Series(dtype="float64")

# Iterate over sunset and sunrise pairs
for i in range(0, len(ss_sr_times) - 1, 2):
sunset = ss_sr_times[i] + pd.to_timedelta(1, "h") # 1 hour past sunset
sunrise = ss_sr_times[i + 1] - pd.to_timedelta(1, "h") # 1 hour before sunrise
sunsets.append(sunset)
sunrises.append(sunrise)

if sunset is None and sunrise is None:
self.logger.info(
"No sunset during this mission. No biolume_raw data will be extracted.",
)
nighttime_bl_raw = pd.Series(dtype="float64")
else:
self.logger.info(
"Extracting biolume_raw data between sunset %s and sunrise %s",
sunset,
sunrise,
)
nighttime_bl_raw = (
(
self.ds["biolume_raw"].where(
(self.ds["biolume_time60hz"] > sunset)
& (self.ds["biolume_time60hz"] < sunrise),
)
nighttime_data = (
self.ds["biolume_raw"]
.where(
(self.ds["biolume_time60hz"] > sunset)
& (self.ds["biolume_time60hz"] < sunrise),
)
.to_pandas()
.dropna()
)
nighttime_bl_raw = pd.concat([nighttime_bl_raw, nighttime_data])

return nighttime_bl_raw, sunset, sunrise
if not sunsets or not sunrises:
self.logger.info("No sunset or sunrise found during this mission.")
return nighttime_bl_raw, sunsets, sunrises

def add_profile(self, depth_threshold: float = 15) -> None:
# Find depth vertices value using scipy's find_peaks algorithm
Expand Down Expand Up @@ -624,7 +610,7 @@ def add_biolume_proxies( # noqa: PLR0913, PLR0915
self.df_r["biolume_bg_biolume"].attrs["units"] = "photons/liter"
self.df_r["biolume_bg_biolume"].attrs["comment"] = zero_note

nighttime_bl_raw, sunset, sunrise = self.select_nighttime_bl_raw()
nighttime_bl_raw, sunsets, sunrises = self.select_nighttime_bl_raw()
if nighttime_bl_raw.empty:
self.logger.info(
"No nighttime_bl_raw data to compute adinos, diatoms, hdinos proxies",
Expand All @@ -639,7 +625,8 @@ def add_biolume_proxies( # noqa: PLR0913, PLR0915
fluo = (
self.resampled_nc["hs2_fl700"]
.where(
(self.resampled_nc["time"] > sunset) & (self.resampled_nc["time"] < sunrise),
(self.resampled_nc["time"] > min(sunsets))
& (self.resampled_nc["time"] < max(sunrises)),
)
.to_pandas()
.resample(freq)
Expand Down