diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 717264b8..d01f699f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,9 +58,9 @@ jobs: - name: Download and unzip required data files run: | mkdir -p "$HOME/auv_data" - # Dorado389.tar.gz contains original log files of short dorado mission for testing: 2003.339.04 + # Dorado389.tar.gz contains original log files of short dorado mission for testing: 2011.256.02 wget -q --no-check-certificate -O "$HOME/auv_data/Dorado389.tar.gz" https://stoqs.mbari.org/auv_data/Dorado389.tar.gz - tar -xf "$HOME/auv_data/Dorado389.tar.gz" -C "$HOME/auv_data" 2> /dev/null + tar -xvf "$HOME/auv_data/Dorado389.tar.gz" -C "$HOME/auv_data" 2> /dev/null # i2MAP.tar.gz contains original log files of i2MAP mission for testing: 2018.34.01 wget -q --no-check-certificate -O "$HOME/auv_data/i2map.tar.gz" https://stoqs.mbari.org/auv_data/i2map.tar.gz tar -xf "$HOME/auv_data/i2map.tar.gz" -C "$HOME/auv_data" 2> /dev/null diff --git a/src/data/calibrate.py b/src/data/calibrate.py index f845476a..e7bff8ca 100755 --- a/src/data/calibrate.py +++ b/src/data/calibrate.py @@ -59,7 +59,7 @@ import pandas as pd import pyproj from AUV import monotonic_increasing_time_indices -from hs2_proc import hs2_calc_bb, hs2_read_cal_file +from hs2_proc import compute_backscatter, hs2_calc_bb, hs2_read_cal_file from logs2netcdfs import BASE_PATH, MISSIONLOGS, MISSIONNETCDFS, TIME, TIME60HZ, AUV_NetCDF from matplotlib import patches from scipy import signal @@ -594,32 +594,6 @@ def _beam_transmittance_from_volts(combined_nc, nc) -> tuple[float, float]: return Tr, c -def _compute_backscatter(wavelength_nm: float, salinity: float, volScat: float): # noqa: N803 - # Cribbed from https://mbari.slack.com/archives/C04ETLY6T7V/p1710457297254969?thread_ts=1710348431.316509&cid=C04ETLY6T7V - # This is the same computation used for LRAUV ecopucks. Used here for Dorado ecopuck - # following the conversion to "scaled" output using scale_factor and dark counts. - theta = 117.0 / 57.29578 # radians - d = 0.09 - - # These calculations are from the Triplet Puck User's Guide, Revision H - Bw = ( - 1.38 - * (wavelength_nm / 500.0) ** (-4.32) - * (1.0 + 0.3 * salinity / 37.0) - * 1e-4 - * (1.0 + np.cos(theta) ** 2.0 * (1.0 - d) / (1.0 + d)) - ) - Bp = volScat - Bw - if salinity < 35.0: # noqa: PLR2004 - bw = 0.0022533 * (wavelength_nm / 500.0) ** (-4.23) * 1e-4 - else: - bw = 0.0029308 * (wavelength_nm / 500.0) ** (-4.24) * 1e-4 - bbw = bw / 2.0 - bbp = 2.0 * np.pi * 1.1 * Bp - - return bbw, bbp - - class SensorInfo: pass @@ -3002,7 +2976,7 @@ def _ecopuck_process(self, sensor, cf): source = self.sinfo[sensor]["data_filename"] coord_str = f"{sensor}_time {sensor}_depth {sensor}_latitude {sensor}_longitude" beta_700 = cf.bbp700_scale_factor * (orig_nc["BB_Sig"].to_numpy() - cf.bbp700_dark_counts) - _, bbp = _compute_backscatter(700, 35.2, beta_700) # Use an average salinity of 35.2 + _, bbp = compute_backscatter(700, 35.2, beta_700) # Use an average salinity of 35.2 self.combined_nc["ecopuck_bbp700"] = xr.DataArray( bbp, @@ -3017,7 +2991,7 @@ def _ecopuck_process(self, sensor, cf): "comment": ( f"BB_Sig from {source} converted to beta_700 using scale factor " f"{cf.bbp700_scale_factor} and dark counts {cf.bbp700_dark_counts}, " - "then converted to bbp700 by the _compute_backscatter() function." + "then converted to bbp700 by the compute_backscatter() function." ), } diff --git a/src/data/create_products.py b/src/data/create_products.py index ac3af8b8..67ea9a05 100755 --- a/src/data/create_products.py +++ b/src/data/create_products.py @@ -385,7 +385,8 @@ def gulper_odv(self, sec_bnds: int = 1) -> str: # noqa: C901, PLR0912, PLR0915 gulper = Gulper() gulper.args = argparse.Namespace() - gulper.args.auv_name = "dorado" + gulper.args.base_path = self.args.base_path + gulper.args.auv_name = self.args.auv_name gulper.args.mission = self.args.mission gulper.args.local = self.args.local gulper.args.verbose = self.args.verbose diff --git a/src/data/gulper.py b/src/data/gulper.py index 9033fc85..4147e940 100755 --- a/src/data/gulper.py +++ b/src/data/gulper.py @@ -19,7 +19,7 @@ import requests import xarray as xr -from logs2netcdfs import TIMEOUT +from logs2netcdfs import MISSIONLOGS, MISSIONNETCDFS, TIMEOUT class Gulper: @@ -33,11 +33,10 @@ def mission_start_esecs(self) -> float: # Get the first time record from mission's navigation.nc file if self.args.local: - base_path = Path(__file__).parent.joinpath("../../data/auv_data").resolve() url = Path( - base_path, - "dorado", - "missionnetcdfs", + self.args.base_path, + self.args.auv_name, + MISSIONNETCDFS, self.args.mission, "navigation.nc", ) @@ -45,7 +44,7 @@ def mission_start_esecs(self) -> float: # Relies on auv-python having processed the mission url = os.path.join( # noqa: PTH118 "http://dods.mbari.org/opendap/data/auvctd/", - "missionnetcdfs", + MISSIONNETCDFS, self.args.mission.split(".")[0], self.args.mission.split(".")[0] + self.args.mission.split(".")[1], self.args.mission, @@ -64,11 +63,10 @@ def parse_gulpers(self, sec_delay: int = 1) -> dict: # noqa: C901, PLR0912, PLR bottles = {} if self.args.local: # Read from local file - useful for testing in auv-python - base_path = Path(__file__).parent.joinpath("../../data/auv_data").resolve() mission_dir = Path( - base_path, - "dorado", - "missionlogs", + self.args.base_path, + self.args.auv_name, + MISSIONLOGS, self.args.mission, ) syslog_file = Path(mission_dir, "syslog") @@ -81,7 +79,7 @@ def parse_gulpers(self, sec_delay: int = 1) -> dict: # noqa: C901, PLR0912, PLR else: syslog_url = os.path.join( # noqa: PTH118 "http://dods.mbari.org/data/auvctd/", - "missionlogs", + MISSIONLOGS, self.args.mission.split(".")[0], self.args.mission.split(".")[0] + self.args.mission.split(".")[1], self.args.mission, diff --git a/src/data/hs2_proc.py b/src/data/hs2_proc.py index 7cac4cd4..029a7d6d 100644 --- a/src/data/hs2_proc.py +++ b/src/data/hs2_proc.py @@ -1,7 +1,7 @@ # noqa: INP001 from collections import defaultdict -from math import exp, pi +from math import exp from pathlib import Path import numpy as np @@ -98,6 +98,32 @@ def _int_signer(ints_in): return np.array(signed_ints) +def compute_backscatter(wavelength_nm: float, salinity: float, volScat: float): # noqa: N803 + # Cribbed from https://mbari.slack.com/archives/C04ETLY6T7V/p1710457297254969?thread_ts=1710348431.316509&cid=C04ETLY6T7V + # This is the same computation used for LRAUV ecopucks. Used here for Dorado ecopuck + # following the conversion to "scaled" output using scale_factor and dark counts. + theta = 117.0 / 57.29578 # radians + d = 0.09 + + # These calculations are from the Triplet Puck User's Guide, Revision H + Bw = ( + 1.38 + * (wavelength_nm / 500.0) ** (-4.32) + * (1.0 + 0.3 * salinity / 37.0) + * 1e-4 + * (1.0 + np.cos(theta) ** 2.0 * (1.0 - d) / (1.0 + d)) + ) + Bp = volScat - Bw + if salinity < 35.0: # noqa: PLR2004 + bw = 0.0022533 * (wavelength_nm / 500.0) ** (-4.23) * 1e-4 + else: + bw = 0.0029308 * (wavelength_nm / 500.0) ** (-4.24) * 1e-4 + bbw = bw / 2.0 + bbp = 2.0 * np.pi * 1.1 * Bp + + return bbw, bbp + + def hs2_calc_bb(orig_nc, cals): # Some original comments from hs2_calc_bb.m # % Date Created: June 21, 2007 @@ -141,34 +167,12 @@ def hs2_calc_bb(orig_nc, cals): # Replaces "RawTempValue" as the name, helpful when looking at things in the debugger beta_uncorr.name = f"beta_uncorr_Ch{chan}" wavelength = int(cals[f"Ch{chan}"]["Name"][2:]) - beta_w, b_bw = purewater_scatter(wavelength) - - chi = 1.08 - b_b_uncorr = ((2 * pi * chi) * (beta_uncorr - beta_w)) + b_bw - - globals()[f"bb{wavelength}_uncorr"] = b_b_uncorr - globals()[f"bbp{wavelength}_uncorr"] = b_b_uncorr - b_bw - - # ESTIMATION OF KBB AND SIGMA FUNCTION - a = typ_absorption(wavelength) - b_b_tilde = 0.015 - b = (b_b_uncorr - b_bw) / b_b_tilde - - K_bb = a + 0.4 * b - k_1 = 1.0 - k_exp = float(cals[f"Ch{chan}"]["SigmaExp"]) - sigma = k_1 * np.exp(k_exp * K_bb) - - b_b_corr = sigma * b_b_uncorr - # Need to test subtracting b_bw here instead of after multiplying by sigma - b_bp_corr = sigma * (b_b_uncorr - b_bw) - setattr(hs2, f"bb{wavelength}", b_b_corr) - # Legacy code that subtracts b_bw after multiplying by sigma - setattr(hs2, f"bbp{wavelength}", b_b_corr - b_bw) - # This is likely the correct way to do it, with b_bw subtracted before multiplying by sigma - setattr(hs2, f"bbp{wavelength}_fixed", b_bp_corr) + # Use compute_backscatter - same as used for ecopucks - to calculate bbp + _, bbp = compute_backscatter(wavelength, 35.2, beta_uncorr) + setattr(hs2, f"bbp{wavelength}", bbp) + # Fluorescence # -% 'hs2.fl700_uncorr = (hs2.Snorm3.*50)./((1 + str2num(CAL.Ch(3).TempCoeff).*(hs2.Temp-str2num(CAL.General.CalTemp))).*hs2.Gain3.*str2num(CAL.Ch(3).RNominal));' # noqa: E501 denom = ( ( diff --git a/src/data/process.py b/src/data/process.py index cb05de76..776bcc09 100755 --- a/src/data/process.py +++ b/src/data/process.py @@ -355,6 +355,7 @@ def archive(self, mission: str, add_logger_handlers: bool = True) -> None: # no def create_products(self, mission: str) -> None: cp = CreateProducts() cp.args = argparse.Namespace() + cp.args.base_path = self.args.base_path cp.args.auv_name = self.vehicle cp.args.mission = mission cp.args.local = self.args.local diff --git a/src/data/test_hs2_proc.py b/src/data/test_hs2_proc.py deleted file mode 100644 index d2dc8538..00000000 --- a/src/data/test_hs2_proc.py +++ /dev/null @@ -1,130 +0,0 @@ -# noqa: INP001 -import numpy as np -from hs2_proc import _get_gains, hs2_calc_bb, purewater_scatter, typ_absorption - -# Arrays used in asserts are for short mission that's in conftest.py -# As of 2025-03-02 it is Dorado389 mission 2011.256.02. - - -def test_typ_absorption(): - assert round(typ_absorption(420), 4) == 0.0235 # noqa: PLR2004, S101 - assert round(typ_absorption(470), 4) == 0.0179 # noqa: PLR2004, S101 - - -def test_purewater_scatter(): - # Matlab RunReprocess on 2020.245.00 - - # K>> [a, b] = purewater_scatter(420) - # a = - # 5.7162e-04 - # b = - # 0.0031 - - # K>> [a, b] = purewater_scatter(700) - # a = - # 6.2910e-05 - # b = - # 3.3764e-04 - - assert np.allclose(purewater_scatter(420), (5.7162e-04, 0.0031), atol=1e-4) # noqa: S101 - assert np.allclose(purewater_scatter(700), (6.2910e-05, 3.3764e-04), atol=1e-4) # noqa: S101 - - -def test_get_gains(mission_data, calibration): - # mission_data and calibration are fixtures from the conftest.py module; - # they are automatically loaded by pytest - md = mission_data - cals = calibration - hs2 = _get_gains(md.hs2.orig_data, cals, md.hs2) - - # Matlab RunReprocess on 2020.245.00 - - # K>> hs2.Gain1(1:5) - # ans = - # 1.0e+03 * - # 1.0077 - # 0.0982 - # 1.0077 - # 1.0077 - # 1.0077 - - # K>> hs2.Gain2(1:5) - # ans = - # 1.0e+03 * - # 9.7439 - # 0.9325 - # 0.9325 - # 0.9325 - # 0.9325 - - # K>> hs2.Gain3(1:5) - # ans = - # 1.0e+03 * - # 9.7439 - # 0.9325 - # 0.9325 - # 0.9325 - # 0.9325 - - assert np.all(hs2.Gain1[:5] == np.array([998.81, 98.1544, 998.81, 998.81, 998.81])) # noqa: S101 - assert np.all(hs2.Gain2[:5] == np.array([94.547, 94.547, 929.18, 9715.1, 9715.1])) # noqa: S101 - assert np.all(hs2.Gain3[:5] == np.array([94.547, 94.547, 929.18, 9715.1, 9715.1])) # noqa: S101 - - -def test_hs2_calc_bb(mission_data, calibration): - # mission_data and calibration are fixtures from the conftest.py module; - # they are automatically loaded by pytest - md = mission_data - cals = calibration - hs2 = hs2_calc_bb(md.hs2.orig_data, cals) - - # Matlab RunReprocess on 2020.245.00 - - # K>> format long - # K>> hs2.bb420(1:5) - # ans = - # 0.005975077727995 - # 0.005245234803124 - # 0.006044696205554 - # 0.005488825740040 - # 0.006203063319423 - - # K>> hs2.bbp420(1:5) - # ans = - # 0.002907206559870 - # 0.002177363634998 - # 0.002976825037428 - # 0.002420954571914 - # 0.003135192151297 - - # K>> hs2.bb700(1:5) - # ans = - # 0.012280996975942 - # 0.012087616414187 - # 0.012479078951715 - # 0.011599823683877 - # 0.012087616414187 - - # K>> hs2.bbp700(1:5) - # ans = - # 0.011943359597475 - # 0.011749979035720 - # 0.012141441573249 - # 0.011262186305410 - # 0.011749979035720 - - # K>> hs2.fl700_uncorr(1:5) - # ans = - # 0.002448326645388 - # -0.001983999313643 - # 0.002104647920554 - # 0.001990702014027 - # 0.001943783111340 - - assert np.allclose(hs2.bb420[:5], [0.00946282, 0.01565809, 0.01053635, 0.00943493, 0.00976532]) # noqa: S101 - assert np.allclose(hs2.bbp420[:5], [0.00639495, 0.01259022, 0.00746848, 0.00636706, 0.00669745]) # noqa: S101 - assert np.allclose(hs2.bb700[:5], [0.00590656, 0.00607707, -0.00457753, 0.00577039, 0.00555023]) # noqa: S101 - assert np.allclose( # noqa: S101 - hs2.bbp700[:5], [0.00556892, 0.00573943, -0.00491517, 0.00543275, 0.00521259] - ) - assert np.allclose(hs2.fl700[:5], [0.00112378, 0.00079326, -0.02851304, 0.00504691, 0.00103833]) # noqa: S101 diff --git a/src/data/test_process_dorado.py b/src/data/test_process_dorado.py index 33389537..4b2f51b3 100644 --- a/src/data/test_process_dorado.py +++ b/src/data/test_process_dorado.py @@ -26,4 +26,18 @@ def test_process_dorado(complete_dorado_processing): assert nc_file.exists() # noqa: S101 assert time() - nc_file.stat().st_mtime < MAX_SECS # noqa: S101 assert nc_file.stat().st_size > 0 # noqa: S101 - assert nc_file.stat().st_size == 621235 # noqa: PLR2004, S101 + # Testing that the file size matches a specific value is crude, + # but it will alert us if a code change unexpectedly changes the file size. + # If code changes are expected to change the file size then we should + # update the expected size here. + EXPECTED_SIZE = 621235 + EXPECTED_SIZE_LOCAL = 621298 + if str(proc.args.base_path).startswith("/home/runner"): + # The size is different in GitHub Actions, maybe due to different metadata + assert nc_file.stat().st_size == EXPECTED_SIZE # noqa: S101 + else: + # The size is different locally, maybe due to different metadata + # It's likely that the size will be different on different machines + # as these kind of metadata items are added to nc_file: + # NC_GLOBAL.history: Created by /Users/mccann/GitHub/auv-python/src/data/process_dorado.py ... # noqa: E501 + assert nc_file.stat().st_size == EXPECTED_SIZE_LOCAL # noqa: S101 diff --git a/src/data/test_process_i2map.py b/src/data/test_process_i2map.py index 4ed42bc9..213625a2 100644 --- a/src/data/test_process_i2map.py +++ b/src/data/test_process_i2map.py @@ -26,8 +26,18 @@ def test_process_i2map(complete_i2map_processing): assert nc_file.exists() # noqa: S101 assert time() - nc_file.stat().st_mtime < MAX_SECS # noqa: S101 assert nc_file.stat().st_size > 0 # noqa: S101 + # Testing that the file size matches a specific value is crude, + # but it will alert us if a code change unexpectedly changes the file size. + # If code changes are expected to change the file size then we should + # update the expected size here. + EXPECTED_SIZE = 60130 + EXPECTED_SIZE_LOCAL = 58896 if str(proc.args.base_path).startswith("/home/runner"): - # The size is smaller in GitHub Actions, maybe due to different metadata - assert nc_file.stat().st_size == 60130 # noqa: PLR2004, S101 + # The size is different in GitHub Actions, maybe due to different metadata + assert nc_file.stat().st_size == EXPECTED_SIZE # noqa: S101 else: - assert nc_file.stat().st_size == 60191 # noqa: PLR2004, S101 + # The size is different locally, maybe due to different metadata + # It's likely that the size will be different on different machines + # as these kind of metadata items are added to nc_file: + # NC_GLOBAL.history: Created by /Users/mccann/GitHub/auv-python/src/data/process_dorado.py ... # noqa: E501 + assert nc_file.stat().st_size == EXPECTED_SIZE_LOCAL # noqa: S101