Merge pull request #400 from euroargodev/releasev1.0.0

Prepare for "v1.0.0 Unicorn Release 🦄"

.github/workflows/pytests-upstream.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     steps:
@@ -202,7 +202,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     steps:

.github/workflows/pytests.yml

@@ -52,7 +52,7 @@ jobs:
       max-parallel: 12
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.10", "3.11"]
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         experimental: [false]
 
@@ -181,7 +181,7 @@ jobs:
       max-parallel: 12
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.10", "3.11"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         experimental: [false]
 

argopy/__init__.py

@@ -42,7 +42,7 @@
 from .utils import MonitoredThreadPoolExecutor  # noqa: E402, F401
 from .utils import monitor_status as status  # noqa: E402
 from .related import TopoFetcher, OceanOPSDeployments, ArgoNVSReferenceTables, ArgoDocs, ArgoDOI  # noqa: E402
-from .extensions import CanyonMED
+from .extensions import CanyonMED  # noqa: E402
 
 
 #

argopy/data_fetchers/argovis_data.py

@@ -7,8 +7,7 @@
 import warnings
 
 from ..stores import httpstore
-from ..options import OPTIONS, DEFAULT, VALIDATE, PARALLEL_SETUP
-from ..utils.format import format_oneline
+from ..options import OPTIONS, DEFAULT, PARALLEL_SETUP
 from ..utils.chunking import Chunker
 from ..utils.decorators import deprecated
 from ..errors import DataNotFound
@@ -87,7 +86,7 @@ def __init__(
         """
         self.definition = "Argovis Argo data fetcher"
         self.dataset_id = OPTIONS["ds"] if ds == "" else ds
-        self.user_mode =  kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"]
+        self.user_mode = kwargs["mode"] if "mode" in kwargs else OPTIONS["mode"]
         self.server = kwargs["server"] if "server" in kwargs else api_server
         timeout = OPTIONS["api_timeout"] if api_timeout == 0 else api_timeout
         self.store_opts = {
@@ -144,7 +143,6 @@ def _add_history(self, this, txt):
             this.attrs["history"] = txt
         return this
 
-
     @property
     def cachepath(self):
         """Return path to cache file for this request"""
@@ -165,7 +163,10 @@ def safe_for_fsspec_cache(url):
 
         return [safe_for_fsspec_cache(url) for url in urls]
 
-    @deprecated('Not serializable')
+    @deprecated(
+        "Not serializable, please use 'argovis_data_processors.pre_process'",
+        version="1.0.0",
+    )
     def json2dataframe(self, profiles):
         """convert json data to Pandas DataFrame"""
         # Make sure we deal with a list
@@ -213,7 +214,7 @@ def to_dataframe(self, errors: str = "ignore") -> pd.DataFrame:
         """Load Argo data and return a Pandas dataframe"""
 
         # Download data:
-        preprocess_opts = {'key_map': self.key_map}
+        preprocess_opts = {"key_map": self.key_map}
         df_list = self.fs.open_mfjson(
             self.uri,
             method=self.parallel_method,

argopy/data_fetchers/erddap_data.py

@@ -11,21 +11,16 @@
 """
 
 import xarray as xr
-import pandas as pd
 import numpy as np
 import copy
-import time
 from abc import abstractmethod
-import getpass
 from typing import Union
 from aiohttp import ClientResponseError
 import logging
 from erddapy.erddapy import ERDDAP, parse_dates
 from erddapy.erddapy import _quote_string_constraints as quote_string_constraints
-import warnings
 
 from ..options import OPTIONS, PARALLEL_SETUP
-from ..utils.format import format_oneline
 from ..utils.lists import list_bgc_s_variables, list_core_parameters
 from ..utils.decorators import deprecated
 from ..errors import ErddapServerError, DataNotFound
@@ -264,7 +259,7 @@ def server(self, value):
             log.debug("The erddap server has been modified, updating internal data")
             self._init_erddapy()
 
-    @deprecated("Not serializable")
+    @deprecated("Not serializable, please use 'erddap_data_processors._add_attributes'", version="1.0.0")
     def _add_attributes(self, this):  # noqa: C901
         """Add variables attributes not return by erddap requests (csv)
 
@@ -672,9 +667,12 @@ def getNfromncHeader(url):
             url = url.replace("." + self.erddap.response, ".ncHeader")
             try:
                 ncHeader = str(self.fs.download_url(url))
-                lines = [line for line in ncHeader.splitlines() if "row = " in line][0]
-                return int(lines.split("=")[1].split(";")[0])
-            except Exception:
+                if "Your query produced no matching results. (nRows = 0)" in ncHeader:
+                    return 0
+                else:
+                    lines = [line for line in ncHeader.splitlines() if "row = " in line][0]
+                    return int(lines.split("=")[1].split(";")[0])
+            except Exception as e:
                 raise ErddapServerError(
                     "Erddap server can't return ncHeader for url: %s " % url
                 )

argopy/data_fetchers/gdac_data.py

@@ -13,7 +13,7 @@
 import getpass
 import logging
 
-from ..utils.format import format_oneline, argo_split_path
+from ..utils.format import argo_split_path
 from ..utils.decorators import deprecated
 from ..options import OPTIONS, check_gdac_path, PARALLEL_SETUP
 from ..errors import DataNotFound
@@ -40,8 +40,8 @@ class GDACArgoDataFetcher(ArgoDataFetcherProto):
     This class is a prototype not meant to be instantiated directly
 
     """
-    data_source = "gdac"
 
+    data_source = "gdac"
 
     ###
     # Methods to be customised for a specific request
@@ -119,7 +119,7 @@ def __init__(
             cachedir=cachedir,
             timeout=self.timeout,
         )
-        self.fs = self.indexfs.fs["src"]  # Re-use the appropriate file system
+        self.fs = self.indexfs.fs["src"]  # Reuse the appropriate file system
 
         nrows = None
         if "N_RECORDS" in kwargs:
@@ -140,7 +140,9 @@ def __repr__(self):
         summary.append(self._repr_access_point)
         summary.append(self._repr_server)
         if hasattr(self.indexfs, "index"):
-            summary.append("📗 Index: %s (%i records)" % (self.indexfs.index_file, self.N_RECORDS))
+            summary.append(
+                "📗 Index: %s (%i records)" % (self.indexfs.index_file, self.N_RECORDS)
+            )
         else:
             summary.append("📕 Index: %s (not loaded)" % self.indexfs.index_file)
         if hasattr(self.indexfs, "search"):
@@ -233,7 +235,10 @@ def clear_cache(self):
         self.fs.clear_cache()
         return self
 
-    @deprecated("Not serializable")
+    @deprecated(
+        "Not serializable, please use 'gdac_data_processors.pre_process_multiprof'",
+        version="1.0.0",
+    )
     def _preprocess_multiprof(self, ds):
         """Pre-process one Argo multi-profile file as a collection of points
 
@@ -332,13 +337,13 @@ def to_xarray(self, errors: str = "ignore"):
 
         if hasattr(self, "BOX"):
             access_point = "BOX"
-            access_point_opts = {'BOX': self.BOX}
+            access_point_opts = {"BOX": self.BOX}
         elif hasattr(self, "CYC"):
             access_point = "CYC"
-            access_point_opts = {'CYC': self.CYC}
+            access_point_opts = {"CYC": self.CYC}
         elif hasattr(self, "WMO"):
             access_point = "WMO"
-            access_point_opts = {'WMO': self.WMO}
+            access_point_opts = {"WMO": self.WMO}
 
         # Download and pre-process data:
         ds = self.fs.open_mfdataset(
@@ -379,7 +384,7 @@ def to_xarray(self, errors: str = "ignore"):
             ds.attrs["Fetched_from"] = self.server
             try:
                 ds.attrs["Fetched_by"] = getpass.getuser()
-            except:
+            except:  # noqa: E722
                 ds.attrs["Fetched_by"] = "anonymous"
             ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime(
                 "%Y/%m/%d"
@@ -393,17 +398,20 @@ def to_xarray(self, errors: str = "ignore"):
 
         return ds
 
-    @deprecated("Refactored to GDAC pre-processor submodule")
+    @deprecated(
+        "Not serializable, please use 'gdac_data_processors.filter_points'",
+        version="1.0.0",
+    )
     def filter_points(self, ds):
         if hasattr(self, "BOX"):
             access_point = "BOX"
-            access_point_opts = {'BOX': self.BOX}
+            access_point_opts = {"BOX": self.BOX}
         elif hasattr(self, "CYC"):
             access_point = "CYC"
-            access_point_opts = {'CYC': self.CYC}
+            access_point_opts = {"CYC": self.CYC}
         elif hasattr(self, "WMO"):
             access_point = "WMO"
-            access_point_opts = {'WMO': self.WMO}
+            access_point_opts = {"WMO": self.WMO}
         return filter_points(ds, access_point=access_point, **access_point_opts)
 
     def transform_data_mode(self, ds: xr.Dataset, **kwargs):

argopy/data_fetchers/gdac_data_processors.py

@@ -1,7 +1,5 @@
 import numpy as np
-import pandas as pd
 import xarray as xr
-import getpass
 
 
 def pre_process_multiprof(
@@ -58,24 +56,6 @@ def pre_process_multiprof(
         ds.argo.profile2point()
     )  # Default output is a collection of points, along N_POINTS
 
-
-    # Attributes are added by the caller
-
-    # if dataset_id == "phy":
-    #     ds.attrs["DATA_ID"] = "ARGO"
-    # if dataset_id in ["bgc", "bgc-s"]:
-    #     ds.attrs["DATA_ID"] = "ARGO-BGC"
-    #
-    # ds.attrs["DOI"] = "http://doi.org/10.17882/42182"
-    #
-    # # ds.attrs["Fetched_from"] = server
-    # ds.attrs["Fetched_constraints"] = cname
-    # try:
-    #     ds.attrs["Fetched_by"] = getpass.getuser()
-    # except:  # noqa: E722
-    #     ds.attrs["Fetched_by"] = "anonymous"
-    # ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime("%Y/%m/%d")
-    # ds.attrs["Fetched_uri"] = ds.encoding["source"]
     ds = ds[np.sort(ds.data_vars)]
 
     if pre_filter_points:

argopy/data_fetchers/proto.py

@@ -6,7 +6,7 @@
 import warnings
 import logging
 from ..plot import dashboard
-from ..utils.lists import list_standard_variables, list_bgc_s_variables
+from ..utils.lists import list_standard_variables
 from ..utils.format import UriCName, format_oneline
 
 

argopy/errors.py

@@ -5,6 +5,7 @@
 
 log = logging.getLogger("argopy.errors")
 
+
 class NoData(ValueError):
     """Raise for no data"""
     def __init__(self, path: str = "?"):

argopy/extensions/canyon_med.py

@@ -47,7 +47,7 @@ class CanyonMED(ArgoAccessorExtension):
     .. [2] Fourrier, M., Coppola, L., Claustre, H., D’Ortenzio, F., Sauzède, R., and Gattuso, J.-P. (2021). Corrigendum: A Regional Neural Network Approach to Estimate Water-Column Nutrient Concentrations and Carbonate System Variables in the Mediterranean Sea: CANYON-MED. Frontiers in Marine Science 8. doi:10.3389/fmars.2021.650509.
     """
 
-    # todo This class work with pandas dataframe, but we should keep xarray dataset internaly for the predictions
+    # todo This class work with pandas dataframe, but we should keep xarray dataset internally for the predictions
 
     ne = 7
     """Number of inputs"""

argopy/extensions/params_data_mode.py

@@ -1,5 +1,4 @@
 import numpy as np
-import pandas as pd
 import xarray as xr
 import logging
 import time
@@ -32,7 +31,7 @@ class ParamsDataMode(ArgoAccessorExtension):
     :meth:`datamode.split`
 
     Examples
-    --------    
+    --------
     >>> from argopy import DataFetcher
     >>> ArgoSet = DataFetcher(mode='expert').float(1902605)
     >>> ds = ArgoSet.to_xarray()

argopy/fetchers.py

@@ -21,7 +21,6 @@
     InvalidFetcherAccessPoint,
     InvalidFetcher,
     OptionValueError,
-    OptionDeprecatedWarning,
 )
 from .related import (
     get_coriolis_profile_id,
@@ -170,7 +169,7 @@ def __init__(self, mode: str = "", src: str = "", ds: str = "", **fetcher_kwargs
         self._loaded = False
         self._request = ""
 
-        # Dev warnings
+        # Warnings
         # Todo Clean-up before each release
         if self._src == "argovis" and (
             self._mode == "expert" or self._mode == "research"
@@ -179,12 +178,6 @@ def __init__(self, mode: str = "", src: str = "", ds: str = "", **fetcher_kwargs
                 "The 'argovis' data source fetching is only available in 'standard' user mode"
             )
 
-        if self._src == "gdac" and "ftp" in self.fetcher_options:
-            OptionDeprecatedWarning(
-                reason="The GDAC 'ftp' argument is deprecated, it will be replaced by 'gdac' in versions >= 0.1.18",
-                version="v0.0.17",
-            )
-
     @property
     def _icon_user_mode(self):
         if self._mode == "standard":
@@ -498,10 +491,10 @@ def workflow(xds):
                     return xds
 
             else:
-                workflow = lambda x: x  # Empty processor
+                workflow = lambda x: x  # noqa: E731
 
         else:
-            workflow = lambda x: x  # Empty processor
+            workflow = lambda x: x  # noqa: E731
 
         self._pp_workflow = workflow
 

argopy/options.py

@@ -134,12 +134,12 @@ def VALIDATE(key, val):
     if key in _VALIDATORS:
         if not _VALIDATORS[key](val):
             raise OptionValueError(
-                f"option '%s' given an invalid value: '%s'" % (key, val)
+                f"option '{key}' given an invalid value: '{val}'"
             )
         else:
             return val
     else:
-        raise ValueError(f"option '%s' has no validation method" % key)
+        raise ValueError(f"option '{key}' has no validation method")
 
 
 def PARALLEL_SETUP(parallel):

argopy/related/reference_tables.py

@@ -31,6 +31,7 @@ class ArgoNVSReferenceTables:
 
     valid_ref = [
         "R01",
+
         "R03",
         "R04",
         "R05",
@@ -44,6 +45,7 @@ class ArgoNVSReferenceTables:
         "R13",
         "R15",
         "R16",
+
         "R18",
         "R19",
         "R20",
@@ -55,7 +57,9 @@ class ArgoNVSReferenceTables:
         "R26",
         "R27",
         "R28",
+
         "R40",
+
         "RD2",
         "RMC",
         "RP2",