Merge pull request #379 from euroargodev/releasev0.1.16

Prepare for v0.1.16 Flamingos Release 🦩

.codespellrc

@@ -1,5 +1,5 @@
 [codespell]
-skip = *.nc,*.ipynb,./local_work,./float_source,./binder,./.github,*.log,./.git,./docs/_build,./docs/_static,./argopy/tests/test_data,./build,./docs/mycache_folder,./docs/examples/cache_bgc
+skip = *.nc,*.ipynb,./local_work,./float_source,./binder,./.github,*.log,./.git,./docs/_build,./docs/_static,./argopy/tests/test_data,./build,./docs/mycache_folder,./docs/examples/cache_bgc,./argopy/static/assets/*.json
 count =
 quiet-level = 3
 ignore-words-list = PRES, pres, idel

.flake8

@@ -10,7 +10,8 @@ ignore =
     E501,
     # line break before binary operator
     W503
-
+    # whitespace before ':' (https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#e203)
+    E203
 exclude =
     # No need to traverse our git directory
     .git,

.github/workflows/pytests-upstream.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.9", "3.10"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     steps:
@@ -200,7 +200,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.9", "3.10"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     steps:

.github/workflows/pytests.yml

@@ -52,7 +52,7 @@ jobs:
       max-parallel: 12
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.9", "3.10"]
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
         experimental: [false]
 
@@ -174,7 +174,7 @@ jobs:
       max-parallel: 12
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9"]
+        python-version: ["3.9", "3.10"]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         experimental: [false]
 

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as
 contributors and maintainers pledge to making participation in our project and
 our community a harassment-free experience for everyone, regardless of age, body
 size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
+level of experience, education, socioeconomic status, nationality, personal
 appearance, race, religion, or sexual identity and orientation.
 
 ## Our Standards

argopy/__init__.py

@@ -29,7 +29,6 @@
 
 # Other Import
 # from . import utils  # noqa: E402
-from . import utilities  # noqa: E402  # being deprecated until 0.1.15, then remove
 from . import stores  # noqa: E402
 from . import errors  # noqa: E402
 from . import plot  # noqa: E402
@@ -72,7 +71,6 @@
     "ArgoDOI",  # Class
 
     # Submodules:
-    "utilities",  # being deprecated until 0.1.15, then remove
     # "utils",
     "errors",
     "plot",

argopy/data_fetchers/argovis_data.py

@@ -1,9 +1,3 @@
-#!/bin/env python
-# -*coding: UTF-8 -*-
-#
-# Argo data fetcher for Argovis.
-#
-
 import numpy as np
 import pandas as pd
 import xarray as xr
@@ -23,7 +17,7 @@
 access_points = ["wmo", "box"]
 exit_formats = ["xarray"]
 dataset_ids = ["phy"]  # First is default
-api_server = "https://argovis-api.colorado.edu" 
+api_server = "https://argovis-api.colorado.edu"
 api_server_check = "https://argovis-api.colorado.edu/ping"
 
 log = logging.getLogger("argopy.argovis.data")
@@ -58,7 +52,7 @@ def __init__(
         chunks: str = "auto",
         chunks_maxsize: dict = {},
         api_timeout: int = 0,
-        **kwargs
+        **kwargs,
     ):
         """Instantiate an Argovis Argo data loader
 
@@ -95,7 +89,7 @@ def __init__(
             "cachedir": cachedir,
             "timeout": timeout,
             # "size_policy": "head",  # deprecated
-            "client_kwargs": {"headers": {'x-argokey': OPTIONS['argovis_api_key']}},
+            "client_kwargs": {"headers": {"x-argokey": OPTIONS["argovis_api_key"]}},
         }
         self.fs = kwargs["fs"] if "fs" in kwargs else httpstore(**self.store_opts)
 
@@ -134,9 +128,12 @@ def __repr__(self):
         summary = ["<datafetcher.argovis>"]
         summary.append("Name: %s" % self.definition)
         summary.append("API: %s" % api_server)
-        api_key = self.fs.fs.client_kwargs['headers']['x-argokey']
-        if api_key == DEFAULT['argovis_api_key']:
-            summary.append("API KEY: '%s' (get a free key at https://argovis-keygen.colorado.edu)" % api_key)
+        api_key = self.fs.fs.client_kwargs["headers"]["x-argokey"]
+        if api_key == DEFAULT["argovis_api_key"]:
+            summary.append(
+                "API KEY: '%s' (get a free key at https://argovis-keygen.colorado.edu)"
+                % api_key
+            )
         else:
             summary.append("API KEY: '%s'" % api_key)
         summary.append("Domain: %s" % format_oneline(self.cname()))
@@ -286,24 +283,32 @@ def json2dataframe(self, profiles):
         for profile in data:
             # construct metadata dictionary that will be repeated for each level
             metadict = {
-                'date': profile['timestamp'],
-                'date_qc': profile['timestamp_argoqc'],
-                'lat': profile['geolocation']['coordinates'][1],
-                'lon': profile['geolocation']['coordinates'][0],
-                'cycle_number': profile['cycle_number'],
-                'DATA_MODE': profile['data_info'][2][0][1],
-                'DIRECTION': profile['profile_direction'],
-                'platform_number': profile['_id'].split('_')[0],
-                'position_qc': profile['geolocation_argoqc'],
-                'index': 0
+                "date": profile["timestamp"],
+                "date_qc": profile["timestamp_argoqc"],
+                "lat": profile["geolocation"]["coordinates"][1],
+                "lon": profile["geolocation"]["coordinates"][0],
+                "cycle_number": profile["cycle_number"],
+                "DATA_MODE": profile["data_info"][2][0][1],
+                "DIRECTION": profile["profile_direction"],
+                "platform_number": profile["_id"].split("_")[0],
+                "position_qc": profile["geolocation_argoqc"],
+                "index": 0,
             }
             # construct a row for each level in the profile
-            for i in range(len(profile['data'][profile['data_info'][0].index('pressure')])):
+            for i in range(
+                len(profile["data"][profile["data_info"][0].index("pressure")])
+            ):
                 row = {
-                    'temp': profile['data'][profile['data_info'][0].index('temperature')][i],
-                    'pres': profile['data'][profile['data_info'][0].index('pressure')][i],
-                    'psal': profile['data'][profile['data_info'][0].index('salinity')][i],
-                    **metadict
+                    "temp": profile["data"][
+                        profile["data_info"][0].index("temperature")
+                    ][i],
+                    "pres": profile["data"][profile["data_info"][0].index("pressure")][
+                        i
+                    ],
+                    "psal": profile["data"][profile["data_info"][0].index("salinity")][
+                        i
+                    ],
+                    **metadict,
                 }
                 rows.append(row)
         df = pd.DataFrame(rows)
@@ -375,8 +380,8 @@ def to_xarray(self, errors: str = "ignore"):
         ds.attrs["Fetched_from"] = self.server
         try:
             ds.attrs["Fetched_by"] = getpass.getuser()
-        except:
-            ds.attrs["Fetched_by"] = 'anonymous'
+        except:  # noqa: E722
+            ds.attrs["Fetched_by"] = "anonymous"
         ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime("%Y/%m/%d")
         ds.attrs["Fetched_constraints"] = self.cname()
         ds.attrs["Fetched_uri"] = self.uri
@@ -435,9 +440,9 @@ def init(self, WMO=[], CYC=None, **kwargs):
     def get_url(self, wmo: int, cyc: int = None) -> str:
         """Return path toward the source file of a given wmo/cyc pair"""
         if cyc is None:
-            return  f'{self.server}/argo?platform={str(wmo)}&data=pressure,temperature,salinity'
+            return f"{self.server}/argo?platform={str(wmo)}&data=pressure,temperature,salinity"
         else:
-            return f'{self.server}/argo?id={str(wmo)}_{str(cyc).zfill(3)}&data=pressure,temperature,salinity'
+            return f"{self.server}/argo?id={str(wmo)}_{str(cyc).zfill(3)}&data=pressure,temperature,salinity"
 
     @property
     def uri(self):
@@ -488,10 +493,7 @@ def init(self, box: list, **kwargs):
 
     def get_url(self):
         """Return the URL used to download data"""
-        shape = [
-                    [self.BOX[0], self.BOX[2]],  # ll
-                    [self.BOX[1], self.BOX[3]]  # ur
-                ]
+        shape = [[self.BOX[0], self.BOX[2]], [self.BOX[1], self.BOX[3]]]  # ll  # ur
         strShape = str(shape).replace(" ", "")
         url = self.server + "/argo?data=pressure,temperature,salinity&box=" + strShape
         url += "&startDate={}".format(
@@ -558,4 +560,4 @@ def uri(self):
             for box in boxes:
                 urls.append(Fetch_box(box=box, ds=self.dataset_id).get_url())
 
-        return self.url_encode(urls)
+        return self.url_encode(urls)

argopy/data_fetchers/erddap_data.py

@@ -27,8 +27,10 @@
 from ..utils.format import format_oneline
 from ..stores import httpstore
 from ..errors import ErddapServerError, DataNotFound
-from ..stores import indexstore_pd as ArgoIndex  # make sure we work with the Pandas index store
-from ..utils import is_list_of_strings, to_list,Chunker
+from ..stores import (
+    indexstore_pd as ArgoIndex,
+)  # make sure we work with the Pandas index store
+from ..utils import is_list_of_strings, to_list, Chunker
 from .proto import ArgoDataFetcherProto
 
 
@@ -159,11 +161,15 @@ def __init__(  # noqa: C901
             # This will be used to:
             # - retrieve the list of BGC variables to ask the erddap server
             # - get <param>_data_mode information because we can't get it from the server
-            self.indexfs = kwargs['indexfs'] if 'indexfs' in kwargs else ArgoIndex(
-                index_file='argo_synthetic-profile_index.txt',  # the only available in the erddap
-                cache=kwargs['cache_index'] if 'cache_index' in kwargs else cache,
-                cachedir=cachedir,
-                timeout=timeout,
+            self.indexfs = (
+                kwargs["indexfs"]
+                if "indexfs" in kwargs
+                else ArgoIndex(
+                    index_file="argo_synthetic-profile_index.txt",  # the only available in the erddap
+                    cache=kwargs["cache_index"] if "cache_index" in kwargs else cache,
+                    cachedir=cachedir,
+                    timeout=timeout,
+                )
             )
 
             # To handle bugs in the erddap server, we need the list of parameters on the server:
@@ -615,7 +621,9 @@ def getNfromncHeader(url):
         return N
 
 
-    def post_process(self, this_ds, add_dm: bool = True, URI: list = None):  # noqa: C901
+    def post_process(
+        self, this_ds, add_dm: bool = True, URI: list = None
+    ):  # noqa: C901
         """Post-process a xarray.DataSet created from a netcdf erddap response
 
         This method can also be applied on a regular dataset to re-enforce format compliance
@@ -667,8 +675,8 @@ def post_process(self, this_ds, add_dm: bool = True, URI: list = None):  # noqa:
 
         # In the case of a parallel download, this is a trick to preserve the chunk uri in the chunk dataset:
         # (otherwise all chunks have the same list of uri)
-        Fetched_url = this_ds.attrs.get('Fetched_url', False)
-        Fetched_constraints = this_ds.attrs.get('Fetched_constraints', False)
+        Fetched_url = this_ds.attrs.get("Fetched_url", False)
+        Fetched_constraints = this_ds.attrs.get("Fetched_constraints", False)
 
         # Finally overwrite erddap attributes with those from argopy:
         raw_attrs = this_ds.attrs.copy()
@@ -691,12 +699,14 @@ def post_process(self, this_ds, add_dm: bool = True, URI: list = None):  # noqa:
         this_ds.attrs["Fetched_from"] = self.erddap.server
         try:
             this_ds.attrs["Fetched_by"] = getpass.getuser()
-        except:
-            this_ds.attrs["Fetched_by"] = 'anonymous'
+        except:  # noqa: E722
+            this_ds.attrs["Fetched_by"] = "anonymous"
         this_ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime(
             "%Y/%m/%d"
         )
-        this_ds.attrs["Fetched_constraints"] = self.cname() if not Fetched_constraints else Fetched_constraints
+        this_ds.attrs["Fetched_constraints"] = (
+            self.cname() if not Fetched_constraints else Fetched_constraints
+        )
         this_ds.attrs["Fetched_uri"] = URI if not Fetched_url else Fetched_url
         this_ds = this_ds[np.sort(this_ds.data_vars)]
 

argopy/data_fetchers/gdacftp_data.py

@@ -283,7 +283,7 @@ def _preprocess_multiprof(self, ds):
         ds.attrs["Fetched_from"] = self.server
         try:
             ds.attrs["Fetched_by"] = getpass.getuser()
-        except:
+        except:  # noqa: E722
             ds.attrs["Fetched_by"] = 'anonymous'
         ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime("%Y/%m/%d")
         ds.attrs["Fetched_constraints"] = self.cname()
@@ -352,7 +352,7 @@ def to_xarray(self, errors: str = "ignore"):
         ds.attrs["Fetched_from"] = self.server
         try:
             ds.attrs["Fetched_by"] = getpass.getuser()
-        except:
+        except:  # noqa: E722
             ds.attrs["Fetched_by"] = 'anonymous'
         ds.attrs["Fetched_date"] = pd.to_datetime("now", utc=True).strftime("%Y/%m/%d")
         ds.attrs["Fetched_constraints"] = self.cname()