From be74713e0091491d6a632115fbf2f54fe147bf4d Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Thu, 7 Mar 2024 07:37:19 +0000
Subject: [PATCH 01/25] fix

---
 ecml_tools/create/loaders.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ecml_tools/create/loaders.py b/ecml_tools/create/loaders.py
index 1b14a5f..62d7aea 100644
--- a/ecml_tools/create/loaders.py
+++ b/ecml_tools/create/loaders.py
@@ -190,7 +190,7 @@ def initialise(self, check_name=True):
         variables = self.minimal_input.variables
         self.print(f"Found {len(variables)} variables : {','.join(variables)}.")
 
-        variables_with_nans = self.config.get("has_nans", [])
+        variables_with_nans = self.main_config.get("has_nans", [])
 
         ensembles = self.minimal_input.ensembles
         self.print(f"Found {len(ensembles)} ensembles : {','.join([str(_) for _ in ensembles])}.")

From b1e206956687232c6d01b38c9bddfe48b329f794 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Thu, 7 Mar 2024 15:14:18 +0000
Subject: [PATCH 02/25] cleanup config

---
 ecml_tools/create/config.py | 139 +++++++++++++-----------------------
 1 file changed, 48 insertions(+), 91 deletions(-)

diff --git a/ecml_tools/create/config.py b/ecml_tools/create/config.py
index c27e8ea..1306f26 100644
--- a/ecml_tools/create/config.py
+++ b/ecml_tools/create/config.py
@@ -19,6 +19,34 @@
 LOG = logging.getLogger(__name__)
 
 
+def _get_first_key_if_dict(x):
+    if isinstance(x, str):
+        return x
+    return list(x.keys())[0]
+
+
+def ensure_element_in_list(lst, elt, index):
+    if elt in lst:
+        assert lst[index] == elt
+        return lst
+
+    _lst = [_get_first_key_if_dict(d) for d in lst]
+    if elt in _lst:
+        assert _lst[index] == elt
+        return lst
+
+    return lst[:index] + [elt] + lst[index:]
+
+
+def check_dict_value_and_set(dic, key, value):
+    if key in dic:
+        if dic[key] == value:
+            return
+        raise ValueError(f"Cannot use {key}={dic[key]}. Must use {value}.")
+    print(f"Setting {key}={value} in config")
+    dic[key] = value
+
+
 class DictObj(dict):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -111,11 +139,12 @@ def statistics(self):
 
 
 class LoadersConfig(Config):
-    purpose = "undefined"
 
     def __init__(self, config, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
 
+        # TODO: should use a json schema to validate the config
+
         if "description" not in self:
             raise ValueError("Must provide a description in the config.")
 
@@ -129,6 +158,8 @@ def __init__(self, config, *args, **kwargs):
 
         if "dates" in self.output:
             raise ValueError("Obsolete: Dates should not be provided in output config.")
+        if not isinstance(self.dates, dict):
+            raise ValueError(f"Dates must be a dict. Got {self.dates}")
 
         # deprecated/obsolete
         if "order" in self.output:
@@ -138,17 +169,24 @@ def __init__(self, config, *args, **kwargs):
         if "loop" in self:
             raise ValueError(f"Do not use 'loop'. Use dates instead. {list(self.keys())}")
 
-        self.normalise()
-
         if "licence" not in self:
-            raise ValueError("Must provide a licence in the config.")
+            self.licence = "unknown"
+            print(f"❗ Setting licence={self.licence} because it was not provided.")
         if "copyright" not in self:
-            raise ValueError("Must provide a copyright in the config.")
+            self.copyright = "unknown"
+            print(f"❗ Setting copyright={self.copyright} because it was not provided.")
 
-        if not isinstance(self.dates, dict):
-            raise ValueError(f"Dates must be a dict. Got {self.dates}")
+        check_dict_value_and_set(self.output, "flatten_grid", True)
+        check_dict_value_and_set(self.output, "ensemble_dimension", 2)
+
+        assert isinstance(self.output.order_by, (list, tuple)), self.output.order_by
+        self.output.order_by = ensure_element_in_list(self.output.order_by, "number", self.output.ensemble_dimension)
+
+        order_by = self.output.order_by
+        assert len(order_by) == 3, order_by
+        assert _get_first_key_if_dict(order_by[0]) == "valid_datetime", order_by
+        assert _get_first_key_if_dict(order_by[2]) == "number", order_by
 
-    def normalise(self):
         if "order_by" in self.output:
             self.output.order_by = normalize_order_by(self.output.order_by)
 
@@ -158,46 +196,7 @@ def normalise(self):
         self.reading_chunks = self.get("reading_chunks")
         assert "flatten_values" not in self.output
         assert "flatten_grid" in self.output, self.output
-
         assert "statistics" in self.output
-        statistics_axis_name = self.output.statistics
-        statistics_axis = -1
-        for i, k in enumerate(self.output.order_by):
-            if k == statistics_axis_name:
-                statistics_axis = i
-
-        assert statistics_axis >= 0, f"{self.output.statistics} not in {list(self.output.order_by.keys())}"
-
-        self.statistics_names = self.output.order_by[statistics_axis_name]
-
-        # TODO: consider 2D grid points
-        self.statistics_axis = statistics_axis
-
-    @classmethod
-    def _get_first_key_if_dict(cls, x):
-        if isinstance(x, str):
-            return x
-        return list(x.keys())[0]
-
-    def check_dict_value_and_set(self, dic, key, value):
-        if key in dic:
-            if dic[key] == value:
-                return
-            raise ValueError(f"Cannot use {key}={dic[key]} with {self.purpose} purpose. Must use {value}.")
-        print(f"Setting {key}={value} because purpose={self.purpose}")
-        dic[key] = value
-
-    def ensure_element_in_list(self, lst, elt, index):
-        if elt in lst:
-            assert lst[index] == elt
-            return lst
-
-        _lst = [self._get_first_key_if_dict(d) for d in lst]
-        if elt in _lst:
-            assert _lst[index] == elt
-            return lst
-
-        return lst[:index] + [elt] + lst[index:]
 
     def get_serialisable_dict(self):
         return _prepare_serialisation(self)
@@ -206,42 +205,6 @@ def get_variables_names(self):
         return self.output.order_by[self.output.statistics]
 
 
-class UnknownPurposeConfig(LoadersConfig):
-    purpose = "unknown"
-
-    def normalise(self):
-        self.output.flatten_grid = self.output.get("flatten_grid", False)
-        self.output.ensemble_dimension = self.output.get("ensemble_dimension", False)
-        super().normalise()  # must be called last
-
-
-class AifsPurposeConfig(LoadersConfig):
-    purpose = "aifs"
-
-    def normalise(self):
-        if "licence" not in self:
-            self.licence = "CC-BY-4.0"
-            print(f"❗ Setting licence={self.licence} because it was not provided.")
-        if "copyright" not in self:
-            self.copyright = "ecmwf"
-            print(f"❗ Setting copyright={self.copyright} because it was not provided.")
-
-        self.check_dict_value_and_set(self.output, "flatten_grid", True)
-        self.check_dict_value_and_set(self.output, "ensemble_dimension", 2)
-
-        assert isinstance(self.output.order_by, (list, tuple)), self.output.order_by
-        self.output.order_by = self.ensure_element_in_list(
-            self.output.order_by, "number", self.output.ensemble_dimension
-        )
-
-        order_by = self.output.order_by
-        assert len(order_by) == 3, order_by
-        assert self._get_first_key_if_dict(order_by[0]) == "valid_datetime", order_by
-        assert self._get_first_key_if_dict(order_by[2]) == "number", order_by
-
-        super().normalise()  # must be called last
-
-
 def _prepare_serialisation(o):
     if isinstance(o, dict):
         dic = {}
@@ -272,21 +235,15 @@ def _prepare_serialisation(o):
     return str(o)
 
 
-CONFIGS = {
-    None: UnknownPurposeConfig,
-    "aifs": AifsPurposeConfig,
-}
-
-
 def loader_config(config):
     config = Config(config)
-    obj = CONFIGS[config.get("purpose")](config)
+    obj = LoadersConfig(config)
 
     # yaml round trip to check that serialisation works as expected
     copy = obj.get_serialisable_dict()
     copy = yaml.load(yaml.dump(copy), Loader=yaml.SafeLoader)
     copy = Config(copy)
-    copy = CONFIGS[config.get("purpose")](config)
+    copy = LoadersConfig(config)
     assert yaml.dump(obj) == yaml.dump(copy), (obj, copy)
 
     return copy

From c658a99b06801ae08a4295753a75eb01d6403e37 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Thu, 7 Mar 2024 16:58:41 +0000
Subject: [PATCH 03/25] cleanup

---
 ecml_tools/create/config.py                   |  3 ---
 .../create/functions/actions/accumulations.py | 21 ++++++++++++++++---
 ecml_tools/create/input.py                    |  8 ++++---
 ecml_tools/create/template.py                 |  3 ++-
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/ecml_tools/create/config.py b/ecml_tools/create/config.py
index 1306f26..7fddb22 100644
--- a/ecml_tools/create/config.py
+++ b/ecml_tools/create/config.py
@@ -201,9 +201,6 @@ def __init__(self, config, *args, **kwargs):
     def get_serialisable_dict(self):
         return _prepare_serialisation(self)
 
-    def get_variables_names(self):
-        return self.output.order_by[self.output.statistics]
-
 
 def _prepare_serialisation(o):
     if isinstance(o, dict):
diff --git a/ecml_tools/create/functions/actions/accumulations.py b/ecml_tools/create/functions/actions/accumulations.py
index 75f0359..5c6250f 100644
--- a/ecml_tools/create/functions/actions/accumulations.py
+++ b/ecml_tools/create/functions/actions/accumulations.py
@@ -36,6 +36,22 @@ def normalise_time_to_hours(r):
     return r
 
 
+def normalise_number(r):
+    if "number" not in r:
+        return r
+    number = r["number"]
+    number = to_list(number)
+
+    if len(number) > 4 and (number[1] == "to" and number[3] == "by"):
+        return list(range(int(number[0]), int(number[2]) + 1, int(number[4])))
+
+    if len(number) > 2 and number[1] == "to":
+        return list(range(int(number[0]), int(number[2]) + 1))
+
+    r["number"] = number
+    return r
+
+
 def accumulations(context, dates, **request):
     to_list(request["param"])
     class_ = request["class"]
@@ -52,6 +68,7 @@ def accumulations(context, dates, **request):
     for r in requests:
         r = {k: v for k, v in r.items() if v != ("-",)}
         r = normalise_time_to_hours(r)
+        r = normalise_number(r)
 
         if DEBUG:
             print(f"load_source({source_name},  {r}")
@@ -76,9 +93,7 @@ def accumulations(context, dates, **request):
 #      accumulation_period: 6h
     """
     )
-    dates = yaml.safe_load(
-        "[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]"
-    )
+    dates = yaml.safe_load("[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]")
     dates = to_datetime_list(dates)
 
     DEBUG = True
diff --git a/ecml_tools/create/input.py b/ecml_tools/create/input.py
index 3631252..3731ae7 100644
--- a/ecml_tools/create/input.py
+++ b/ecml_tools/create/input.py
@@ -67,10 +67,11 @@ def time_delta_to_string(delta):
 
 
 def import_function(name, kind):
-    return importlib.import_module(
+    module = importlib.import_module(
         f"..functions.{kind}.{name}",
         package=__name__,
-    ).execute
+    )
+    return module.execute
 
 
 def is_function(name, kind):
@@ -78,7 +79,8 @@ def is_function(name, kind):
     try:
         import_function(name, kind)
         return True
-    except ImportError:
+    except ImportError as e:
+        print(e)
         return False
 
 
diff --git a/ecml_tools/create/template.py b/ecml_tools/create/template.py
index 230a5e3..518787b 100644
--- a/ecml_tools/create/template.py
+++ b/ecml_tools/create/template.py
@@ -109,7 +109,8 @@ def get_result(self, key):
         key = tuple(key)
         if key in self.results:
             return self.results[key]
-        raise ValueError(f"Cannot find result {key}")
+        all_keys = sorted(list(self.results.keys()))
+        raise ValueError(f"Cannot find result {key} in {all_keys}")
 
 
 class Substitution:

From 8117b916863a04cdc73b09a995a4189c63954ffc Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Thu, 7 Mar 2024 17:03:34 +0000
Subject: [PATCH 04/25] updated ensemble perturbations

---
 .../actions/ensemble_perturbations.py         | 55 ++++++-----
 tests/create-perturbations.yaml               | 94 ++++++++++++-------
 2 files changed, 89 insertions(+), 60 deletions(-)

diff --git a/ecml_tools/create/functions/actions/ensemble_perturbations.py b/ecml_tools/create/functions/actions/ensemble_perturbations.py
index 1ff3fdf..dc599c7 100644
--- a/ecml_tools/create/functions/actions/ensemble_perturbations.py
+++ b/ecml_tools/create/functions/actions/ensemble_perturbations.py
@@ -15,7 +15,8 @@
 from climetlab.readers.grib.output import new_grib_output
 
 from ecml_tools.create.check import check_data_values
-from ecml_tools.create.functions import assert_is_fieldset, wrapped_mars_source
+from ecml_tools.create.functions import assert_is_fieldset
+from ecml_tools.create.functions.actions.mars import mars
 
 
 def to_list(x):
@@ -48,35 +49,39 @@ def normalise_request(request):
     return request
 
 
-def ensembles_perturbations(ensembles, center, mean, remapping={}, patches={}):
-    from climetlab import load_source
+def load_if_needed(context, dates, dict_or_dataset):
+    if isinstance(dict_or_dataset, dict):
+        dict_or_dataset = normalise_request(dict_or_dataset)
+        dict_or_dataset = mars(context, dates, dict_or_dataset)
+    return dict_or_dataset
 
-    ensembles = normalise_request(ensembles)
-    center = normalise_request(center)
-    mean = normalise_request(mean)
 
-    number_list = ensembles["number"]
-    n_numbers = len(number_list)
+def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}, patches={}):
+    ensembles = load_if_needed(context, dates, ensembles)
+    center = load_if_needed(context, dates, center)
+    mean = load_if_needed(context, dates, mean)
 
     keys = ["param", "level", "valid_datetime", "date", "time", "step", "number"]
 
     print(f"Retrieving ensemble data with {ensembles}")
-    ensembles = wrapped_mars_source(**ensembles).order_by(*keys)
     print(f"Retrieving center data with {center}")
-    center = wrapped_mars_source(**center).order_by(*keys)
     print(f"Retrieving mean data with {mean}")
-    mean = wrapped_mars_source(**mean).order_by(*keys)
-
-    assert len(mean) * n_numbers == len(ensembles), (
-        len(mean),
-        n_numbers,
-        len(ensembles),
-    )
-    assert len(center) * n_numbers == len(ensembles), (
-        len(center),
-        n_numbers,
-        len(ensembles),
-    )
+
+    ensembles = ensembles.order_by(*keys)
+    center = center.order_by(*keys)
+    mean = mean.order_by(*keys)
+
+    number_list = ensembles.unique_values("number")["number"]
+    n_numbers = len(number_list)
+
+    assert len(mean) == len(center), (len(mean), len(center))
+    if len(center) * n_numbers != len(ensembles):
+        print(len(center), n_numbers, len(ensembles))
+        for f in ensembles:
+            print("Ensembles: ", f)
+        for f in center:
+            print("Center: ", f)
+        raise ValueError(f"Inconsistent number of fields: {len(center)} * {n_numbers} != {len(ensembles)}")
 
     # prepare output tmp file so we can read it back
     tmp = temp_file()
@@ -134,6 +139,8 @@ def ensembles_perturbations(ensembles, center, mean, remapping={}, patches={}):
 
     out.close()
 
+    from climetlab import load_source
+
     ds = load_source("file", path)
     assert_is_fieldset(ds)
     # save a reference to the tmp file so it is deleted
@@ -145,7 +152,7 @@ def ensembles_perturbations(ensembles, center, mean, remapping={}, patches={}):
     return ds
 
 
-execute = ensembles_perturbations
+execute = ensemble_perturbations
 
 if __name__ == "__main__":
     import yaml
@@ -192,5 +199,5 @@ def ensembles_perturbations(ensembles, center, mean, remapping={}, patches={}):
     for k, v in config.items():
         print(k, v)
 
-    for f in ensembles_perturbations(**config):
+    for f in ensemble_perturbations(**config):
         print(f, f.to_numpy().mean())
diff --git a/tests/create-perturbations.yaml b/tests/create-perturbations.yaml
index 2b9c795..c4873f4 100644
--- a/tests/create-perturbations.yaml
+++ b/tests/create-perturbations.yaml
@@ -6,55 +6,77 @@ config_format_version: 2
 
 common:
   mars_request: &common
-    name: mars
     class: ea
-    date: $datetime_format($dates,%Y%m%d)
-    time: $datetime_format($dates,%H%M)
-    expver: '0001'
+    expver: "0001"
     grid: 20.0/20.0
     levtype: sfc
-    param: [2t, tp]
-  dates: &dates_anchor
-    start: 2020-12-30 00:00:00
-    end: 2021-01-03 12:00:00
-    frequency: 12h
+    param: [2t]
+  mars_request_acc: &common_acc
+    class: ea
+    expver: "0001"
+    grid: 20.0/20.0
+    levtype: sfc
+    param: [tp]
+
+  ensembles: &ensembles
+    stream: enda
+    type: an
+    number: [0, 2, 4]
+    # number: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+  center: &center
+    stream: oper
+    type: an
+  mean: &mean
+    stream: enda
+    type: em
 
 dates:
-  <<: *dates_anchor
+  start: 2020-12-30 00:00:00
+  end: 2021-01-03 12:00:00
+  frequency: 12h
   group_by: monthly
 
-input:
-  dates:
-    <<: *dates_anchor
-    function:
-      name: ensemble_perturbations
-      ensembles: # the ensemble data has one additional dimension
-        <<: *common
-        stream: enda
-        type: an
-        number: 0/to/4/by/2
-        # number: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-      center: # the new center of the data
-        <<: *common
-        stream: oper
-        type: an
-
-      mean: # the previous center of the data
-        <<: *common
-        stream: enda
-        type: em
+include: # This "include" will be renamed
+  - join:
+      - mars:
+          <<: *ensembles
+          <<: *common
+      - accumulations:
+          <<: *ensembles
+          <<: *common_acc
+  - join:
+      - mars:
+          <<: *mean
+          <<: *common
+      - accumulations:
+          <<: *mean
+          <<: *common_acc
+  - join:
+      - mars:
+          <<: *center
+          <<: *common
+      - accumulations:
+          <<: *center
+          <<: *common_acc
 
+input:
+  ensemble_perturbations:
+    # the ensemble data which has one additional dimension
+    ensembles: ${include.0.join}
+    # the previous center of the data
+    mean: ${include.1.join}
+    # the new center of the data
+    center: ${include.2.join}
 
 output:
-  chunking: {dates: 1}
+  chunking: { dates: 1 }
   dtype: float32
   flatten_grid: True
   order_by:
-  - valid_datetime
-  - param_level
-  - number
+    - valid_datetime
+    - param_level
+    - number
   statistics: param_level
   statistics_end: 2021
   remapping:
-    param_level: '{param}_{levelist}'
+    param_level: "{param}_{levelist}"

From c5f925eb27599625aef18f202a93586991b1c1cb Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Fri, 8 Mar 2024 10:39:30 +0000
Subject: [PATCH 05/25] apply qa : apply black with line lenght 120 apply isort
 with --force-single-line-imports comply with ruff update precomit

---
 .pre-commit-config.yaml                       | 74 ++++++++++++-------
 ecml_tools/__main__.py                        |  4 +-
 ecml_tools/commands/__init__.py               |  8 +-
 ecml_tools/commands/copy.py                   |  3 +-
 ecml_tools/commands/create.py                 |  4 +-
 ecml_tools/commands/inspect/__init__.py       |  6 +-
 ecml_tools/commands/inspect/zarr.py           | 44 ++++-------
 ecml_tools/commands/scan.py                   |  4 +-
 ecml_tools/create/check.py                    |  2 +-
 ecml_tools/create/functions/actions/grib.py   |  8 +-
 ecml_tools/create/functions/actions/netcdf.py |  8 +-
 ecml_tools/create/functions/actions/source.py |  4 +-
 .../create/functions/actions/tendencies.py    |  8 +-
 .../create/functions/steps/rotate_winds.py    | 10 +--
 ecml_tools/create/input.py                    | 25 +++----
 ecml_tools/create/loaders.py                  | 29 ++++----
 ecml_tools/create/patch.py                    |  4 +-
 ecml_tools/create/statistics.py               |  4 +-
 ecml_tools/create/utils.py                    |  4 +-
 ecml_tools/create/writer.py                   |  6 --
 ecml_tools/data.py                            | 15 ++--
 ecml_tools/grids.py                           |  4 +-
 ecml_tools/provenance.py                      | 10 +--
 ecml_tools/utils/dates/__init__.py            |  4 +-
 ecml_tools/utils/humanize.py                  |  2 +-
 ecml_tools/utils/text.py                      |  7 +-
 scripts/speed-test.py                         |  4 +-
 scripts/zip-dataset.py                        |  8 +-
 tests/test_data.py                            | 28 ++++---
 tests/test_indexing.py                        |  4 +-
 30 files changed, 143 insertions(+), 202 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0269aba..94fcb3c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,29 +1,47 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-    -   id: check-yaml # Check YAML files for syntax errors
-    -   id: debug-statements # Check for debugger imports and py37+ breakpoint()
-#    -   id: end-of-file-fixer # Ensure files end in a newline
-#    -   id: trailing-whitespace # Trailing whitespace checker
-    -   id: no-commit-to-branch # Prevent committing to main / master
--   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.1.1
-    hooks:
-      - id: black
-        args: [--line-length=120]
--   repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        name: isort (python)
-      - id: isort
-        name: isort (cython)
-        types: [cython]
-      - id: isort
-        name: isort (pyi)
-        types: [pyi]
--   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-    -   id: flake8
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml # Check YAML files for syntax errors only
+    args: [--unsafe, --allow-multiple-documents]
+  - id: debug-statements # Check for debugger imports and py37+ breakpoint()
+#  - id: end-of-file-fixer # Ensure files end in a newline
+#  - id: trailing-whitespace # Trailing whitespace checker
+  - id: no-commit-to-branch # Prevent committing to main / master
+  - id: check-added-large-files # Check for large files added to git
+  - id: check-merge-conflict # Check for files that contain merge conflict
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 24.1.1
+  hooks:
+    - id: black
+      args: [--line-length=120]
+- repo: https://github.com/pycqa/isort
+  rev: 5.13.2
+  hooks:
+  - id: isort
+    args:
+    - -l 120
+    - --force-single-line-imports
+    - --profile black
+#- repo: https://github.com/pycqa/flake8
+#  rev: 7.0.0
+#  hooks:
+#  -   id: flake8
+#  #    args: [--exit-zero]
+#  #    verbose: true
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.3.0
+  hooks:
+  - id: ruff
+    args:
+    - --line-length=120
+    - --ignore=E203
+    - --fix
+    - --exit-non-zero-on-fix
+    - --preview 
+#- repo: https://github.com/pre-commit/mirrors-mypy
+#  rev: v0.720
+#  hooks:
+#  -   id: mypy
+#      verbose: true
+#      entry: bash -c 'mypy "$@" || true' --
diff --git a/ecml_tools/__main__.py b/ecml_tools/__main__.py
index bad1f0f..2ac1a15 100644
--- a/ecml_tools/__main__.py
+++ b/ecml_tools/__main__.py
@@ -21,9 +21,7 @@
 
 
 def main():
-    parser = argparse.ArgumentParser(
-        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
-    )
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
 
     parser.add_argument(
         "--version",
diff --git a/ecml_tools/commands/__init__.py b/ecml_tools/commands/__init__.py
index 41bf43f..0932e6e 100644
--- a/ecml_tools/commands/__init__.py
+++ b/ecml_tools/commands/__init__.py
@@ -25,13 +25,7 @@ def register(here, package, select, fail=None):
         full = os.path.join(here, p)
         if p.startswith("_"):
             continue
-        if not (
-            p.endswith(".py")
-            or (
-                os.path.isdir(full)
-                and os.path.exists(os.path.join(full, "__init__.py"))
-            )
-        ):
+        if not (p.endswith(".py") or (os.path.isdir(full) and os.path.exists(os.path.join(full, "__init__.py")))):
             continue
 
         name, _ = os.path.splitext(p)
diff --git a/ecml_tools/commands/copy.py b/ecml_tools/commands/copy.py
index fe62948..2a458ad 100644
--- a/ecml_tools/commands/copy.py
+++ b/ecml_tools/commands/copy.py
@@ -8,7 +8,8 @@
 import logging
 import os
 import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import as_completed
 
 import tqdm
 
diff --git a/ecml_tools/commands/create.py b/ecml_tools/commands/create.py
index 2aff08c..5c28ed3 100644
--- a/ecml_tools/commands/create.py
+++ b/ecml_tools/commands/create.py
@@ -8,9 +8,7 @@ class Create(Command):
     timestamp = True
 
     def add_arguments(self, command_parser):
-        command_parser.add_argument(
-            "--overwrite", action="store_true", help="Overwrite existing files"
-        )
+        command_parser.add_argument("--overwrite", action="store_true", help="Overwrite existing files")
         command_parser.add_argument("config", help="Configuration file")
         command_parser.add_argument("path", help="Path to store the created data")
 
diff --git a/ecml_tools/commands/inspect/__init__.py b/ecml_tools/commands/inspect/__init__.py
index 93b6b65..d62ab19 100644
--- a/ecml_tools/commands/inspect/__init__.py
+++ b/ecml_tools/commands/inspect/__init__.py
@@ -31,11 +31,7 @@ def add_arguments(self, command_parser):
     def run(self, args):
         dic = vars(args)
         for path in dic.pop("path"):
-            if (
-                os.path.isdir(path)
-                or path.endswith(".zarr.zip")
-                or path.endswith(".zarr")
-            ):
+            if os.path.isdir(path) or path.endswith(".zarr.zip") or path.endswith(".zarr"):
                 self.inspect_zarr(path=path, **dic)
             else:
                 raise ValueError(f"Unknown file type: {path}")
diff --git a/ecml_tools/commands/inspect/zarr.py b/ecml_tools/commands/inspect/zarr.py
index 84b364d..311000a 100644
--- a/ecml_tools/commands/inspect/zarr.py
+++ b/ecml_tools/commands/inspect/zarr.py
@@ -16,9 +16,14 @@
 import semantic_version
 import tqdm
 
-from ecml_tools.data import open_dataset, open_zarr
-from ecml_tools.utils.humanize import bytes, number, when
-from ecml_tools.utils.text import dotted_line, progress, table
+from ecml_tools.data import open_dataset
+from ecml_tools.data import open_zarr
+from ecml_tools.utils.humanize import bytes
+from ecml_tools.utils.humanize import number
+from ecml_tools.utils.humanize import when
+from ecml_tools.utils.text import dotted_line
+from ecml_tools.utils.text import progress
+from ecml_tools.utils.text import table
 
 LOG = logging.getLogger(__name__)
 
@@ -28,9 +33,7 @@ def compute_directory_size(path):
         return None, None
     size = 0
     n = 0
-    for dirpath, _, filenames in tqdm.tqdm(
-        os.walk(path), desc="Computing size", leave=False
-    ):
+    for dirpath, _, filenames in tqdm.tqdm(os.walk(path), desc="Computing size", leave=False):
         for filename in filenames:
             file_path = os.path.join(dirpath, filename)
             size += os.path.getsize(file_path)
@@ -311,11 +314,7 @@ def progress(self):
         assert build_flags.size == build_lengths.size
 
         latest_write_timestamp = self.zarr.attrs.get("latest_write_timestamp")
-        latest = (
-            datetime.datetime.fromisoformat(latest_write_timestamp)
-            if latest_write_timestamp
-            else None
-        )
+        latest = datetime.datetime.fromisoformat(latest_write_timestamp) if latest_write_timestamp else None
 
         if not all(build_flags):
             if latest:
@@ -323,9 +322,7 @@ def progress(self):
             else:
                 print("🪫  Dataset not ready.")
             total = sum(build_lengths)
-            built = sum(
-                ln if flag else 0 for ln, flag in zip(build_lengths, build_flags)
-            )
+            built = sum(ln if flag else 0 for ln, flag in zip(build_lengths, build_flags))
             print(
                 "📈 Progress:",
                 progress(built, total, width=50),
@@ -411,9 +408,7 @@ def last_date(self):
         assert isinstance(time, int), (time, type(time))
         if time > 100:
             time = time // 100
-        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(
-            hours=time
-        )
+        return datetime.datetime.fromisoformat(monthly["stop"]) + datetime.timedelta(hours=time)
 
     @property
     def frequency(self):
@@ -472,12 +467,8 @@ def _info(self, verbose, history, statistics, **kwargs):
 
         # for backward compatibility
         if "climetlab" in z.attrs:
-            climetlab_version = (
-                z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
-            )
-            print(
-                f"climetlab version used to create this zarr: {climetlab_version}. Not supported."
-            )
+            climetlab_version = z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
+            print(f"climetlab version used to create this zarr: {climetlab_version}. Not supported.")
             return
 
         version = z.attrs.get("version")
@@ -503,12 +494,7 @@ def initialised(self):
                 return datetime.datetime.fromisoformat(record["timestamp"])
 
         # Sometimes the first record is missing
-        timestamps = sorted(
-            [
-                datetime.datetime.fromisoformat(d["timestamp"])
-                for d in self.metadata.get("history", [])
-            ]
-        )
+        timestamps = sorted([datetime.datetime.fromisoformat(d["timestamp"]) for d in self.metadata.get("history", [])])
         if timestamps:
             return timestamps[0]
 
diff --git a/ecml_tools/commands/scan.py b/ecml_tools/commands/scan.py
index de72d07..e5e22a2 100644
--- a/ecml_tools/commands/scan.py
+++ b/ecml_tools/commands/scan.py
@@ -16,9 +16,7 @@ class Scan(Command):
     timestamp = True
 
     def add_arguments(self, command_parser):
-        command_parser.add_argument(
-            "--extension", default=".grib", help="Extension of the files to scan"
-        )
+        command_parser.add_argument("--extension", default=".grib", help="Extension of the files to scan")
         command_parser.add_argument(
             "--magic",
             help="File 'magic' to use to identify the file type. Overrides --extension",
diff --git a/ecml_tools/create/check.py b/ecml_tools/create/check.py
index ecc37d9..aff292d 100644
--- a/ecml_tools/create/check.py
+++ b/ecml_tools/create/check.py
@@ -153,7 +153,7 @@ class StatisticsValueError(ValueError):
 
 def check_data_values(arr, *, name: str, log=[], allow_nan=False):
     if allow_nan is False:
-        allow_nan = lambda x: False
+        allow_nan = lambda x: False  # noqa: E731
     if allow_nan(name):
         arr = arr[~np.isnan(arr)]
 
diff --git a/ecml_tools/create/functions/actions/grib.py b/ecml_tools/create/functions/actions/grib.py
index 0607c22..d20841f 100644
--- a/ecml_tools/create/functions/actions/grib.py
+++ b/ecml_tools/create/functions/actions/grib.py
@@ -19,9 +19,7 @@ def check(ds, paths, **kwargs):
             count *= len(v)
 
     if len(ds) != count:
-        raise ValueError(
-            f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})"
-        )
+        raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
 
 
 def execute(context, dates, path, *args, **kwargs):
@@ -31,9 +29,7 @@ def execute(context, dates, path, *args, **kwargs):
     dates = [d.isoformat() for d in dates]
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(
-            *args, date=dates, **kwargs
-        )
+        paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
 
         for name in ("grid", "area", "rotation", "frame", "resol", "bitmap"):
             if name in kwargs:
diff --git a/ecml_tools/create/functions/actions/netcdf.py b/ecml_tools/create/functions/actions/netcdf.py
index 335bd92..1dd8a7e 100644
--- a/ecml_tools/create/functions/actions/netcdf.py
+++ b/ecml_tools/create/functions/actions/netcdf.py
@@ -18,9 +18,7 @@ def check(what, ds, paths, **kwargs):
             count *= len(v)
 
     if len(ds) != count:
-        raise ValueError(
-            f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, {what}s={paths})"
-        )
+        raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, {what}s={paths})")
 
 
 def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
@@ -30,9 +28,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
     ds = load_source("empty")
 
     for path in given_paths:
-        paths = Pattern(path, ignore_missing_keys=True).substitute(
-            *args, date=dates, **kwargs
-        )
+        paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
 
         levels = kwargs.get("level", kwargs.get("levelist"))
 
diff --git a/ecml_tools/create/functions/actions/source.py b/ecml_tools/create/functions/actions/source.py
index 13f00b5..f721ec0 100644
--- a/ecml_tools/create/functions/actions/source.py
+++ b/ecml_tools/create/functions/actions/source.py
@@ -41,9 +41,7 @@ def source(context, dates, **kwargs):
       time: $from_dates
     """
     )
-    dates = yaml.safe_load(
-        "[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]"
-    )
+    dates = yaml.safe_load("[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]")
     dates = to_datetime_list(dates)
 
     DEBUG = True
diff --git a/ecml_tools/create/functions/actions/tendencies.py b/ecml_tools/create/functions/actions/tendencies.py
index b859146..fc8e6e6 100644
--- a/ecml_tools/create/functions/actions/tendencies.py
+++ b/ecml_tools/create/functions/actions/tendencies.py
@@ -98,9 +98,7 @@ def tendencies(dates, time_increment, **kwargs):
             ################
 
             assert x.shape == c.shape, c.shape
-            print(
-                f"Computing data for {field.metadata('valid_datetime')}={field}-{b_field}"
-            )
+            print(f"Computing data for {field.metadata('valid_datetime')}={field}-{b_field}")
             out.write(x, template=field)
 
     out.close()
@@ -137,9 +135,7 @@ def tendencies(dates, time_increment, **kwargs):
     """
     )["config"]
 
-    dates = yaml.safe_load(
-        "[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]"
-    )
+    dates = yaml.safe_load("[2022-12-30 18:00, 2022-12-31 00:00, 2022-12-31 06:00, 2022-12-31 12:00]")
     dates = to_datetime_list(dates)
 
     DEBUG = True
diff --git a/ecml_tools/create/functions/steps/rotate_winds.py b/ecml_tools/create/functions/steps/rotate_winds.py
index 4d9feb8..a41e2d7 100644
--- a/ecml_tools/create/functions/steps/rotate_winds.py
+++ b/ecml_tools/create/functions/steps/rotate_winds.py
@@ -25,9 +25,7 @@ def rotate_winds(lats, lons, x_wind, y_wind, source_projection, target_projectio
     source_projection = pyproj.Proj(source_projection)
     target_projection = pyproj.Proj(target_projection)
 
-    transformer = pyproj.transformer.Transformer.from_proj(
-        source_projection, target_projection
-    )
+    transformer = pyproj.transformer.Transformer.from_proj(source_projection, target_projection)
 
     # To compute the new vector components:
     # 1) perturb each position in the direction of the winds
@@ -127,11 +125,7 @@ def execute(
             lons,
             x.to_numpy(reshape=False),
             y.to_numpy(reshape=False),
-            (
-                source_projection
-                if source_projection is not None
-                else CRS.from_cf(x.grid_mapping)
-            ),
+            (source_projection if source_projection is not None else CRS.from_cf(x.grid_mapping)),
             target_projection,
         )
 
diff --git a/ecml_tools/create/input.py b/ecml_tools/create/input.py
index 3731ae7..638ac57 100644
--- a/ecml_tools/create/input.py
+++ b/ecml_tools/create/input.py
@@ -12,7 +12,8 @@
 import time
 from collections import defaultdict
 from copy import deepcopy
-from functools import cached_property, wraps
+from functools import cached_property
+from functools import wraps
 
 import numpy as np
 from climetlab.core.order import build_remapping
@@ -20,15 +21,13 @@
 
 from ecml_tools.utils.dates import Dates
 
-from .template import (
-    Context,
-    notify_result,
-    resolve,
-    substitute,
-    trace,
-    trace_datasource,
-    trace_select,
-)
+from .template import Context
+from .template import notify_result
+from .template import resolve
+from .template import substitute
+from .template import trace
+from .template import trace_datasource
+from .template import trace_select
 from .utils import seconds
 
 LOG = logging.getLogger(__name__)
@@ -75,7 +74,7 @@ def import_function(name, kind):
 
 
 def is_function(name, kind):
-    name, delta = parse_function_name(name)
+    name, delta = parse_function_name(name)  # noqa
     try:
         import_function(name, kind)
         return True
@@ -541,7 +540,7 @@ def select(self, dates):
 
     @property
     def function(self):
-        name, delta = parse_function_name(self.name)
+        # name, delta = parse_function_name(self.name)
         return import_function(self.name, "actions")
 
     def __repr__(self):
@@ -785,7 +784,7 @@ def action_factory(config, context, action_path):
         args, kwargs = [], config[key]
 
     cls = dict(
-        date_shift=DateShiftAction,
+        # date_shift=DateShiftAction,
         # date_filter=DateFilterAction,
         include=IncludeAction,
         concat=ConcatAction,
diff --git a/ecml_tools/create/loaders.py b/ecml_tools/create/loaders.py
index 62d7aea..0fa7ed3 100644
--- a/ecml_tools/create/loaders.py
+++ b/ecml_tools/create/loaders.py
@@ -17,19 +17,22 @@
 from ecml_tools.data import open_dataset
 from ecml_tools.utils.dates.groups import Groups
 
-from .check import DatasetName, check_data_values
-from .config import build_output, loader_config
+from .check import DatasetName
+from .check import check_data_values
+from .config import build_output
+from .config import loader_config
 from .input import build_input
-from .statistics import TempStatistics, compute_statistics
-from .utils import (
-    bytes,
-    compute_directory_sizes,
-    normalize_and_check_dates,
-    progress_bar,
-    seconds,
-)
-from .writer import CubesFilter, ViewCacheArray
-from .zarr import ZarrBuiltRegistry, add_zarr_dataset
+from .statistics import TempStatistics
+from .statistics import compute_statistics
+from .utils import bytes
+from .utils import compute_directory_sizes
+from .utils import normalize_and_check_dates
+from .utils import progress_bar
+from .utils import seconds
+from .writer import CubesFilter
+from .writer import ViewCacheArray
+from .zarr import ZarrBuiltRegistry
+from .zarr import add_zarr_dataset
 
 LOG = logging.getLogger(__name__)
 
@@ -246,7 +249,7 @@ def initialise(self, check_name=True):
         metadata["missing_dates"] = [_.isoformat() for _ in dates.missing]
 
         if check_name:
-            basename, ext = os.path.splitext(os.path.basename(self.path))
+            basename, ext = os.path.splitext(os.path.basename(self.path))  # noqa: F841
             ds_name = DatasetName(
                 basename,
                 resolution,
diff --git a/ecml_tools/create/patch.py b/ecml_tools/create/patch.py
index 5228b47..8240d00 100755
--- a/ecml_tools/create/patch.py
+++ b/ecml_tools/create/patch.py
@@ -127,6 +127,4 @@ def apply_patch(path, verbose=True, dry_run=False):
     if before != after:
         print("CHANGED")
 
-    assert json.dumps(z.attrs.asdict(), sort_keys=True) == json.dumps(
-        fixed_attrs, sort_keys=True
-    )
+    assert json.dumps(z.attrs.asdict(), sort_keys=True) == json.dumps(fixed_attrs, sort_keys=True)
diff --git a/ecml_tools/create/statistics.py b/ecml_tools/create/statistics.py
index d0c2edb..8fbc068 100644
--- a/ecml_tools/create/statistics.py
+++ b/ecml_tools/create/statistics.py
@@ -21,7 +21,9 @@
 
 from ecml_tools.provenance import gather_provenance_info
 
-from .check import StatisticsValueError, check_data_values, check_stats
+from .check import StatisticsValueError
+from .check import check_data_values
+from .check import check_stats
 
 LOG = logging.getLogger(__name__)
 
diff --git a/ecml_tools/create/utils.py b/ecml_tools/create/utils.py
index 8cc884d..66c9896 100644
--- a/ecml_tools/create/utils.py
+++ b/ecml_tools/create/utils.py
@@ -69,9 +69,7 @@ def load_json_or_yaml(path):
             return json.load(f)
         if path.endswith(".yaml") or path.endswith(".yml"):
             return yaml.safe_load(f)
-        raise ValueError(
-            f"Cannot read file {path}. Need json or yaml with appropriate extension."
-        )
+        raise ValueError(f"Cannot read file {path}. Need json or yaml with appropriate extension.")
 
 
 def compute_directory_sizes(path):
diff --git a/ecml_tools/create/writer.py b/ecml_tools/create/writer.py
index d5f324b..2182f8a 100644
--- a/ecml_tools/create/writer.py
+++ b/ecml_tools/create/writer.py
@@ -7,16 +7,10 @@
 # nor does it submit to any jurisdiction.
 #
 
-import datetime
 import logging
-import time
 import warnings
 
 import numpy as np
-import zarr
-
-from .check import check_data_values
-from .utils import progress_bar, seconds
 
 LOG = logging.getLogger(__name__)
 
diff --git a/ecml_tools/data.py b/ecml_tools/data.py
index f859a76..7e17e32 100644
--- a/ecml_tools/data.py
+++ b/ecml_tools/data.py
@@ -12,7 +12,8 @@
 import re
 import textwrap
 import warnings
-from functools import cached_property, wraps
+from functools import cached_property
+from functools import wraps
 from pathlib import PurePath
 
 import numpy as np
@@ -21,13 +22,11 @@
 
 import ecml_tools
 
-from .indexing import (
-    apply_index_to_slices_changes,
-    expand_list_indexing,
-    index_to_slices,
-    length_to_slices,
-    update_tuple,
-)
+from .indexing import apply_index_to_slices_changes
+from .indexing import expand_list_indexing
+from .indexing import index_to_slices
+from .indexing import length_to_slices
+from .indexing import update_tuple
 
 LOG = logging.getLogger(__name__)
 
diff --git a/ecml_tools/grids.py b/ecml_tools/grids.py
index bfb3664..f730156 100644
--- a/ecml_tools/grids.py
+++ b/ecml_tools/grids.py
@@ -155,9 +155,7 @@ def cutout_mask(
 
     zero = np.array([0.0, 0.0, 0.0])
     ok = []
-    for i, (global_point, distance, index) in enumerate(
-        zip(global_points, distances, indices)
-    ):
+    for i, (global_point, distance, index) in enumerate(zip(global_points, distances, indices)):
         t = Triangle3D(points[index[0]], points[index[1]], points[index[2]])
         distance = np.min(distance)
         # The point is inside the triangle if the intersection with the ray
diff --git a/ecml_tools/provenance.py b/ecml_tools/provenance.py
index be000a3..374ba5a 100644
--- a/ecml_tools/provenance.py
+++ b/ecml_tools/provenance.py
@@ -14,7 +14,8 @@
 
 
 def lookup_git_repo(path):
-    from git import InvalidGitRepositoryError, Repo
+    from git import InvalidGitRepositoryError
+    from git import Repo
 
     while path != "/":
         try:
@@ -106,10 +107,7 @@ def module_versions(full):
             roots[path] = name
 
     # Sort by length of path, so that we get the most specific first
-    roots = {
-        path: name
-        for path, name in sorted(roots.items(), key=lambda x: len(x[0]), reverse=True)
-    }
+    roots = {path: name for path, name in sorted(roots.items(), key=lambda x: len(x[0]), reverse=True)}
 
     paths = set()
 
@@ -179,7 +177,7 @@ def assets_info(paths):
 
     for path in paths:
         try:
-            (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(path)
+            (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(path)  # noqa: F841
             md5 = path_md5(path)
         except Exception as e:
             result[path] = str(e)
diff --git a/ecml_tools/utils/dates/__init__.py b/ecml_tools/utils/dates/__init__.py
index 3f46204..bd0fb40 100644
--- a/ecml_tools/utils/dates/__init__.py
+++ b/ecml_tools/utils/dates/__init__.py
@@ -103,9 +103,7 @@ def _(x):
         start = _(start)
         end = _(end)
 
-        if isinstance(start, datetime.date) and not isinstance(
-            start, datetime.datetime
-        ):
+        if isinstance(start, datetime.date) and not isinstance(start, datetime.datetime):
             start = datetime.datetime(start.year, start.month, start.day)
 
         if isinstance(end, datetime.date) and not isinstance(end, datetime.datetime):
diff --git a/ecml_tools/utils/humanize.py b/ecml_tools/utils/humanize.py
index e5f3e57..090cbe4 100644
--- a/ecml_tools/utils/humanize.py
+++ b/ecml_tools/utils/humanize.py
@@ -273,7 +273,7 @@ def string_distance(s, t):
 
 
 def did_you_mean(word, vocabulary):
-    distance, best = min((string_distance(word, w), w) for w in vocabulary)
+    _, best = min((string_distance(word, w), w) for w in vocabulary)
     # if distance < min(len(word), len(best)):
     return best
 
diff --git a/ecml_tools/utils/text.py b/ecml_tools/utils/text.py
index f8b1b9e..6c9ec30 100644
--- a/ecml_tools/utils/text.py
+++ b/ecml_tools/utils/text.py
@@ -218,12 +218,7 @@ def _(x):
     result = []
     for i, row in enumerate(all_rows):
         result.append(
-            " │ ".join(
-                [
-                    x.ljust(i) if align[j] == "<" else x.rjust(i)
-                    for j, (x, i) in enumerate(zip(row, lens))
-                ]
-            )
+            " │ ".join([x.ljust(i) if align[j] == "<" else x.rjust(i) for j, (x, i) in enumerate(zip(row, lens))])
         )
         if i == 0:
             result.append("─┼─".join(["─" * i for i in lens]))
diff --git a/scripts/speed-test.py b/scripts/speed-test.py
index 183f8bc..a57ef46 100755
--- a/scripts/speed-test.py
+++ b/scripts/speed-test.py
@@ -111,9 +111,7 @@ def main():
     ds = open_dataset(path)
 
     total = len(ds)
-    print(
-        f"Dataset has {total} items. Opening {args.count} items using {nworkers} workers."
-    )
+    print(f"Dataset has {total} items. Opening {args.count} items using {nworkers} workers.")
     indexes = list(range(len(open_dataset(path))))
 
     if args.shuffle:
diff --git a/scripts/zip-dataset.py b/scripts/zip-dataset.py
index 8958cf7..80e8854 100755
--- a/scripts/zip-dataset.py
+++ b/scripts/zip-dataset.py
@@ -67,14 +67,10 @@ def main():
             relpath = os.path.relpath(path, args.source)
             files.append(relpath)
     end = time.time()
-    print(
-        f"Found {len(files)} files in {args.source} (took {to_human_readable(end-start)})"
-    )
+    print(f"Found {len(files)} files in {args.source} (took {to_human_readable(end-start)})")
 
     def open_zip(path, mode):
-        return zipfile.ZipFile(
-            path, mode=mode, compression=zipfile.ZIP_STORED, allowZip64=True
-        )
+        return zipfile.ZipFile(path, mode=mode, compression=zipfile.ZIP_STORED, allowZip64=True)
 
     def read_metadata(path):
         zf = open_zip(path, mode="r")
diff --git a/tests/test_data.py b/tests/test_data.py
index 06ffeff..6fcae56 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -12,21 +12,19 @@
 import zarr
 
 import ecml_tools.data
-from ecml_tools.data import (
-    Concat,
-    Ensemble,
-    Grids,
-    Join,
-    Rename,
-    Select,
-    Statistics,
-    Subset,
-    Zarr,
-    _as_first_date,
-    _as_last_date,
-    _frequency_to_hours,
-    open_dataset,
-)
+from ecml_tools.data import Concat
+from ecml_tools.data import Ensemble
+from ecml_tools.data import Grids
+from ecml_tools.data import Join
+from ecml_tools.data import Rename
+from ecml_tools.data import Select
+from ecml_tools.data import Statistics
+from ecml_tools.data import Subset
+from ecml_tools.data import Zarr
+from ecml_tools.data import _as_first_date
+from ecml_tools.data import _as_last_date
+from ecml_tools.data import _frequency_to_hours
+from ecml_tools.data import open_dataset
 
 VALUES = 10
 
diff --git a/tests/test_indexing.py b/tests/test_indexing.py
index 5485b12..50ea63f 100644
--- a/tests/test_indexing.py
+++ b/tests/test_indexing.py
@@ -27,9 +27,7 @@ def test_length_to_slices():
                 result = [d[i] for (d, i) in zip(datasets, slices) if i is not None]
                 result = np.concatenate(result)
 
-                if (combined[index].shape != result.shape) or not (
-                    combined[index] == result
-                ).all():
+                if (combined[index].shape != result.shape) or not (combined[index] == result).all():
                     print(index)
                     print(combined[index])
                     print(result)

From 38b05a8485a47ab014f3c7903d91dfd18dd444c6 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Fri, 8 Mar 2024 12:26:12 +0000
Subject: [PATCH 06/25] use precommit in github actions qa

---
 .github/workflows/python-publish.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index 2545bdc..2ebcd11 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -15,14 +15,11 @@ jobs:
     name: Code QA
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - run: pip install black flake8 isort
-    - run: black --version
-    - run: isort --version
-    - run: flake8 --version
-    - run: isort --check .
-    - run: black --check .
-    - run: flake8 .
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.x
+    - uses: pre-commit/action@v3.0.1
 
   checks:
     if: ${{ github.event_name == 'release' }}
@@ -38,7 +35,7 @@ jobs:
     needs: quality
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-python@v2
       with:
@@ -61,7 +58,7 @@ jobs:
     needs: checks
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Set up Python
       uses: actions/setup-python@v2

From 45de1a0c4cf371cf5ce1279faa751cfa88e60ec7 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Fri, 8 Mar 2024 12:52:24 +0000
Subject: [PATCH 07/25] github action

---
 .pre-commit-config.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 94fcb3c..f537085 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,6 +39,8 @@ repos:
     - --fix
     - --exit-non-zero-on-fix
     - --preview 
+    - --exclude
+    - 'dev/*.py'
 #- repo: https://github.com/pre-commit/mirrors-mypy
 #  rev: v0.720
 #  hooks:

From 553c2dc1990dd5609474ec959a27f9e5feb7a2ed Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Fri, 8 Mar 2024 14:12:16 +0000
Subject: [PATCH 08/25] github action

---
 .github/workflows/python-publish.yml | 32 ++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index 2ebcd11..f71dfbb 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -21,6 +21,38 @@ jobs:
         python-version: 3.x
     - uses: pre-commit/action@v3.0.1
 
+  notify-failure:
+    if: failure()
+    runs-on: ubuntu-latest
+    needs: quality
+    name: Notify failure
+    steps:
+      - uses: jdcargile/ms-teams-notification@v1.4
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
+          # notification-summary: ${{ steps.qa.outputs.status }}
+          notification-summary: Build failed on ecml-tools. Please check the logs.
+          notification-color: dc3545
+          timezone: Europe/Paris
+          verbose-logging: true
+
+  notify-success:
+    if: success()
+    runs-on: ubuntu-latest
+    needs: quality
+    name: Notify success
+    steps:
+      - uses: jdcargile/ms-teams-notification@v1.4
+        with:
+          github-token: ${{ github.token }}
+          ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
+          # notification-summary: ${{ steps.qa.outputs.status }}
+          notification-summary: New commit pushed on ecml-tools. Everything is fine.
+          notification-color: 17a2b8
+          timezone: Europe/Paris
+          verbose-logging: true
+
   checks:
     if: ${{ github.event_name == 'release' }}
 

From a09876831e1949831faf22926fcb66c17bab5324 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Fri, 8 Mar 2024 14:24:21 +0000
Subject: [PATCH 09/25] bump versoin 0.6.0

---
 ecml_tools/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ecml_tools/__init__.py b/ecml_tools/__init__.py
index af526ba..d64a42f 100644
--- a/ecml_tools/__init__.py
+++ b/ecml_tools/__init__.py
@@ -5,4 +5,4 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-__version__ = "0.5.0"
+__version__ = "0.6.0"

From 263dc223ed88d5b97f7577f5c4e6b30b5304c4cf Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 09:41:25 +0000
Subject: [PATCH 10/25] some renaming : include(list) -> data_sources(dict),
 move group_by in options:

---
 ecml_tools/create/config.py     |   5 ++
 ecml_tools/create/input.py      | 100 ++++++++++++++++++--------------
 ecml_tools/create/loaders.py    |   2 +-
 tests/create-concat.yaml        |   3 +-
 tests/create-join.yaml          |   4 +-
 tests/create-missing.yaml       |   5 +-
 tests/create-nan.yaml           |   3 +-
 tests/create-perturbations.yaml |  22 ++++---
 tests/create-pipe.yaml          |   3 +-
 9 files changed, 86 insertions(+), 61 deletions(-)

diff --git a/ecml_tools/create/config.py b/ecml_tools/create/config.py
index 7fddb22..fe0f61d 100644
--- a/ecml_tools/create/config.py
+++ b/ecml_tools/create/config.py
@@ -169,6 +169,8 @@ def __init__(self, config, *args, **kwargs):
         if "loop" in self:
             raise ValueError(f"Do not use 'loop'. Use dates instead. {list(self.keys())}")
 
+        self.options = self.get("options", {})
+
         if "licence" not in self:
             self.licence = "unknown"
             print(f"❗ Setting licence={self.licence} because it was not provided.")
@@ -198,6 +200,9 @@ def __init__(self, config, *args, **kwargs):
         assert "flatten_grid" in self.output, self.output
         assert "statistics" in self.output
 
+        if "group_by" in self.options:
+            self.dates["group_by"] = self.options.group_by
+
     def get_serialisable_dict(self):
         return _prepare_serialisation(self)
 
diff --git a/ecml_tools/create/input.py b/ecml_tools/create/input.py
index 638ac57..5190480 100644
--- a/ecml_tools/create/input.py
+++ b/ecml_tools/create/input.py
@@ -554,16 +554,6 @@ def _trace_select(self, dates):
         return f"{self.name}({shorten(dates)})"
 
 
-class ActionWithList(Action):
-    def __init__(self, context, action_path, *configs):
-        super().__init__(context, action_path, *configs)
-        self.actions = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
-
-    def __repr__(self):
-        content = "\n".join([str(i) for i in self.actions])
-        return super().__repr__(content)
-
-
 class PipeAction(Action):
     def __init__(self, context, action_path, *configs):
         super().__init__(context, action_path, *configs)
@@ -694,40 +684,52 @@ def __repr__(self):
         return super().__repr__(content)
 
 
-class IncludeResult(Result):
-    def __init__(self, context, action_path, dates, result, results):
+class DataSourcesResult(Result):
+    def __init__(self, context, action_path, dates, input_result, sources_results):
         super().__init__(context, action_path, dates)
-        # result is the content of the include
-        self.result = result
-        # results is the list of the included results
-        self.results = results
+        # result is the main input result
+        self.input_result = input_result
+        # sources_results is the list of the sources_results
+        self.sources_results = sources_results
 
     @cached_property
     def datasource(self):
-        for i in self.results:
-            # for each include trigger the datasource to be computed
-            # and saved in context but drop it
-            i.datasource
-        # then return the content of the result
+        for i in self.sources_results:
+            # for each result trigger the datasource to be computed
+            # and saved in context
+            self.context.notify_result(i.action_path[:-1], i.datasource)
+        # then return the input result
         # which can use the datasources of the included results
-        return self.result.datasource
+        return self.input_result.datasource
 
 
-class IncludeAction(ActionWithList):
-    def __init__(self, context, action_path, includes, content):
-        super().__init__(context, ["include"], *includes)
-        self.content = action_factory(content, context, ["input"])
+class DataSourcesAction(Action):
+    def __init__(self, context, action_path, sources, input):
+        super().__init__(context, ["data_sources"], *sources)
+        if isinstance(sources, dict):
+            configs = [(str(k), c) for k, c in sources.items()]
+        elif isinstance(sources, list):
+            configs = [(str(i), c) for i, c in enumerate(sources)]
+        else:
+            raise ValueError(f"Invalid data_sources, expecting list or dict, got {type(sources)}: {sources}")
+
+        self.sources = [action_factory(config, context, ["data_sources"] + [a_path]) for a_path, config in configs]
+        self.input = action_factory(input, context, ["input"])
 
     def select(self, dates):
-        results = [a.select(dates) for a in self.actions]
-        return IncludeResult(
+        sources_results = [a.select(dates) for a in self.sources]
+        return DataSourcesResult(
             self.context,
             self.action_path,
             dates,
-            self.content.select(dates),
-            results,
+            self.input.select(dates),
+            sources_results,
         )
 
+    def __repr__(self):
+        content = "\n".join([str(i) for i in self.sources])
+        return super().__repr__(content)
+
 
 class ConcatAction(Action):
     def __init__(self, context, action_path, *configs):
@@ -761,7 +763,15 @@ def select(self, dates):
         return ConcatResult(self.context, self.action_path, dates, results)
 
 
-class JoinAction(ActionWithList):
+class JoinAction(Action):
+    def __init__(self, context, action_path, *configs):
+        super().__init__(context, action_path, *configs)
+        self.actions = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
+
+    def __repr__(self):
+        content = "\n".join([str(i) for i in self.actions])
+        return super().__repr__(content)
+
     @trace_select
     def select(self, dates):
         results = [a.select(dates) for a in self.actions]
@@ -783,15 +793,15 @@ def action_factory(config, context, action_path):
     if isinstance(config[key], dict):
         args, kwargs = [], config[key]
 
-    cls = dict(
-        # date_shift=DateShiftAction,
-        # date_filter=DateFilterAction,
-        include=IncludeAction,
-        concat=ConcatAction,
-        join=JoinAction,
-        pipe=PipeAction,
-        function=FunctionAction,
-    ).get(key)
+    cls = {
+        # "date_shift": DateShiftAction,
+        # "date_filter": DateFilterAction,
+        "data_sources": DataSourcesAction,
+        "concat": ConcatAction,
+        "join": JoinAction,
+        "pipe": PipeAction,
+        "function": FunctionAction,
+    }.get(key)
 
     if cls is None:
         if not is_function(key, "actions"):
@@ -852,15 +862,15 @@ def __init__(self, /, order_by, flatten_grid, remapping):
 
 
 class InputBuilder:
-    def __init__(self, config, include, **kwargs):
+    def __init__(self, config, data_sources, **kwargs):
         self.kwargs = kwargs
 
         config = deepcopy(config)
-        if include:
+        if data_sources:
             config = dict(
-                include=dict(
-                    includes=include,
-                    content=config,
+                data_sources=dict(
+                    sources=data_sources,
+                    input=config,
                 )
             )
         self.config = config
diff --git a/ecml_tools/create/loaders.py b/ecml_tools/create/loaders.py
index 0fa7ed3..ad45b2b 100644
--- a/ecml_tools/create/loaders.py
+++ b/ecml_tools/create/loaders.py
@@ -78,7 +78,7 @@ def build_input(self):
 
         builder = build_input(
             self.main_config.input,
-            include=self.main_config.get("include", {}),
+            data_sources=self.main_config.get("data_sources", {}),
             order_by=self.output.order_by,
             flatten_grid=self.output.flatten_grid,
             remapping=build_remapping(self.output.remapping),
diff --git a/tests/create-concat.yaml b/tests/create-concat.yaml
index fd920a1..43f2780 100644
--- a/tests/create-concat.yaml
+++ b/tests/create-concat.yaml
@@ -8,6 +8,8 @@ dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
+
+options:
   group_by: monthly
 
 common:
@@ -38,7 +40,6 @@ input:
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by:
     - valid_datetime
     - param_level
diff --git a/tests/create-join.yaml b/tests/create-join.yaml
index f09bdc3..be09149 100644
--- a/tests/create-join.yaml
+++ b/tests/create-join.yaml
@@ -14,6 +14,9 @@ dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
+
+
+options:
   group_by: monthly
 
 input:
@@ -47,7 +50,6 @@ input:
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
diff --git a/tests/create-missing.yaml b/tests/create-missing.yaml
index b45b815..c4fcd15 100644
--- a/tests/create-missing.yaml
+++ b/tests/create-missing.yaml
@@ -14,9 +14,11 @@ dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
-  group_by: monthly
   missing: ['2020-12-30 12:00:00', '2021-01-03 00:00:00']
 
+options:
+  group_by: monthly
+
 include:
   - mars:
       <<: *mars_request
@@ -35,7 +37,6 @@ input:
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
diff --git a/tests/create-nan.yaml b/tests/create-nan.yaml
index 7b708b7..ef6d87a 100644
--- a/tests/create-nan.yaml
+++ b/tests/create-nan.yaml
@@ -8,6 +8,8 @@ dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
+
+options:
   group_by: monthly
 
 input:
@@ -26,7 +28,6 @@ has_nans: [sst]
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
diff --git a/tests/create-perturbations.yaml b/tests/create-perturbations.yaml
index c4873f4..29c177a 100644
--- a/tests/create-perturbations.yaml
+++ b/tests/create-perturbations.yaml
@@ -34,44 +34,48 @@ dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
+
+options:
   group_by: monthly
 
-include: # This "include" will be renamed
-  - join:
+data_sources:
+  ensembles:
+    join:
       - mars:
           <<: *ensembles
           <<: *common
       - accumulations:
           <<: *ensembles
           <<: *common_acc
-  - join:
+  mean:
+    join:
       - mars:
           <<: *mean
           <<: *common
       - accumulations:
           <<: *mean
           <<: *common_acc
-  - join:
+  center:
+    join:
       - mars:
           <<: *center
           <<: *common
       - accumulations:
           <<: *center
           <<: *common_acc
-
+          
 input:
   ensemble_perturbations:
     # the ensemble data which has one additional dimension
-    ensembles: ${include.0.join}
+    ensembles: ${data_sources.ensembles}
     # the previous center of the data
-    mean: ${include.1.join}
+    mean: ${data_sources.mean}
     # the new center of the data
-    center: ${include.2.join}
+    center: ${data_sources.center}
 
 output:
   chunking: { dates: 1 }
   dtype: float32
-  flatten_grid: True
   order_by:
     - valid_datetime
     - param_level
diff --git a/tests/create-pipe.yaml b/tests/create-pipe.yaml
index c8a80ac..594c4c2 100644
--- a/tests/create-pipe.yaml
+++ b/tests/create-pipe.yaml
@@ -14,6 +14,8 @@ dates: &dates_anchor
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
+
+options:
   group_by: monthly
 
 input:
@@ -48,7 +50,6 @@ input:
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by:
     - valid_datetime
     - param_level

From 90daa16b074a85e31607df540e0f6ba214c302f0 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 13:38:48 +0000
Subject: [PATCH 11/25] all yaml files defined. reference data on s3

---
 ecml_tools/create/input.py                    |   2 +-
 .../create-1.zarr.sync/.zattrs                |   0
 .../create-1.zarr.sync/.zgroup                |   0
 .../create-1.zarr.sync/_build/flags/.zarray   |   0
 .../create-1.zarr.sync/_build/lengths/.zarray |   0
 tests/create-1.yaml                           |  79 ---------
 .../concat.yaml}                              |   5 +-
 .../data_sources.yaml}                        |  21 ++-
 tests/{create-join.yaml => create/join.yaml}  |   2 +-
 .../missing.yaml}                             |   4 +-
 tests/{create-nan.yaml => create/nan.yaml}    |   1 -
 tests/{create-pipe.yaml => create/pipe.yaml}  |   1 -
 tests/create/test_create.py                   | 151 ++++++++++++++++++
 13 files changed, 167 insertions(+), 99 deletions(-)
 delete mode 100644 tests/create-1-reference/create-1.zarr.sync/.zattrs
 delete mode 100644 tests/create-1-reference/create-1.zarr.sync/.zgroup
 delete mode 100644 tests/create-1-reference/create-1.zarr.sync/_build/flags/.zarray
 delete mode 100644 tests/create-1-reference/create-1.zarr.sync/_build/lengths/.zarray
 delete mode 100644 tests/create-1.yaml
 rename tests/{create-concat.yaml => create/concat.yaml} (94%)
 rename tests/{create-include.yaml => create/data_sources.yaml} (74%)
 rename tests/{create-join.yaml => create/join.yaml} (98%)
 rename tests/{create-missing.yaml => create/missing.yaml} (94%)
 rename tests/{create-nan.yaml => create/nan.yaml} (97%)
 rename tests/{create-pipe.yaml => create/pipe.yaml} (98%)
 create mode 100755 tests/create/test_create.py

diff --git a/ecml_tools/create/input.py b/ecml_tools/create/input.py
index 5190480..b6f1800 100644
--- a/ecml_tools/create/input.py
+++ b/ecml_tools/create/input.py
@@ -633,7 +633,7 @@ class FilterStepResult(StepResult):
     @assert_fieldset
     @trace_datasource
     def datasource(self):
-        ds = self.content.datasource
+        ds = self.upstream_result.datasource
         ds = ds.sel(**self.action.kwargs)
         return ds
 
diff --git a/tests/create-1-reference/create-1.zarr.sync/.zattrs b/tests/create-1-reference/create-1.zarr.sync/.zattrs
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/create-1-reference/create-1.zarr.sync/.zgroup b/tests/create-1-reference/create-1.zarr.sync/.zgroup
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/create-1-reference/create-1.zarr.sync/_build/flags/.zarray b/tests/create-1-reference/create-1.zarr.sync/_build/flags/.zarray
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/create-1-reference/create-1.zarr.sync/_build/lengths/.zarray b/tests/create-1-reference/create-1.zarr.sync/_build/lengths/.zarray
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/create-1.yaml b/tests/create-1.yaml
deleted file mode 100644
index 68688eb..0000000
--- a/tests/create-1.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-purpose: aifs
-
-name: test-small
-
-remapping: &remapping
-  #param_level: '{param}_{levtype}{levelist}'
-  param_level: '{param}_{levelist}'
-
-dates:
-  start: 2020-12-30 00:00:00
-  end: 2021-01-03 12:00:00
-  frequency: 12h
-  group_by: monthly
-
-input:
-  join:
-    remapping: *remapping
-    dates:
-      start: 2020-12-30 00:00:00
-      end: 2021-01-03 12:00:00
-      frequency: 12h
-    content:
-    - source:
-        name: sfc_data
-        kwargs:
-          name: mars
-          class: ea
-          date: $datetime_format($dates,%Y%m%d)
-          time: $datetime_format($dates,%H%M)
-          #date: $datetime_format($dates,'2023%m%d')
-          #hdate: $dates
-          expver: '0001'
-          grid: 20./20.
-          levtype: sfc
-          param: [2t]
-          stream: oper
-          type: an
-
-    - source:
-        name: acc_data
-        inherit: sfc_data
-        kwargs:
-          name: era5-accumulations
-          param: [cp, tp]
-          # era5-accumulations requires time in hours
-          time: $datetime_format($dates,%H)
-
-    - source:
-        name: pl_data
-        inherit: sfc_data
-        kwargs:
-          name: mars
-          levtype: pl
-          param: [q, t]
-          level: [50, 100]
-
-    - source:
-        name: forcings
-        kwargs:
-          name: constants
-          param:
-          - cos_latitude
-          source_or_dataset: $previous
-
-
-
-output:
-  chunking:
-    dates: 1
-  dtype: float32
-  flatten_grid: True
-  order_by:
-  - valid_datetime
-  - param_level
-  - number
-  statistics: param_level
-  statistics_end: 2021
diff --git a/tests/create-concat.yaml b/tests/create/concat.yaml
similarity index 94%
rename from tests/create-concat.yaml
rename to tests/create/concat.yaml
index 43f2780..416e9c6 100644
--- a/tests/create-concat.yaml
+++ b/tests/create/concat.yaml
@@ -1,7 +1,6 @@
 description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
 dataset_status: testing
-purpose: aifs
-name: test-small
+name: test-concat
 config_format_version: 2
 
 dates:
@@ -46,5 +45,5 @@ output:
     - number
   statistics: param_level
   statistics_end: 2021
-  remapping: &remapping
+  remapping: 
     param_level: "{param}_{levelist}"
diff --git a/tests/create-include.yaml b/tests/create/data_sources.yaml
similarity index 74%
rename from tests/create-include.yaml
rename to tests/create/data_sources.yaml
index ad3e53f..ad83c08 100644
--- a/tests/create-include.yaml
+++ b/tests/create/data_sources.yaml
@@ -1,7 +1,6 @@
 description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
 dataset_status: testing
-purpose: aifs
-name: test-small
+name: test-data-sources
 config_format_version: 2
 
 common:
@@ -16,24 +15,24 @@ dates:
   frequency: 12h
   group_by: monthly
 
-include:
-  - mars:
-      <<: *mars_request
-      param: [2t]
-      levtype: sfc
-      stream: oper
-      type: an
+data_sources:
+  my_predefined_source:
+    mars:
+        <<: *mars_request
+        param: [2t]
+        levtype: sfc
+        stream: oper
+        type: an
 
 input:
   constants:
-    template: ${include.0.mars}
+    template: ${data_sources.my_predefined_source}
     param:
       - cos_latitude
 
 output:
   chunking: { dates: 1, ensembles: 1 }
   dtype: float32
-  flatten_grid: True
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
diff --git a/tests/create-join.yaml b/tests/create/join.yaml
similarity index 98%
rename from tests/create-join.yaml
rename to tests/create/join.yaml
index be09149..782e41f 100644
--- a/tests/create-join.yaml
+++ b/tests/create/join.yaml
@@ -1,7 +1,7 @@
 description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
 dataset_status: testing
 purpose: aifs
-name: test-small
+name: test-join
 config_format_version: 2
 
 common:
diff --git a/tests/create-missing.yaml b/tests/create/missing.yaml
similarity index 94%
rename from tests/create-missing.yaml
rename to tests/create/missing.yaml
index c4fcd15..f9a8993 100644
--- a/tests/create-missing.yaml
+++ b/tests/create/missing.yaml
@@ -19,7 +19,7 @@ dates:
 options:
   group_by: monthly
 
-include:
+data_sources:
   - mars:
       <<: *mars_request
       param: [2t]
@@ -29,7 +29,7 @@ include:
 
 input:
   constants:
-    template: ${include.0.mars}
+    template: ${data_sources.0.mars}
     param:
       - cos_latitude
       #- sin_latitude
diff --git a/tests/create-nan.yaml b/tests/create/nan.yaml
similarity index 97%
rename from tests/create-nan.yaml
rename to tests/create/nan.yaml
index ef6d87a..002b90f 100644
--- a/tests/create-nan.yaml
+++ b/tests/create/nan.yaml
@@ -1,6 +1,5 @@
 description: "testing for nans using sst"
 dataset_status: testing
-purpose: aifs
 name: test-nan
 config_format_version: 2
 
diff --git a/tests/create-pipe.yaml b/tests/create/pipe.yaml
similarity index 98%
rename from tests/create-pipe.yaml
rename to tests/create/pipe.yaml
index 594c4c2..484aebf 100644
--- a/tests/create-pipe.yaml
+++ b/tests/create/pipe.yaml
@@ -1,6 +1,5 @@
 description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
 dataset_status: testing
-purpose: aifs
 name: test-small
 config_format_version: 2
 
diff --git a/tests/create/test_create.py b/tests/create/test_create.py
new file mode 100755
index 0000000..90fd3d8
--- /dev/null
+++ b/tests/create/test_create.py
@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+import json
+import os
+
+import numpy as np
+
+from ecml_tools.create import Creator
+from ecml_tools.data import open_dataset
+
+TEST_DATA_ROOT = "/s3/ml-tests/test-data/anemoi-datasets/create/"
+
+
+class Reference:
+    def __init__(self, name):
+        self.name = name
+        self.path = os.path.join(TEST_DATA_ROOT, name)
+
+    def compare_dot_zattrs(other):
+        a = json.load(open(self.path)).attrs
+        if isinstance(a, dict):
+            a_keys = list(a.keys())
+            b_keys = list(b.keys())
+            for k in set(a_keys) & set(b_keys):
+                if k in ["timestamp", "uuid", "latest_write_timestamp", "yaml_config"]:
+                    assert type(a[k]) == type(b[k]), (  # noqa: E721
+                        type(a[k]),
+                        type(b[k]),
+                        a[k],
+                        b[k],
+                    )
+                assert k in a_keys, (k, a_keys)
+                assert k in b_keys, (k, b_keys)
+                return compare_dot_zattrs(a[k], b[k])
+
+        if isinstance(a, list):
+            assert len(a) == len(b), (a, b)
+            for v, w in zip(a, b):
+                return compare_dot_zattrs(v, w)
+
+        assert type(a) == type(b), (type(a), type(b), a, b)  # noqa: E721
+        return a == b, (a, b)
+
+
+def compare_zarr(dir1, dir2):
+    a = open_dataset(dir1)
+    b = open_dataset(dir2)
+    assert a.shape == b.shape, (a.shape, b.shape)
+    assert (a.dates == b.dates).all(), (a.dates, b.dates)
+    for a_, b_ in zip(a.variables, b.variables):
+        assert a_ == b_, (a, b)
+    for i_date, date in zip(range(a.shape[0]), a.dates):
+        for i_param in range(a.shape[1]):
+            param = a.variables[i_param]
+            assert param == b.variables[i_param], (
+                date,
+                param,
+                a.variables[i_param],
+                b.variables[i_param],
+            )
+            a_ = a[i_date, i_param]
+            b_ = b[i_date, i_param]
+            assert a.shape == b.shape, (date, param, a.shape, b.shape)
+            delta = a_ - b_
+            max_delta = np.max(np.abs(delta))
+            assert max_delta == 0.0, (date, param, a_, b_, a_ - b_, max_delta)
+    compare(dir1, dir2)
+
+
+def compare(dir1, dir2):
+    """Compare two directories recursively."""
+    files1 = set(os.listdir(dir1))
+    files2 = set(os.listdir(dir2))
+    files = files1.union(files2)
+
+    for f in files:
+        path1 = os.path.join(dir1, f)
+        path2 = os.path.join(dir2, f)
+
+        if not os.path.isfile(path1):
+            assert os.path.isdir(path2), f"Directory {path2} does not exist"
+            compare(path1, path2)
+            continue
+
+        assert os.path.isfile(path2), f"File {path2} does not exist"
+
+        content1 = open(path1, "rb").read()
+        content2 = open(path2, "rb").read()
+
+        if f == ".zattrs":
+            compare_dot_zattrs(json.loads(content1), json.loads(content2))
+            continue
+
+        if f == "provenance_load.json":
+            assert False, "test not implemented to compare temporary statistics"
+
+        assert content1 == content2, f"{path1} != {path2}"
+
+
+def _test_create(name):
+    here = os.path.dirname(__file__)
+    config = os.path.join(here, name + ".yaml")
+    output = os.path.join(here, name + "-output", name + ".zarr")
+    reference = os.path.join(here, name + "-reference", name + ".zarr")
+
+    # cache=None is using the default cache
+    c = Creator(
+        output,
+        config=config,
+        cache=None,
+        overwrite=True,
+    )
+    c.create()
+
+    compare_zarr(reference, output)
+
+
+def test_create_concat():
+    _test_create("create-concat")
+
+
+def test_create_join():
+    _test_create("create-join")
+
+
+def test_create_pipe():
+    _test_create("create-pipe")
+
+
+def test_create_perturbations():
+    _test_create("create-perturbations")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--name", help="Name of the test case")
+    parser.add_argument("--compare", nargs=2, help="Compare two directories")
+
+    args = parser.parse_args()
+
+    if args.compare:
+        compare_zarr(args.compare[0], args.compare[1])
+    else:
+        _test_create(args.name)

From 2731f0844f383766d9cb781f35bd6e20d25c3450 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 14:40:24 +0000
Subject: [PATCH 12/25] github action dependencies

---
 .github/workflows/python-publish.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index f71dfbb..654d76b 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -64,7 +64,6 @@ jobs:
 
     name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
     runs-on: ${{ matrix.platform }}
-    needs: quality
 
     steps:
     - uses: actions/checkout@v4
@@ -87,7 +86,7 @@ jobs:
 
     if: ${{ github.event_name == 'release' }}
     runs-on: ubuntu-latest
-    needs: checks
+    needs: [checks, quality]
 
     steps:
     - uses: actions/checkout@v4

From b7756aee0135cf7fdb0c3403547115e59e508eeb Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 15:00:00 +0000
Subject: [PATCH 13/25] github action dependencies

---
 .github/workflows/python-publish.yml | 60 ++++++++++++++--------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index 654d76b..d3adc83 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -21,10 +21,37 @@ jobs:
         python-version: 3.x
     - uses: pre-commit/action@v3.0.1
 
+  checks:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ["ubuntu-latest", "macos-latest"]
+        python-version: ["3.10"]
+
+    name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install
+      run: |
+        pip install pytest
+        pip install -e .[all]
+        pip install -r tests/requirements.txt
+        pip freeze
+
+    - name: Tests
+      run: pytest
+
   notify-failure:
     if: failure()
     runs-on: ubuntu-latest
-    needs: quality
+    needs: [quality, checks]
     name: Notify failure
     steps:
       - uses: jdcargile/ms-teams-notification@v1.4
@@ -40,7 +67,7 @@ jobs:
   notify-success:
     if: success()
     runs-on: ubuntu-latest
-    needs: quality
+    needs: [quality, checks]
     name: Notify success
     steps:
       - uses: jdcargile/ms-teams-notification@v1.4
@@ -53,35 +80,6 @@ jobs:
           timezone: Europe/Paris
           verbose-logging: true
 
-  checks:
-    if: ${{ github.event_name == 'release' }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        platform: ["ubuntu-latest", "macos-latest"]
-        python-version: ["3.10"]
-
-    name: Python  ${{ matrix.python-version }} on ${{ matrix.platform }}
-    runs-on: ${{ matrix.platform }}
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install
-      run: |
-        pip install pytest
-        pip install -e .[all]
-        pip install -r tests/requirements.txt
-        pip freeze
-
-    - name: Tests
-      run: pytest
-
   deploy:
 
     if: ${{ github.event_name == 'release' }}

From e830768ed6157b5567bcdbcb1b16ed7e8c49767d Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 16:39:26 +0000
Subject: [PATCH 14/25] zarr create tests pass

---
 tests/_test_create.py                         | 143 -------
 .../create-concat.zarr/.zattrs                | 287 --------------
 .../create-concat.zarr/.zgroup                |   3 -
 .../create-concat.zarr/_build/.zgroup         |   3 -
 .../create-concat.zarr/_build/count/.zarray   |  22 --
 .../create-concat.zarr/_build/count/0.0       | Bin 96 -> 0 bytes
 .../create-concat.zarr/_build/flags/.zarray   |  20 -
 .../create-concat.zarr/_build/flags/0         | Bin 18 -> 0 bytes
 .../create-concat.zarr/_build/lengths/.zarray |  20 -
 .../create-concat.zarr/_build/lengths/0       | Bin 24 -> 0 bytes
 .../create-concat.zarr/_build/maximum/.zarray |  22 --
 .../create-concat.zarr/_build/maximum/0.0     | Bin 96 -> 0 bytes
 .../create-concat.zarr/_build/minimum/.zarray |  22 --
 .../create-concat.zarr/_build/minimum/0.0     | Bin 96 -> 0 bytes
 .../create-concat.zarr/_build/squares/.zarray |  22 --
 .../create-concat.zarr/_build/squares/0.0     | Bin 96 -> 0 bytes
 .../create-concat.zarr/_build/sums/.zarray    |  22 --
 .../create-concat.zarr/_build/sums/0.0        | Bin 96 -> 0 bytes
 .../create-concat.zarr/count/.zarray          |  20 -
 .../create-concat.zarr/count/0                | Bin 24 -> 0 bytes
 .../create-concat.zarr/data/.zarray           |  26 --
 .../create-concat.zarr/data/0.0.0.0           | Bin 533 -> 0 bytes
 .../create-concat.zarr/data/1.0.0.0           | Bin 533 -> 0 bytes
 .../create-concat.zarr/data/2.0.0.0           | Bin 531 -> 0 bytes
 .../create-concat.zarr/data/3.0.0.0           | Bin 530 -> 0 bytes
 .../create-concat.zarr/data/4.0.0.0           | Bin 532 -> 0 bytes
 .../create-concat.zarr/data/5.0.0.0           | Bin 533 -> 0 bytes
 .../create-concat.zarr/data/6.0.0.0           | Bin 533 -> 0 bytes
 .../create-concat.zarr/data/7.0.0.0           | Bin 530 -> 0 bytes
 .../create-concat.zarr/data/8.0.0.0           | Bin 533 -> 0 bytes
 .../create-concat.zarr/data/9.0.0.0           | Bin 529 -> 0 bytes
 .../create-concat.zarr/dates/.zarray          |  20 -
 .../create-concat.zarr/dates/0                | Bin 96 -> 0 bytes
 .../create-concat.zarr/latitudes/.zarray      |  20 -
 .../create-concat.zarr/latitudes/0            | Bin 179 -> 0 bytes
 .../create-concat.zarr/longitudes/.zarray     |  20 -
 .../create-concat.zarr/longitudes/0           | Bin 199 -> 0 bytes
 .../create-concat.zarr/maximum/.zarray        |  20 -
 .../create-concat.zarr/maximum/0              | Bin 24 -> 0 bytes
 .../create-concat.zarr/mean/.zarray           |  20 -
 .../create-concat.zarr/mean/0                 | Bin 24 -> 0 bytes
 .../create-concat.zarr/minimum/.zarray        |  20 -
 .../create-concat.zarr/minimum/0              | Bin 24 -> 0 bytes
 .../create-concat.zarr/squares/.zarray        |  20 -
 .../create-concat.zarr/squares/0              | Bin 24 -> 0 bytes
 .../create-concat.zarr/stdev/.zarray          |  20 -
 .../create-concat.zarr/stdev/0                | Bin 24 -> 0 bytes
 .../create-concat.zarr/sums/.zarray           |  20 -
 .../create-concat.zarr/sums/0                 | Bin 24 -> 0 bytes
 .../create-join.zarr/.zattrs                  | 356 ------------------
 .../create-join.zarr/.zgroup                  |   3 -
 .../create-join.zarr/_build/.zgroup           |   3 -
 .../create-join.zarr/_build/flags/.zarray     |  20 -
 .../create-join.zarr/_build/flags/0           | Bin 18 -> 0 bytes
 .../create-join.zarr/_build/lengths/.zarray   |  20 -
 .../create-join.zarr/_build/lengths/0         | Bin 24 -> 0 bytes
 .../create-join.zarr/count/.zarray            |  20 -
 .../create-join.zarr/count/0                  | Bin 80 -> 0 bytes
 .../create-join.zarr/data/.zarray             |  26 --
 .../create-join.zarr/data/0.0.0.0             | Bin 3320 -> 0 bytes
 .../create-join.zarr/data/1.0.0.0             | Bin 3340 -> 0 bytes
 .../create-join.zarr/data/2.0.0.0             | Bin 3336 -> 0 bytes
 .../create-join.zarr/data/3.0.0.0             | Bin 3297 -> 0 bytes
 .../create-join.zarr/data/4.0.0.0             | Bin 3385 -> 0 bytes
 .../create-join.zarr/data/5.0.0.0             | Bin 3366 -> 0 bytes
 .../create-join.zarr/data/6.0.0.0             | Bin 3399 -> 0 bytes
 .../create-join.zarr/data/7.0.0.0             | Bin 3386 -> 0 bytes
 .../create-join.zarr/data/8.0.0.0             | Bin 3367 -> 0 bytes
 .../create-join.zarr/data/9.0.0.0             | Bin 3377 -> 0 bytes
 .../create-join.zarr/dates/.zarray            |  20 -
 .../create-join.zarr/dates/0                  | Bin 96 -> 0 bytes
 .../create-join.zarr/latitudes/.zarray        |  20 -
 .../create-join.zarr/latitudes/0              | Bin 179 -> 0 bytes
 .../create-join.zarr/longitudes/.zarray       |  20 -
 .../create-join.zarr/longitudes/0             | Bin 199 -> 0 bytes
 .../create-join.zarr/maximum/.zarray          |  20 -
 .../create-join.zarr/maximum/0                | Bin 80 -> 0 bytes
 .../create-join.zarr/mean/.zarray             |  20 -
 .../create-join.zarr/mean/0                   | Bin 80 -> 0 bytes
 .../create-join.zarr/minimum/.zarray          |  20 -
 .../create-join.zarr/minimum/0                | Bin 80 -> 0 bytes
 .../create-join.zarr/squares/.zarray          |  20 -
 .../create-join.zarr/squares/0                | Bin 80 -> 0 bytes
 .../create-join.zarr/stdev/.zarray            |  20 -
 .../create-join.zarr/stdev/0                  | Bin 80 -> 0 bytes
 .../create-join.zarr/sums/.zarray             |  20 -
 .../create-join.zarr/sums/0                   | Bin 80 -> 0 bytes
 .../create-pipe.zarr/.zattrs                  | 354 -----------------
 .../create-pipe.zarr/.zgroup                  |   3 -
 .../create-pipe.zarr/_build/.zgroup           |   3 -
 .../create-pipe.zarr/_build/count/.zarray     |  22 --
 .../create-pipe.zarr/_build/count/0.0         | Bin 41 -> 0 bytes
 .../create-pipe.zarr/_build/flags/.zarray     |  20 -
 .../create-pipe.zarr/_build/flags/0           | Bin 18 -> 0 bytes
 .../create-pipe.zarr/_build/lengths/.zarray   |  20 -
 .../create-pipe.zarr/_build/lengths/0         | Bin 24 -> 0 bytes
 .../create-pipe.zarr/_build/maximum/.zarray   |  22 --
 .../create-pipe.zarr/_build/maximum/0.0       | Bin 229 -> 0 bytes
 .../create-pipe.zarr/_build/minimum/.zarray   |  22 --
 .../create-pipe.zarr/_build/minimum/0.0       | Bin 217 -> 0 bytes
 .../create-pipe.zarr/_build/squares/.zarray   |  22 --
 .../create-pipe.zarr/_build/squares/0.0       | Bin 246 -> 0 bytes
 .../create-pipe.zarr/_build/sums/.zarray      |  22 --
 .../create-pipe.zarr/_build/sums/0.0          | Bin 245 -> 0 bytes
 .../create-pipe.zarr/count/.zarray            |  20 -
 .../create-pipe.zarr/count/0                  | Bin 56 -> 0 bytes
 .../create-pipe.zarr/data/.zarray             |  26 --
 .../create-pipe.zarr/data/0.0.0.0             | Bin 1937 -> 0 bytes
 .../create-pipe.zarr/data/1.0.0.0             | Bin 1973 -> 0 bytes
 .../create-pipe.zarr/data/2.0.0.0             | Bin 1954 -> 0 bytes
 .../create-pipe.zarr/data/3.0.0.0             | Bin 1915 -> 0 bytes
 .../create-pipe.zarr/data/4.0.0.0             | Bin 2000 -> 0 bytes
 .../create-pipe.zarr/data/5.0.0.0             | Bin 1970 -> 0 bytes
 .../create-pipe.zarr/data/6.0.0.0             | Bin 1992 -> 0 bytes
 .../create-pipe.zarr/data/7.0.0.0             | Bin 1991 -> 0 bytes
 .../create-pipe.zarr/data/8.0.0.0             | Bin 1981 -> 0 bytes
 .../create-pipe.zarr/data/9.0.0.0             | Bin 1981 -> 0 bytes
 .../create-pipe.zarr/dates/.zarray            |  20 -
 .../create-pipe.zarr/dates/0                  | Bin 96 -> 0 bytes
 .../create-pipe.zarr/latitudes/.zarray        |  20 -
 .../create-pipe.zarr/latitudes/0              | Bin 179 -> 0 bytes
 .../create-pipe.zarr/longitudes/.zarray       |  20 -
 .../create-pipe.zarr/longitudes/0             | Bin 199 -> 0 bytes
 .../create-pipe.zarr/maximum/.zarray          |  20 -
 .../create-pipe.zarr/maximum/0                | Bin 56 -> 0 bytes
 .../create-pipe.zarr/mean/.zarray             |  20 -
 .../create-pipe.zarr/mean/0                   | Bin 56 -> 0 bytes
 .../create-pipe.zarr/minimum/.zarray          |  20 -
 .../create-pipe.zarr/minimum/0                | Bin 56 -> 0 bytes
 .../create-pipe.zarr/squares/.zarray          |  20 -
 .../create-pipe.zarr/squares/0                | Bin 56 -> 0 bytes
 .../create-pipe.zarr/stdev/.zarray            |  20 -
 .../create-pipe.zarr/stdev/0                  | Bin 56 -> 0 bytes
 .../create-pipe.zarr/sums/.zarray             |  20 -
 .../create-pipe.zarr/sums/0                   | Bin 56 -> 0 bytes
 tests/create/test_create.py                   | 169 ++++-----
 136 files changed, 77 insertions(+), 2268 deletions(-)
 delete mode 100755 tests/_test_create.py
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/.zattrs
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/.zgroup
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/.zgroup
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/count/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/count/0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/flags/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/flags/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/lengths/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/lengths/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/maximum/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/maximum/0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/minimum/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/minimum/0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/squares/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/squares/0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/sums/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/_build/sums/0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/count/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/count/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/0.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/1.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/2.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/3.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/4.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/5.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/6.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/7.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/8.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/data/9.0.0.0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/dates/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/dates/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/latitudes/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/latitudes/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/longitudes/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/longitudes/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/maximum/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/maximum/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/mean/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/mean/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/minimum/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/minimum/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/squares/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/squares/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/stdev/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/stdev/0
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/sums/.zarray
 delete mode 100644 tests/create-concat-reference/create-concat.zarr/sums/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/.zattrs
 delete mode 100644 tests/create-join-reference/create-join.zarr/.zgroup
 delete mode 100644 tests/create-join-reference/create-join.zarr/_build/.zgroup
 delete mode 100644 tests/create-join-reference/create-join.zarr/_build/flags/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/_build/flags/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/_build/lengths/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/_build/lengths/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/count/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/count/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/0.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/1.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/2.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/3.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/4.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/5.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/6.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/7.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/8.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/data/9.0.0.0
 delete mode 100644 tests/create-join-reference/create-join.zarr/dates/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/dates/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/latitudes/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/latitudes/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/longitudes/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/longitudes/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/maximum/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/maximum/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/mean/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/mean/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/minimum/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/minimum/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/squares/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/squares/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/stdev/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/stdev/0
 delete mode 100644 tests/create-join-reference/create-join.zarr/sums/.zarray
 delete mode 100644 tests/create-join-reference/create-join.zarr/sums/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/.zattrs
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/.zgroup
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/.zgroup
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/count/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/count/0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/flags/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/flags/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/lengths/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/lengths/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/maximum/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/maximum/0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/minimum/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/minimum/0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/squares/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/squares/0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/sums/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/_build/sums/0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/count/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/count/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/0.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/1.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/2.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/3.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/4.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/5.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/6.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/7.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/8.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/data/9.0.0.0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/dates/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/dates/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/latitudes/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/latitudes/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/longitudes/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/longitudes/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/maximum/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/maximum/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/mean/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/mean/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/minimum/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/minimum/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/squares/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/squares/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/stdev/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/stdev/0
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/sums/.zarray
 delete mode 100644 tests/create-pipe-reference/create-pipe.zarr/sums/0

diff --git a/tests/_test_create.py b/tests/_test_create.py
deleted file mode 100755
index 0b98be0..0000000
--- a/tests/_test_create.py
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-import json
-import os
-
-import numpy as np
-
-from ecml_tools.create import Creator
-from ecml_tools.data import open_dataset
-
-
-def compare_dot_zattrs(a, b):
-    if isinstance(a, dict):
-        a_keys = list(a.keys())
-        b_keys = list(b.keys())
-        for k in set(a_keys) & set(b_keys):
-            if k in ["timestamp", "uuid", "latest_write_timestamp", "yaml_config"]:
-                assert type(a[k]) == type(b[k]), (  # noqa: E721
-                    type(a[k]),
-                    type(b[k]),
-                    a[k],
-                    b[k],
-                )
-            assert k in a_keys, (k, a_keys)
-            assert k in b_keys, (k, b_keys)
-            return compare_dot_zattrs(a[k], b[k])
-
-    if isinstance(a, list):
-        assert len(a) == len(b), (a, b)
-        for v, w in zip(a, b):
-            return compare_dot_zattrs(v, w)
-
-    assert type(a) == type(b), (type(a), type(b), a, b)  # noqa: E721
-    return a == b, (a, b)
-
-
-def compare_zarr(dir1, dir2):
-    a = open_dataset(dir1)
-    b = open_dataset(dir2)
-    assert a.shape == b.shape, (a.shape, b.shape)
-    assert (a.dates == b.dates).all(), (a.dates, b.dates)
-    for a_, b_ in zip(a.variables, b.variables):
-        assert a_ == b_, (a, b)
-    for i_date, date in zip(range(a.shape[0]), a.dates):
-        for i_param in range(a.shape[1]):
-            param = a.variables[i_param]
-            assert param == b.variables[i_param], (
-                date,
-                param,
-                a.variables[i_param],
-                b.variables[i_param],
-            )
-            a_ = a[i_date, i_param]
-            b_ = b[i_date, i_param]
-            assert a.shape == b.shape, (date, param, a.shape, b.shape)
-            delta = a_ - b_
-            max_delta = np.max(np.abs(delta))
-            assert max_delta == 0.0, (date, param, a_, b_, a_ - b_, max_delta)
-    compare(dir1, dir2)
-
-
-def compare(dir1, dir2):
-    """Compare two directories recursively."""
-    files1 = set(os.listdir(dir1))
-    files2 = set(os.listdir(dir2))
-    files = files1.union(files2)
-
-    for f in files:
-        path1 = os.path.join(dir1, f)
-        path2 = os.path.join(dir2, f)
-
-        if not os.path.isfile(path1):
-            assert os.path.isdir(path2), f"Directory {path2} does not exist"
-            compare(path1, path2)
-            continue
-
-        assert os.path.isfile(path2), f"File {path2} does not exist"
-
-        content1 = open(path1, "rb").read()
-        content2 = open(path2, "rb").read()
-
-        if f == ".zattrs":
-            compare_dot_zattrs(json.loads(content1), json.loads(content2))
-            continue
-
-        if f == "provenance_load.json":
-            assert False, "test not implemented to compare temporary statistics"
-
-        assert content1 == content2, f"{path1} != {path2}"
-
-
-def _test_create(name):
-    here = os.path.dirname(__file__)
-    config = os.path.join(here, name + ".yaml")
-    output = os.path.join(here, name + "-output", name + ".zarr")
-    reference = os.path.join(here, name + "-reference", name + ".zarr")
-
-    # cache=None is using the default cache
-    c = Creator(
-        output,
-        config=config,
-        cache=None,
-        overwrite=True,
-    )
-    c.create()
-
-    compare_zarr(reference, output)
-
-
-def test_create_concat():
-    _test_create("create-concat")
-
-
-def test_create_join():
-    _test_create("create-join")
-
-
-def test_create_pipe():
-    _test_create("create-pipe")
-
-
-def test_create_perturbations():
-    _test_create("create-perturbations")
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--name", help="Name of the test case")
-    parser.add_argument("--compare", nargs=2, help="Compare two directories")
-
-    args = parser.parse_args()
-
-    if args.compare:
-        compare_zarr(args.compare[0], args.compare[1])
-    else:
-        _test_create(args.name)
diff --git a/tests/create-concat-reference/create-concat.zarr/.zattrs b/tests/create-concat-reference/create-concat.zarr/.zattrs
deleted file mode 100644
index a5d7e31..0000000
--- a/tests/create-concat-reference/create-concat.zarr/.zattrs
+++ /dev/null
@@ -1,287 +0,0 @@
-{
-    "_create_yaml_config": {
-        "common": {
-            "mars_request": {
-                "class": "ea",
-                "date": "$datetime_format($dates,%Y%m%d)",
-                "expver": "0001",
-                "grid": "20./20.",
-                "levtype": "sfc",
-                "name": "mars",
-                "stream": "oper",
-                "time": "$datetime_format($dates,%H%M)",
-                "type": "an"
-            }
-        },
-        "config_path": "/etc/ecmwf/nfs/dh1_home_a/mafp/dev/ecml-tools/tests/create-concat.yaml",
-        "copyright": "ecmwf",
-        "dataset_status": "testing",
-        "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-        "input": {
-            "concat": [
-                {
-                    "dates": {
-                        "end": "2021-01-01T12:00:00",
-                        "frequency": "12h",
-                        "source": {
-                            "class": "ea",
-                            "date": "$datetime_format($dates,%Y%m%d)",
-                            "expver": "0001",
-                            "grid": "20./20.",
-                            "levtype": "sfc",
-                            "name": "mars",
-                            "param": [
-                                "2t"
-                            ],
-                            "stream": "oper",
-                            "time": "$datetime_format($dates,%H%M)",
-                            "type": "an"
-                        },
-                        "start": "2020-12-30T00:00:00"
-                    }
-                },
-                {
-                    "dates": {
-                        "end": "2021-01-03T12:00:00",
-                        "frequency": "12h",
-                        "source": {
-                            "class": "ea",
-                            "date": "$datetime_format($dates,%Y%m%d)",
-                            "expver": "0001",
-                            "grid": "20./20.",
-                            "levtype": "sfc",
-                            "name": "mars",
-                            "param": [
-                                "2t"
-                            ],
-                            "stream": "oper",
-                            "time": "$datetime_format($dates,%H%M)",
-                            "type": "an"
-                        },
-                        "start": "2021-01-02T00:00:00"
-                    }
-                }
-            ]
-        },
-        "licence": "CC-BY-4.0",
-        "loop": [
-            {
-                "dates": {
-                    "end": "2021-01-03T12:00:00",
-                    "frequency": "12h",
-                    "group_by": "monthly",
-                    "start": "2020-12-30T00:00:00"
-                }
-            }
-        ],
-        "name": "test-small",
-        "output": {
-            "append_axis": 0,
-            "chunking": {
-                "dates": 1
-            },
-            "dtype": "float32",
-            "ensemble_dimension": 2,
-            "flatten_grid": true,
-            "order_by": [
-                {
-                    "valid_datetime": "ascending"
-                },
-                {
-                    "param_level": "ascending"
-                },
-                {
-                    "number": "ascending"
-                }
-            ],
-            "remapping": {
-                "param_level": "{param}_{levelist}"
-            },
-            "statistics": "param_level",
-            "statistics_end": 2021
-        },
-        "purpose": "aifs",
-        "reading_chunks": null,
-        "statistics_axis": 1,
-        "statistics_names": "ascending"
-    },
-    "copyright": "ecmwf",
-    "data_request": {
-        "area": [
-            80.0,
-            0.0,
-            -80.0,
-            340.0
-        ],
-        "grid": [
-            20.0,
-            20.0
-        ],
-        "param_level": {
-            "sfc": [
-                "2t"
-            ]
-        },
-        "param_step": {}
-    },
-    "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-    "end_date": "2021-01-03T12:00:00",
-    "ensemble_dimension": 1,
-    "flatten_grid": true,
-    "frequency": 12,
-    "history": [
-        {
-            "action": "initialised",
-            "timestamp": "2024-01-30T15:27:48.718761"
-        },
-        {
-            "action": "statistics_registry_initialised",
-            "timestamp": "2024-01-30T15:27:48.724454",
-            "version": 2
-        },
-        {
-            "action": "init finished",
-            "timestamp": "2024-01-30T15:27:48.729622"
-        },
-        {
-            "action": "loading_data_start",
-            "parts": null,
-            "timestamp": "2024-01-30T15:27:48.759870"
-        },
-        {
-            "action": "loading_data_end",
-            "parts": null,
-            "timestamp": "2024-01-30T15:27:50.901469"
-        },
-        {
-            "action": "compute_statistics_end",
-            "end": "2021-01-03T12:00:00",
-            "i_end": 9,
-            "i_start": 0,
-            "start": "2020-12-30T00:00:00",
-            "timestamp": "2024-01-30T15:27:51.594454"
-        }
-    ],
-    "latest_write_timestamp": "2024-01-30T15:27:50.892094",
-    "licence": "CC-BY-4.0",
-    "order_by": [
-        {
-            "valid_datetime": "ascending"
-        },
-        {
-            "param_level": "ascending"
-        },
-        {
-            "number": "ascending"
-        }
-    ],
-    "provenance_load": {
-        "git_versions": {
-            "climetlab": {
-                "git": {
-                    "modified_files": 7,
-                    "sha1": "0c12855c1e2c8ec669714d4b873eb2f3971d6e24",
-                    "untracked_files": 0
-                }
-            },
-            "ecml_tools": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 21
-                }
-            },
-            "prepml": {
-                "git": {
-                    "modified_files": 0,
-                    "sha1": "b9fce5cff2ec951899b13b9d97a4bcdb3e0fe4ef",
-                    "untracked_files": 4
-                }
-            },
-            "test_create": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 21
-                }
-            }
-        },
-        "module_versions": {
-            "_cffi_backend": "1.15.1",
-            "_csv": "1.0",
-            "_ctypes": "1.1.0",
-            "_decimal": "1.70",
-            "_pytest": "7.3.1",
-            "argparse": "1.1",
-            "attr": "23.1.0",
-            "certifi": "2022.12.07",
-            "cffi": "1.15.1",
-            "cgi": "2.6",
-            "chardet": "4.0.0",
-            "charset_normalizer": "3.1.0",
-            "climetlab": "0.20.1",
-            "csv": "1.0",
-            "ctypes": "1.1.0",
-            "dateutil": "2.8.2",
-            "decimal": "1.70",
-            "eccodes": "2.30.0",
-            "ecml_tools": "0.3.0",
-            "ecmwf.opendata": "0.2.0",
-            "ecmwfapi": "1.6.3",
-            "ecmwflibs": "0.5.3",
-            "entrypoints": "0.4",
-            "exceptiongroup": "1.1.1",
-            "fasteners": "0.18",
-            "filelock": "3.12.0",
-            "findlibs": "0.0.5",
-            "fsspec": "2023.9.2",
-            "gribapi": "2.30.0",
-            "hydra": "1.3.0",
-            "idna": "2.10",
-            "ipaddress": "1.0",
-            "json": "2.0.9",
-            "logging": "0.5.1.2",
-            "markdown": "3.4.3",
-            "multiurl": "0.2.1",
-            "numcodecs": "0.11.0",
-            "numpy": "1.23.5",
-            "omegaconf": "2.3.0",
-            "packaging": "23.1",
-            "pandas": "2.0.1",
-            "platform": "1.0.8",
-            "pluggy": "1.0.0",
-            "prepml": "0.52.0",
-            "pycparser": "2.21",
-            "pytest": "7.3.1",
-            "pytz": "2023.3",
-            "re": "2.2.1",
-            "requests": "2.31.0",
-            "six": "1.16.0",
-            "sniffio": "1.3.0",
-            "socketserver": "0.4",
-            "socks": "1.7.1",
-            "test_create": ".../test_create.py",
-            "tqdm": "4.65.0",
-            "urllib3": "1.26.15",
-            "yaml": "6.0",
-            "zarr": "2.16.1",
-            "zlib": "1.0"
-        },
-        "python": "3.10.10",
-        "time": "2024-01-30T15:27:51.293709"
-    },
-    "remapping": {
-        "param_level": "{param}_{levelist}"
-    },
-    "resolution": 20.0,
-    "start_date": "2020-12-30T00:00:00",
-    "statistics_end_date": "2021-01-03T12:00:00",
-    "statistics_start_date": "2020-12-30T00:00:00",
-    "total_number_of_files": 48,
-    "total_size": 21367,
-    "uuid": "c4d1f4c6-0fdc-4ac0-adb4-b78affb9dcac",
-    "variables": [
-        "2t"
-    ],
-    "version": "0.20"
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/.zgroup b/tests/create-concat-reference/create-concat.zarr/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-concat-reference/create-concat.zarr/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/.zgroup b/tests/create-concat-reference/create-concat.zarr/_build/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/count/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/count/.zarray
deleted file mode 100644
index 2808085..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/count/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i8",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/count/0.0 b/tests/create-concat-reference/create-concat.zarr/_build/count/0.0
deleted file mode 100644
index 53471d811d385382ad7ff6551c7cf1a644f2c5c8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
ZcmZQ#H0B6kU|;~@1R!1n#2`Q|8UTf02a^B*

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/flags/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/flags/.zarray
deleted file mode 100644
index e07a6bc..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/flags/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "|b1",
-    "fill_value": false,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/flags/0 b/tests/create-concat-reference/create-concat.zarr/_build/flags/0
deleted file mode 100644
index e86af411a0cf2fad9169e2fae28ded98597aecb7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18
VcmZQ#G-hOCU|;}YAs}XC1ONmj08jt`

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/lengths/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/lengths/.zarray
deleted file mode 100644
index 76820ba..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/lengths/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i4",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/lengths/0 b/tests/create-concat-reference/create-concat.zarr/_build/lengths/0
deleted file mode 100644
index 9d719554d8dadd511f1953deae146050e52ab286..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
YcmZQ#G-lyoU|;}Y2_R+xVm2TK00)f#Z2$lO

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/maximum/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/maximum/.zarray
deleted file mode 100644
index 38c8024..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/maximum/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/maximum/0.0 b/tests/create-concat-reference/create-concat.zarr/_build/maximum/0.0
deleted file mode 100644
index 9571e56f64c3045f00026c439d3b703677ad4da1..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1Rw^21AEPi9e{L$hA@P_EDNR?)Fr^QLyi!HR_2GuPY?vt2Urxq
ObiyYQ2yJ9h>;M1-P7%KV

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/minimum/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/minimum/.zarray
deleted file mode 100644
index 38c8024..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/minimum/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/minimum/0.0 b/tests/create-concat-reference/create-concat.zarr/_build/minimum/0.0
deleted file mode 100644
index 6c965bda7679e8f388a3ddf7a19f0ea9f8383a85..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1Rw^21*d&-96*eV*TJ-c_-hDlw<{MUo=_1DrW@2&f$0YetRehP
Sb1;42$Q7{q17CjSH~;{zHy9ZJ

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/squares/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/squares/.zarray
deleted file mode 100644
index 38c8024..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/squares/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/squares/0.0 b/tests/create-concat-reference/create-concat.zarr/_build/squares/0.0
deleted file mode 100644
index b3f4e11a8ae0dc5ff65e2376856995aa322eb1d9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1Rw?h-IwW(K<Yp%YX+Eh{siVLEaHanML&Z142xO7`~^v$A?oLH
Qg82<Tzrg$jpZPKz0fXKaB>(^b

diff --git a/tests/create-concat-reference/create-concat.zarr/_build/sums/.zarray b/tests/create-concat-reference/create-concat.zarr/_build/sums/.zarray
deleted file mode 100644
index 38c8024..0000000
--- a/tests/create-concat-reference/create-concat.zarr/_build/sums/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/_build/sums/0.0 b/tests/create-concat-reference/create-concat.zarr/_build/sums/0.0
deleted file mode 100644
index fc07651a44cf3d6ed04dd65beb493056348f3f02..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1Rw^22X&vHIsoYdn;D*g=!P@j!Tg3z%wWDlJcRGC7b1UP#SgGN
T!x1L1{DF19A^M$IpE&>k)*&4m

diff --git a/tests/create-concat-reference/create-concat.zarr/count/.zarray b/tests/create-concat-reference/create-concat.zarr/count/.zarray
deleted file mode 100644
index a199478..0000000
--- a/tests/create-concat-reference/create-concat.zarr/count/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i8",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/count/0 b/tests/create-concat-reference/create-concat.zarr/count/0
deleted file mode 100644
index a8193818c9a2b9a0ee78144d7977ea30ca7938d3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
YcmZQ#H0I!7U|;}Y2_Oz(0}?<0019&e!2kdN

diff --git a/tests/create-concat-reference/create-concat.zarr/data/.zarray b/tests/create-concat-reference/create-concat.zarr/data/.zarray
deleted file mode 100644
index fdec9a1..0000000
--- a/tests/create-concat-reference/create-concat.zarr/data/.zarray
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-    "chunks": [
-        1,
-        1,
-        1,
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f4",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        1,
-        1,
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/data/0.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/0.0.0.0
deleted file mode 100644
index 33e89f5964b0e856071f43e39b1cca47376bca55..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 533
zcmZQ#RAlL3VqgGaQ6Ls!U|?7T#Cdvpdh71KyL<O;ED+v}jg8fNrw5|$uG4#$mlqqG
z2js8ITlX$@-8#K>K%DnZ@7=p~Ky5(zcY1n2X5PDZc|a9Fg?S*sbwKzI$OdAd97q62
z=;?t>0W$LPz$~D4kivC9W$S>lKuMrFpvreZi@|OIOT?}N+5mFpdD}bJ+$TpY{dk{K
z(MwfG$nE0wForGMX-rpNac5`tKYg}nfAyM{G>zZV-s+F6D!Th$Pp#dh)DXCV;n#dt
z)g>u^)_yjhvc2&Bf<<9WS3-=}Y`!VlXL7?LalP!Zmx~T;D~Y@=Wv+MgyXNMtzON$h
zbxY^UhfF!&$j7u=)mC1hUs}-FyCx+&r~K`tv)Y$FpU71fxl{dqip1?gUIo5To&N!x
zB}O1uwzju6G}Ja!mXws0l~vc)7FTq4H5FASmFKr~b#ydV*0ptZb@w&3HPkeA^z`&j
z=xlHApU^+CZ|W2vo;G#rl*yAOf{3Y8fV|1mrh$Z}Oa}6%PMS1r%9JTHfFjd?3MS5+
zI=O$s#D1V=Af7mJ!i4_GlO}Y4KqnA%b#`=i0f~<C{M@|k{NnPKw$|3>`lgnaCT^fB
N<((NBIs=?R0071)1%Chl

diff --git a/tests/create-concat-reference/create-concat.zarr/data/1.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/1.0.0.0
deleted file mode 100644
index e0e139074f0d668e31119b33be55d9c0a71f6837..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 533
zcmXAmPe@cz6vm&!1qWv%WfVy^QVR(MLnBIX(cD^02uTHzi=ahB(4T2$9*zz%<2#P`
zy*JGJ|0c0Tl!ehIT(~f7b#W0DNkT!1i^_twEs6;Gt{v{Vob&z8;eK!vE!|8cG)?6$
zWQV3{50H#;$we}WgWxPFIF~}8WyFbuz{NO8hMP$g92-g@LP{o~5j04Ez+4goAc+I2
zHVGIUlU!mOGC14b;}Y6-MJgnT|4szMLj*sZl*^w_zVz5@8z0YR*}KKQuFd7@#@BiM
z?~7OG7luE~y*{yasPp*c;kNF=fv#t}-!6okLBl!S|G0mkHDSGM|2>kb7q>bar`kGh
z&W;Y&Pj2<RSs(lLX|J_-_0Q5+W~ev4oE=E}-Z$&V{h`)JxAuL{HkRJc-J7qUySC%v
zjSB~#>ZSCaaP|4;_QR`3SMoc1kBz4;9T{2gvuE=wVzB&hRJk;2wW;c4d7_Xn6!ghT
zC0Dd9(kHGIOJvr>)OdNuvK_~?rYkh#qB2O-Jtq$1uo(q$6h(n=MPV4k&ATzwI95#%
z22tcUfkUVm3Vjd2@jSQTI*#M2a6I3|rwL(NrfCb)v}`n`eD3!3Tv0cs4WoLeN;M+|
Nm-{m<nt3Lp;vev{<Zu80

diff --git a/tests/create-concat-reference/create-concat.zarr/data/2.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/2.0.0.0
deleted file mode 100644
index 93818d7277d3f0fd2844a05c3e0de4fff9d9e6ac..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 531
zcmZQ#RAlL3VqgGaVIUS^U|?7P#2?vYWaiw7vD^uPJ9k=I?z|%-!_V-6XU-j&n3$cG
zb7FuhV&=re%(-(1$N=IwGNuflb%3ho0Of!ZK(RYO>JCUDQ1s3mAO?#A8FRo4kRVtq
zNKK54B*O=ZIWj;4LHa;KKn&DBXAaPIp#3{7L8?K{x)ZYq=*H+Dg;}{1&Ll0Y?R$26
z#=_d&z2+>+d5jU)^4qt~uU>ij(~<3YR^mr4OLRr|bi}Vz+PSPo&PBv`mF$s8*=KgW
z&yt#J@4e~ro)_=>PTB=-?!Q*DQ+$Qh&#NDIB|e#aNaL2^I>v6+_NBA!cDyt3%3J*U
z$yW2t;qm%YqZM|`&kMPnn_BL=_W2p63EzLcGH0`WoFDk|hL^5`-O7E38jo;X^LV%M
zOBcwM&8_V%jde9uMMcFW<u#S{g(W@RZDkedWz{X6?HvsjHLac9J$((WjSWq0J^j5C
zx;y*&CiYM1n>KmU#7R?uVDhB?sZ*v*0%9P0>a=N7fvm|>CQqF@Y0@+xbH;R_$kZv5
zC(f8UrEkK734Q(j{Szim=${B=PnysH2Ay3%*wNYD-PzGmnxB)KlV4oc+}768(9qb@
S*2oQXrMxpELuY_92mk<HPWYGr

diff --git a/tests/create-concat-reference/create-concat.zarr/data/3.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/3.0.0.0
deleted file mode 100644
index 22c2a5776ef92a881c1c7cee79c10aa3a10db8e4..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 530
zcmZQ#RAlL3VqgGaAs`lEU|^UJ#2><U{P^*sqVUH@5GVwq!jC_G>?i~=D|S>=SbY4r
zV~2&s4<K`g1xVlrkosW()B+Ux0hFi!!XF^X9X~9<Vn255_yIPv!UCipsA9*C3X4<*
zMiGXMKp~4AU}Pc6Z~!a?l-mI`f}7z33(zj0?LcS&wCUqV3!q7hfKIFrVeGoTVz=n~
zjF|g+ZnuqAn<`n|+APYw|Mt=euYPOz_8m??n(<@Wyd~xu4ib|>uaqC*@QjEFJ$~`z
zi@STZu4Rc$jk^(ZXSdG(2M6!WjC^+C_^NkPqz}IndZQ|&R=md1*!hfgew|<7{Z*G|
zvLBi<Gu`sjfdlm-2YZ_C-u0Z6n7*|xxm^BM@aAG$?zkh&b{j0(p8w;zxJbeE(;W7n
z>Yo2f<fK8aY-#OiYpQFgC@L-~tEj1}D=P2lZY?b@D6eSlY;9|(tZwP<?(gkvZ*6UB
z@1D>*p{=EJ!o>dRQ>IRyHhI!iAecO<d+L-alc!Ff0pd-Y3gk?gJb4OGVg`sa10*zc
z%A|?YrcUYYpEz+sUw{9Ei4%ZC|HMfXJ3yebv#Yba3rKc%b#zn|6&B<cmXtQPwl&q)
UH8i(2aRXf`@65>18Q=^80LVQNWB>pF

diff --git a/tests/create-concat-reference/create-concat.zarr/data/4.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/4.0.0.0
deleted file mode 100644
index 2d0f33a1c9d8c54038ee4b4f3f1ef61a9670d37a..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 532
zcmZQ#RAlL3VqgGa5fElzU|0ymAAJ`kEqVmRhYufi27<#$hn<s@lt3&X{U|BP`S2no
zrA3PtB`GC60xDDjQjdUGX%SEkh?O1zxj<1RpdgR}$pLi%(V|C>7A;a{C;)0o0zxqc
zfkpZZAApLIl7Q+LB`sR?2&n22P%TIShz3f5?FBgmXc3SHvTqU4k<G8|=1q5!`quwf
z{^FX&T?=Pz5*NQ=`6vH#?<EbFU*YN^%N;sxv@SExwR|Nwt#etMTH0}gB_@@;*A|q{
zDti64`pfPqrSEyIbq~1KG+vLIwA_6*_d3@Hi;lTBJgm!@@G@j_{o&284z5wTF(c@8
z*aL-)-TkYQJO6xIwPR1P*Xo516LYk?-puFZaDUdBa9iu!F29?zPTJo7XwrUBX!|l=
zvxCpvFMwRx+S<|H+SpK9T3k|ARa4VcR@T$qR*;upT+r0n*4kWE-QLyR)6>}2(A3=C
z-QU;W)zL9&V*jN6sgoy7m^@|b)G3oE^?}H#Q-GYwQ>RS>a;8k4GG*G-Nt31lnKOU_
zAo+<v*}lGs6DRcbPXK~`AUSd3#11g%?CR?71cL6)j*jBI?3}FJqSBVO)|SSG#-^4=
RZlEjWof#QA13*#j3;+zw0!07-

diff --git a/tests/create-concat-reference/create-concat.zarr/data/5.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/5.0.0.0
deleted file mode 100644
index 86f03f803c0431727c6849772e1b7d33b5a5acfc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 533
zcmZQ#RAlL3VqgGaQ6Ls!U|?7T#J8fNZq14^*|Nn1h_`H+WwK>fRMZv|AiOn8>K2fa
zniX~HRuoV$DoX0sty!~XMa=?AL`lt(ijo4#04X3nOA5r30!e~IZb^YuNZkVI0<u8r
zf#4QM?iP?a3n&T!vu=US1etbA3doU?0@@5hCbOh~+Cj3nK&~`<VH;F<HmCH9#D~|5
z!VVj3JTLHBdT-zt-POw?sy02-Ec~Wd9HUg8dg1#?KEY4U&pOj&tm8QrKD@U-s+v#e
z%1Xxk(B`e1LmwW}taMKN_be{9<Z(O03YD1(lYehNa#o;OB%|8Y%v0ljtI8_3>WbZ4
ze)68$z0&Akua9us=FbN^g8$7pIV)Lt*My4}l6Q6Y9MHOQ`~M9$1}O)JiQ6Y|hAd8Q
zj{>=}t+k`Qt)Z^GthBPSvZ}VZrn;xAy(BlkIJc>@t+lqSwymqXudlPcwYjCeySKlu
zwY8&vV*j+sQzuWKJY^~nOrF#^b;^`UQ>V|EHg(F>X+UDil*v=3Oq)6hBt2!u)M+4U
z(nKK9-`_WJLVy2+elR&<;)IDEAkf*_)!E$%1l^q-9TkNI1$l+VrOhqP%?%9=O|325
PKv&8;Gct4rID-HHSF`wc

diff --git a/tests/create-concat-reference/create-concat.zarr/data/6.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/6.0.0.0
deleted file mode 100644
index 292b20262c24c51769f11be8e151885bd7e2db47..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 533
zcmZQ#RAlL3VqgGaQ6Ls!U|?7T#7A6Oj<mS2v$MCix3jkcX_gk379hpm&T^!Mg{8gS
zrNxECg@vUBsO$(3uz+YF7l>J0j<_5FsRB|*faZWiK&pYDrR4~a3BoNcAOnFyK&?kW
z!bd>NBR~cVkOemU$Pu7|BP~G9KxIHDfE>~e)CxjC#UNKMJFK(l-i#?u#@`?4_MSRt
zwujfoIV0<Cr<tbvncz$Q@6MQ?N>DWJcr@vO!{=V_{xxB`g;NCM9&{YuUSjwl|IGr!
zJE;%PTo>B+YWJPTtZkd#3-$6IiN1F{o~7vjn$A+OlAfYtT=L%i467R4qYG<|S-79A
z?w{4a_nxZkX)TG5vnFjyTK{_P!>YAWPWgX#vvW!<ssCuPbkQW;mkEmRZWx??5q}xv
z%C@$S_O`~l^0JD$y2`4$`r?M}uJ+Q*?BcAZ&bGFOs;bt`p6;HumWGDrw(h>Z{;tmM
z-iiGa`=(Bw)IWJD5CG|^Q>ILsI(f>}X+VK#Q-M4Xd+MY~AodK9nyFJJPn<q=O8<nu
zNfZ0}CiG97(BD5{!lVfkJHVi`v$LxUNOX5~c9i5~<z?p<mNmDww>CC5HMTTy16?Wa
L%*fCg;0yu)%7p5c

diff --git a/tests/create-concat-reference/create-concat.zarr/data/7.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/7.0.0.0
deleted file mode 100644
index 51f86a0c922a0648a34503a25bf32505b3466850..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 530
zcmZQ#RAlL3VqgGaAs`lEU|^UJ#2;1C($tncTXx3h%$YN3XU_Nl>1WH90U2j}($t<k
z12WWr{AbV9K)h#ZY0uK0sc|uU)B=jAfwh8U)1HA;fMwH`r7Z*M01;_v%YZac5(L3C
zNWhvw;n^~2hL3D%Y0sVk1%Spb0}2D#K$CsC84mdPsHFi-2J+L?76G03KzPx;h8%}A
zvND0%>ld^YmA^R4d_&JKV*j4Szs%qK{Lr0R_N(>!lpA4tFXi6v=wvN8c;n_(x%`P{
zAFLyi8l(F}x~o1<I$Lp$XCi~wW*t^F=fZovDkl_p_%j7OkMzXUeta+~q0m45T|q;S
z^{P)2O5SxkSyQ+7s=f<+`}x$zsI%SC?Luch<uV)X&A$F%))Ki(PcPoSSu&HM`$*e_
z51n7H^a_An+1lRT*4|uOUQu1&P+3(~TU6iO)mE96SD4w{+1OlHT;0;u)7#hC*4EtG
z-rd_bp`*3CZ$kgnDbprTpFCwM5KNxbF?Gt6Nz-P`oHlLhG$5WbWeQMc+SJKFJax)+
zkeaDeCQY0^bxQw){{D&mKyu=Q34Q$&CiPG30D(>*=mrs8T^$`2g#`t<g~esfEiFxr
V4ULU0E!;p?$~!YMbOtzs001$n3R(aF

diff --git a/tests/create-concat-reference/create-concat.zarr/data/8.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/8.0.0.0
deleted file mode 100644
index d8771c6d40a950205273c587b73a83af08a2c21c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 533
zcmW+wT}TvB6rS09$b6A*8Vl7L$vg_7K14>4;FE-L-;<Ie!^+rEa%^jLXU4fRI)7*G
zv_XVqtUXlF24O$?VECpI-hLnmqIy;cf*v;M+&!G{-tT_reCM8wdbGjGD2hZ2;(($k
zj}Sm95d<olCKI7i0(AmZ5F|hVga9H0mA4>-09t@hLIt|9N+ki*F$D%1QKtymiwsAB
zB=spcLkPgBQbGe(3HG6YvcMnZ5*qem5v@XQ*)(vFKVJD>JX5GextZLBY-^1dv}*B6
z<+<6vth=ALI{v*F+C9;^XT0msxo4GG?Ud2pH@NfZw>EDyH>PBLVf^dn@Q*)3KRY&_
z?7uiyJ+fk{>|XZH%dVw-Hotgj`Qx#r?bC1eoqhkh?fj~KWA&H%?!kw?`O(AWr!-z`
z&TRJob%O=9TQgrRcJJ=J-_!nf=gZp0w^GByTJ`uzxus*z*tS(PCXC6u#x3LaXu)Il
z#IUYkE-}-b)C(nd)|<1<=_%9keBbw6FAjsy=Meh|PdVNZPm&msa!6BDk|g2yh-1mi
zc%(dmsLUaVLO%-pAdJE=2to)#ryh(kk1-d+aq=TquZ>*4Ic8awS(?IMr<-ugT(7FI
IfnFK^0g4IrTmS$7

diff --git a/tests/create-concat-reference/create-concat.zarr/data/9.0.0.0 b/tests/create-concat-reference/create-concat.zarr/data/9.0.0.0
deleted file mode 100644
index 4c4b2c89b5ee0fbd2e342a102815c3ac6169483f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 529
zcmZQ#RAlL3VqgGaK_C`kU|^UB#2?s&fnaj5@Z@Dcpa#ammxa~17`_<@2MY_UEmH%@
z3JV8M4xS8@2o4Uu3?#u!pm;EleR(oS3@9`iC<+2V^DYAo)L{5v02Bu*nJf&H02%_u
zK#nlTG#~<L0_g&oad|SxE)Y_iEDSVx^5n^vg%<%`xIeZ3^QW^jWwiJ1o;6X%fA+uf
z3j*u7UL^g#t~>qshrh>{Ts?V4D){T|{8=6*$&)8|P7r_UKdtc8OT!EGwP(UbBVXR?
zFfn=`EyUb=Fz8S}-v<Be?+Kna%s%opeqJ@NFCpb6JL?kOc76V%YplNhRXdirGT`Nc
z1;;$tvI@!`$EynG9m!YMu6d{VTKlo<rp0eXJU&kPbY+iwh0nC7dsnx;EX?yZ1-Y`N
zy}hlqwXvbPvZ}VKtg^D8rn|GfDzCCAy}6^grK+T=wX>(MzoWgisjaiCzqh}$y{msh
z|I{hdCQqL-W$Lu4QzlR9nmT35<Z07qPMbDm8b}1ln*wA_0^+GtrUOMlswPgKI=O$+
zgbDo<`uZnKoG`J!zkl+C2^}EN+1c6E)ztwcIy*bc3-b$d3yMlxTAG_08yXv1nz@0l
Nly_!i=nQZM0RRlb@Mi!3

diff --git a/tests/create-concat-reference/create-concat.zarr/dates/.zarray b/tests/create-concat-reference/create-concat.zarr/dates/.zarray
deleted file mode 100644
index 9a6e5f3..0000000
--- a/tests/create-concat-reference/create-concat.zarr/dates/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        10
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<M8[s]",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/dates/0 b/tests/create-concat-reference/create-concat.zarr/dates/0
deleted file mode 100644
index c04549e88081fd35feac96ed26ced48006882a1f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1R!QO{5l>)9mswIrW-`wf@z1n5Sk(F9Ymh*Jw$#xgmy^y0Om7r
KeuT(xgwO!$(iTVn

diff --git a/tests/create-concat-reference/create-concat.zarr/latitudes/.zarray b/tests/create-concat-reference/create-concat.zarr/latitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-concat-reference/create-concat.zarr/latitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/latitudes/0 b/tests/create-concat-reference/create-concat.zarr/latitudes/0
deleted file mode 100644
index bc43c5131b3b14758aa08ae329fc209b28364f0d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 179
zcmZQ#ROAp~Wnci|%?u0-B0$Uy#PSS`44nZ$235J5K;vXX7#U>!fY=3yO@J6^6R$Fm
j=?)aiW{~vjX9x%Z0gzMV9T*u*Kq}-9Ff!-|8~_0TnG_Rp

diff --git a/tests/create-concat-reference/create-concat.zarr/longitudes/.zarray b/tests/create-concat-reference/create-concat.zarr/longitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-concat-reference/create-concat.zarr/longitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/longitudes/0 b/tests/create-concat-reference/create-concat.zarr/longitudes/0
deleted file mode 100644
index 3fa39c0fa3bb6f35f29e18b16de9bf93759c73e1..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 199
zcmZQ#ROAp~Wnci|;|vT8B0$Uy#PSS`44nZ$24!41pnf5sQGyLj3`q_R2N)a}gn^bZ
z@Cz~I27vkcKo$Q*8BARKLL%c5Q_?cC^9l-!N=h+BR2b3%z*77`EjkPij384g0vtd9
E0Dg2I-v9sr

diff --git a/tests/create-concat-reference/create-concat.zarr/maximum/.zarray b/tests/create-concat-reference/create-concat.zarr/maximum/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/maximum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/maximum/0 b/tests/create-concat-reference/create-concat.zarr/maximum/0
deleted file mode 100644
index 057d1b38cb8ec72c558aae5946632cdfe7e682c2..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
ZcmZQ#H0I!7U|;}Y2_Oc71S5-L2LKEo0ptJx

diff --git a/tests/create-concat-reference/create-concat.zarr/mean/.zarray b/tests/create-concat-reference/create-concat.zarr/mean/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/mean/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/mean/0 b/tests/create-concat-reference/create-concat.zarr/mean/0
deleted file mode 100644
index da56c6a8eae4e58903c183484ca243d60f5957d2..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
bcmZQ#H0I!7U|;}Y2_WYADR=o;e4zsX83F_|

diff --git a/tests/create-concat-reference/create-concat.zarr/minimum/.zarray b/tests/create-concat-reference/create-concat.zarr/minimum/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/minimum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/minimum/0 b/tests/create-concat-reference/create-concat.zarr/minimum/0
deleted file mode 100644
index 4ba3788cd5848fcc8f54f407fd8b22f9d97fe8d3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
ZcmZQ#H0I!7U|;}Y2_Oc71*d&-8~_jT0?GgY

diff --git a/tests/create-concat-reference/create-concat.zarr/squares/.zarray b/tests/create-concat-reference/create-concat.zarr/squares/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/squares/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/squares/0 b/tests/create-concat-reference/create-concat.zarr/squares/0
deleted file mode 100644
index 18671fc30a080e204c488161a849bc4bd296ecf1..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
ZcmZQ#H0I!7U|;}Y2_Oan8==2*9RUrG0+0Xz

diff --git a/tests/create-concat-reference/create-concat.zarr/stdev/.zarray b/tests/create-concat-reference/create-concat.zarr/stdev/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/stdev/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/stdev/0 b/tests/create-concat-reference/create-concat.zarr/stdev/0
deleted file mode 100644
index c944588354f2e0ff03b5023819c3e9d3ff1ead2f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
bcmZQ#H0I!7U|;}Y2_U`|s90C4V(b6_7V-l{

diff --git a/tests/create-concat-reference/create-concat.zarr/sums/.zarray b/tests/create-concat-reference/create-concat.zarr/sums/.zarray
deleted file mode 100644
index 35cc5cb..0000000
--- a/tests/create-concat-reference/create-concat.zarr/sums/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        1
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        1
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-concat-reference/create-concat.zarr/sums/0 b/tests/create-concat-reference/create-concat.zarr/sums/0
deleted file mode 100644
index 654c2d1450457f3ffe0afab3a3b88812569ad4e4..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
ZcmZQ#H0I!7U|;}Y2_Oc7p4J*^M*t3F0#E<|

diff --git a/tests/create-join-reference/create-join.zarr/.zattrs b/tests/create-join-reference/create-join.zarr/.zattrs
deleted file mode 100644
index 5c072db..0000000
--- a/tests/create-join-reference/create-join.zarr/.zattrs
+++ /dev/null
@@ -1,356 +0,0 @@
-{
-    "_create_yaml_config": {
-        "common": {
-            "mars_request": {
-                "class": "ea",
-                "date": "$datetime_format($dates,%Y%m%d)",
-                "expver": "0001",
-                "grid": "20./20.",
-                "name": "mars",
-                "stream": "oper",
-                "time": "$datetime_format($dates,%H%M)",
-                "type": "an"
-            }
-        },
-        "config_path": "/etc/ecmwf/nfs/dh1_home_a/mafp/dev/ecml-tools/tests/create-join.yaml",
-        "copyright": "ecmwf",
-        "dataset_status": "testing",
-        "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-        "input": {
-            "dates": {
-                "end": "2021-01-03T12:00:00",
-                "frequency": "12h",
-                "join": [
-                    {
-                        "label": {
-                            "name": "previous_data",
-                            "source": {
-                                "class": "ea",
-                                "date": "$datetime_format($dates,%Y%m%d)",
-                                "expver": "0001",
-                                "grid": "20./20.",
-                                "levtype": "sfc",
-                                "name": "mars",
-                                "param": [
-                                    "2t"
-                                ],
-                                "stream": "oper",
-                                "time": "$datetime_format($dates,%H%M)",
-                                "type": "an"
-                            }
-                        }
-                    },
-                    {
-                        "source": {
-                            "class": "ea",
-                            "date": "$datetime_format($dates,%Y%m%d)",
-                            "expver": "0001",
-                            "grid": "20./20.",
-                            "level": [
-                                50,
-                                100
-                            ],
-                            "levtype": "pl",
-                            "name": "mars",
-                            "param": [
-                                "q",
-                                "t"
-                            ],
-                            "stream": "oper",
-                            "time": "$datetime_format($dates,%H%M)",
-                            "type": "an"
-                        }
-                    },
-                    {
-                        "source": {
-                            "class": "ea",
-                            "date": "$datetime_format($dates,%Y%m%d)",
-                            "expver": "0001",
-                            "grid": "20./20.",
-                            "name": "era5-accumulations",
-                            "param": [
-                                "cp",
-                                "tp"
-                            ],
-                            "stream": "oper",
-                            "time": "$datetime_format($dates,%H)",
-                            "type": "an"
-                        }
-                    },
-                    {
-                        "source": {
-                            "date": "$dates",
-                            "name": "constants",
-                            "param": [
-                                "cos_latitude"
-                            ],
-                            "source_or_dataset": "$previous_data"
-                        }
-                    }
-                ],
-                "start": "2020-12-30T00:00:00"
-            }
-        },
-        "licence": "CC-BY-4.0",
-        "loop": [
-            {
-                "dates": {
-                    "end": "2021-01-03T12:00:00",
-                    "frequency": "12h",
-                    "group_by": "monthly",
-                    "start": "2020-12-30T00:00:00"
-                }
-            }
-        ],
-        "name": "test-small",
-        "output": {
-            "append_axis": 0,
-            "chunking": {
-                "dates": 1
-            },
-            "dtype": "float32",
-            "ensemble_dimension": 2,
-            "flatten_grid": true,
-            "order_by": [
-                {
-                    "valid_datetime": "ascending"
-                },
-                {
-                    "param_level": "ascending"
-                },
-                {
-                    "number": "ascending"
-                }
-            ],
-            "remapping": {
-                "param_level": "{param}_{levelist}"
-            },
-            "statistics": "param_level",
-            "statistics_end": 2021
-        },
-        "purpose": "aifs",
-        "reading_chunks": null,
-        "statistics_axis": 1,
-        "statistics_names": "ascending"
-    },
-    "copyright": "ecmwf",
-    "data_request": {
-        "area": [
-            80.0,
-            0.0,
-            -80.0,
-            340.0
-        ],
-        "grid": [
-            20.0,
-            20.0
-        ],
-        "param_level": {
-            "pl": [
-                [
-                    "q",
-                    50
-                ],
-                [
-                    "q",
-                    100
-                ],
-                [
-                    "t",
-                    50
-                ],
-                [
-                    "t",
-                    100
-                ]
-            ],
-            "sfc": [
-                "2t",
-                "cp",
-                "tp"
-            ]
-        },
-        "param_step": {
-            "sfc": [
-                [
-                    "cp",
-                    6
-                ],
-                [
-                    "tp",
-                    6
-                ]
-            ]
-        }
-    },
-    "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-    "end_date": "2021-01-03T12:00:00",
-    "ensemble_dimension": 1,
-    "flatten_grid": true,
-    "frequency": 12,
-    "history": [
-        {
-            "action": "initialised",
-            "timestamp": "2024-01-30T15:27:52.136204"
-        },
-        {
-            "action": "statistics_registry_initialised",
-            "timestamp": "2024-01-30T15:27:52.141925",
-            "version": 2
-        },
-        {
-            "action": "init finished",
-            "timestamp": "2024-01-30T15:27:52.146785"
-        },
-        {
-            "action": "loading_data_start",
-            "parts": null,
-            "timestamp": "2024-01-30T15:27:52.180610"
-        },
-        {
-            "action": "loading_data_end",
-            "parts": null,
-            "timestamp": "2024-01-30T15:27:52.494787"
-        },
-        {
-            "action": "compute_statistics_end",
-            "end": "2021-01-03T12:00:00",
-            "i_end": 9,
-            "i_start": 0,
-            "start": "2020-12-30T00:00:00",
-            "timestamp": "2024-01-30T15:27:53.003118"
-        }
-    ],
-    "latest_write_timestamp": "2024-01-30T15:27:52.485231",
-    "licence": "CC-BY-4.0",
-    "order_by": [
-        {
-            "valid_datetime": "ascending"
-        },
-        {
-            "param_level": "ascending"
-        },
-        {
-            "number": "ascending"
-        }
-    ],
-    "provenance_load": {
-        "git_versions": {
-            "climetlab": {
-                "git": {
-                    "modified_files": 7,
-                    "sha1": "0c12855c1e2c8ec669714d4b873eb2f3971d6e24",
-                    "untracked_files": 0
-                }
-            },
-            "ecml_tools": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 25
-                }
-            },
-            "prepml": {
-                "git": {
-                    "modified_files": 0,
-                    "sha1": "b9fce5cff2ec951899b13b9d97a4bcdb3e0fe4ef",
-                    "untracked_files": 4
-                }
-            },
-            "test_create": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 25
-                }
-            }
-        },
-        "module_versions": {
-            "_cffi_backend": "1.15.1",
-            "_csv": "1.0",
-            "_ctypes": "1.1.0",
-            "_decimal": "1.70",
-            "_pytest": "7.3.1",
-            "argparse": "1.1",
-            "attr": "23.1.0",
-            "certifi": "2022.12.07",
-            "cffi": "1.15.1",
-            "cgi": "2.6",
-            "chardet": "4.0.0",
-            "charset_normalizer": "3.1.0",
-            "climetlab": "0.20.1",
-            "csv": "1.0",
-            "ctypes": "1.1.0",
-            "dateutil": "2.8.2",
-            "decimal": "1.70",
-            "eccodes": "2.30.0",
-            "ecml_tools": "0.3.0",
-            "ecmwf.opendata": "0.2.0",
-            "ecmwfapi": "1.6.3",
-            "ecmwflibs": "0.5.3",
-            "entrypoints": "0.4",
-            "exceptiongroup": "1.1.1",
-            "fasteners": "0.18",
-            "filelock": "3.12.0",
-            "findlibs": "0.0.5",
-            "fsspec": "2023.9.2",
-            "git": "3.1.31",
-            "gitdb": "4.0.10",
-            "gribapi": "2.30.0",
-            "hydra": "1.3.0",
-            "idna": "2.10",
-            "ipaddress": "1.0",
-            "json": "2.0.9",
-            "logging": "0.5.1.2",
-            "markdown": "3.4.3",
-            "multiurl": "0.2.1",
-            "numcodecs": "0.11.0",
-            "numpy": "1.23.5",
-            "omegaconf": "2.3.0",
-            "packaging": "23.1",
-            "pandas": "2.0.1",
-            "platform": "1.0.8",
-            "pluggy": "1.0.0",
-            "prepml": "0.52.0",
-            "pycparser": "2.21",
-            "pytest": "7.3.1",
-            "pytz": "2023.3",
-            "re": "2.2.1",
-            "requests": "2.31.0",
-            "six": "1.16.0",
-            "smmap": "5.0.0",
-            "sniffio": "1.3.0",
-            "socketserver": "0.4",
-            "socks": "1.7.1",
-            "test_create": ".../test_create.py",
-            "tqdm": "4.65.0",
-            "urllib3": "1.26.15",
-            "yaml": "6.0",
-            "zarr": "2.16.1",
-            "zlib": "1.0"
-        },
-        "python": "3.10.10",
-        "time": "2024-01-30T15:27:52.702421"
-    },
-    "remapping": {
-        "param_level": "{param}_{levelist}"
-    },
-    "resolution": 20.0,
-    "start_date": "2020-12-30T00:00:00",
-    "statistics_end_date": "2021-01-03T12:00:00",
-    "statistics_start_date": "2020-12-30T00:00:00",
-    "total_number_of_files": 48,
-    "total_size": 53084,
-    "uuid": "e36369f3-1209-46a6-aab5-d8b3e1e718c3",
-    "variables": [
-        "2t",
-        "cos_latitude",
-        "cp",
-        "q_100",
-        "q_50",
-        "t_100",
-        "t_50",
-        "tp"
-    ],
-    "version": "0.20"
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/.zgroup b/tests/create-join-reference/create-join.zarr/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-join-reference/create-join.zarr/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/_build/.zgroup b/tests/create-join-reference/create-join.zarr/_build/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-join-reference/create-join.zarr/_build/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/_build/flags/.zarray b/tests/create-join-reference/create-join.zarr/_build/flags/.zarray
deleted file mode 100644
index e07a6bc..0000000
--- a/tests/create-join-reference/create-join.zarr/_build/flags/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "|b1",
-    "fill_value": false,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/_build/flags/0 b/tests/create-join-reference/create-join.zarr/_build/flags/0
deleted file mode 100644
index e86af411a0cf2fad9169e2fae28ded98597aecb7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18
VcmZQ#G-hOCU|;}YAs}XC1ONmj08jt`

diff --git a/tests/create-join-reference/create-join.zarr/_build/lengths/.zarray b/tests/create-join-reference/create-join.zarr/_build/lengths/.zarray
deleted file mode 100644
index 76820ba..0000000
--- a/tests/create-join-reference/create-join.zarr/_build/lengths/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i4",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/_build/lengths/0 b/tests/create-join-reference/create-join.zarr/_build/lengths/0
deleted file mode 100644
index 9d719554d8dadd511f1953deae146050e52ab286..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
YcmZQ#G-lyoU|;}Y2_R+xVm2TK00)f#Z2$lO

diff --git a/tests/create-join-reference/create-join.zarr/count/.zarray b/tests/create-join-reference/create-join.zarr/count/.zarray
deleted file mode 100644
index 5c525c6..0000000
--- a/tests/create-join-reference/create-join.zarr/count/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i8",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/count/0 b/tests/create-join-reference/create-join.zarr/count/0
deleted file mode 100644
index 066958e7c46bda1de9dea638cc81da792bbe7b9b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
YcmZQ#H0E$%U|;~@03Z%w0}>R_0J4Dt-v9sr

diff --git a/tests/create-join-reference/create-join.zarr/data/.zarray b/tests/create-join-reference/create-join.zarr/data/.zarray
deleted file mode 100644
index 442f5cb..0000000
--- a/tests/create-join-reference/create-join.zarr/data/.zarray
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-    "chunks": [
-        1,
-        8,
-        1,
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f4",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        8,
-        1,
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/data/0.0.0.0 b/tests/create-join-reference/create-join.zarr/data/0.0.0.0
deleted file mode 100644
index d6dced9b9dfd22166944400292add9cbe6c9ac7f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3320
zcmZ`*d0f*+7Jmug5zCE=B1<Hq;*naEoa7)kfrP{m4j~W-UMLZiB303#AP|%@ptiL|
zsM}g8T0E#yBo@|>jkUEN6%UFoZoS#4SY>VNzF+L_{<ZIynR&<ey?JjkGru31G~Hbo
z06?^tZvo%{Eo6WvrDi&v-aarqFfd@iXTV@E(1+>7FwjmPHk%CwGs^Ae_F+SNJG~v7
zd6+&t+>UN&AEwihH4hJ)5rIH6p=`%z7<p`HBNW)t=|mJ{%w`vd?gX$MVeM!|BjOM_
zj1%tw(iAvgXvgT63G#qoAVF|3whj__MxsbVRt|qDX294=a3h1)4DcpHvD*P+2#x&=
zDGJ{5gTT*%!HW#!5Fn2MGM|8wyn*ZoLj@xR1qCA`*b_ewm^m^s2Hd^Cw_qap6pRIM
z2bchED2z-6H+OKyB};;_fdEqgW`G~Mz)bMF7Xuy`b1IN=H75WpzyKt8l1VesXr@+^
zX{dDwIa({tWNK~2zO@xqCK~nxf=~<%4WR`^1fybT2;H!w4bf;Om#=Fe;HEdqRvL{k
zqdU4FjBuvOz_|f27;6YsCKL9pI3;qe1V4lwG2I&$c@stHP+FB%iqh&(VzW9N6o-`{
zB{GyZ5<(ml3ZfjSRU%BOBserutX3t_(2jb-Q;Cr%*b%Pt!HzQe5o(9S2qOV5Gn%Xv
zRG=OwBoKr%s*tP>F>FCK5hT#9csv<2xJLm*_&<#9{(*P%CwW8i{wIUurz^w!{A52~
zO@yNpmXWVKn6PSP%|Cu=>Dky;x@`8Vpx9Y|=M+`dJZvmJ8EVlUhJStR5%x*;Z(lrP
z+B)+Gn_CjemlGDY9l18KdQm?s^HA`a`z`k41u0jjGU?Y|gdaJkdXVySRZwn7g6)SA
zFY>`KUWiXkkgqs)Q}(LW>mS#jpL_Ay*<4z{&5i%G1>VT_q<ZP+z1EB`h`2z|pF1@O
zcWTZ5c4`go)F1|ouL6>5rv_O3!QVI@o>1+77HM&&xsGIiv(Z&FQ1Jn<;DH?n%>>s(
zSQzlc=a7#E&YZ2NMWrywLM3c)3;OTCVPxSJ8^}4Jy9I$3L@z?jcmOi^<axl8*RQvL
z;Ka7ic>~}5JckiAt-80rdui{i4%3m>MP&`QXLJ_kKfJaur1ST}wVV>_(%-ii#@2^P
zzk8rm@Xy>0%`JU=XwVeJ{Ds%(lha$bz9^M3C+zg#u2XZH8}izD;a^PmJk@pE+%sp5
zDn`HV(A9A}^3NY4U2=G3$i98jU+_m?uE#*<oqe@?zg{=<{@W*fUz{EYA4<{R7C%17
z<aL$*yj6E3^7y&C`iM(2&#Lv_0f~0M_ec6K%s9LI?mhq2=gxeXVF(*KsY`k5ql1|P
zTRdL2d_Q)$ujT!oul`E&<BzG5^^Kd@7CyQ+cb)%iUD5saccZ7=$Z7fYM|#imcxUHb
z$rtYx@Eolltxvk5xkF`L|FSK+d{1(4CGYgzPyI$WU*OO>JSQI0bk`Vvi@1JuXXIsy
z<MjDIOq;G*cx~KillJ*+sv16h{rJQFNBcLYanC(`VOcOK?QO5u5B*ZAHhJB<!DF_D
zd>(lK_)8-Emk170Pjhb`*f@Vr;pq)u@7x3<XHq-wjqeGXx!>yO(S*`3nP$yQilH)6
z-kCf5Rp|a@#!UVP+bYS=pK5+v{pBh3x6b_8`_JwD<g=5CXQZdKM^D7N?^<9~RsAjs
z^;>iKQOWneW^lTwwyfLsv~E#)z%PkMBhs&o_Al7G@72<tl?ubu&FN;vB_Hii8v-7B
zy64`x^#OIkwPPs>4q2Y2xlV4(sP3J`3#IPbb9=j%nI&0up;K+1{;0ceT=RA9i+dXb
zWvdOlPyV<e=aZaP=5dLw*#B?$pS!D9?F=7tD3s>5yZGB(O<83XUsD{TUT0dAFK%Q`
ztY%2pd>MMqcgpM>+rAsS%l`;EHmzBBf;y=)Q~av<n)uy0k0yVvJ81En9amXY$}ghh
zzON;ZAp=!tq`Set$W+Y=3OvW74*kS72B$C|R^V7`UjTmgvEZ(E^LKi~Q&jD+--6m^
zOEa<-n?JxjH{T1%;QlsoxH>Fw!2)(X5*2n|;$qG_Y)2eGF@_%9T~=OZu@qb0D<~)|
zEZkUJysoIqx^?Zwto3V3trZm|?`_^*X|1Z>y4|v=q+-X8owb$a<u$c6b=8eFY)y@g
zwubsTqHDCF)X>yKsB8@=HP+WR*=)AmsA)n#-JZsVn%cS=bjDU!S6f@tP+wa?P$iPJ
zvZB(8UB&t}x#m@C)~zqyURJhk%huA;t?v-@=Pe?^TO<#g1#c33-{r|6Sh5P=KL5E!
zXokW7@NXCg&oaeeb(MK)An4XH<a!_0yJQ%JbdW6G4%2>mj&Cb~lD^B8<)ROv`JyIB
zSp+wZ-(ScAXB#QH0g|BvK83bNfbYJ;Fmr2T7ve7-Rt~`VIdf+v;#Gq3HrMM9)`QcI
zS2HRA6f(I?Dv?NKNKz>=NF)S9kW4C*Nab?506Upjf-sp#E|V>iOIZ@Gf;Z*B<7Y1g
z^CnM7({aLtlcr9RCWrP7K3_;7k!J-hoITl-yFBuWukS?a;sn_o(HgB<rPRi#V^qo*
z8E2kMrNAIx3B_WD@B&YYGf2hq-b>$f_jGl2ojJx~u$hdx^Sygd^u#GuYPCiks}M_K
zRJwSLN~PAs#>K1RWD>bTCXG=^F|!k37Za&V#P~=gJ&=#3C@}2_;5;soO8G7lAc>*{
zLLr|o5QzkQ9-l9Y7Kz0|L3FfGz*)42!DLftTo#uX*uiIqv$#?C1hJ#o95#o;@pa>J
z2~Pnpa^X8sF5>frGI6v>rqZVwGc!_?6LlJuQl(VJs`RPZ%a$6`Qj+xvS}i84idQG-
zlMtX=tcllZ6>?l%u}CDA$&r=F6fq)9LMY(zc*1D0Bw7$Hik6C^u^usEWHPz@5<Gnk
zdYu-d#cQ-Wtwy7XkJsw5Os33?^z?M25qm>w%98Y~?Cfk)R+ee$vgON{FEwQvGqDPx
zEI!GQnU#^2q>Yt{1h`TPRZ_;XW!ahO$qDMX7=;>Xu`V@r$&!>ry;iG<SL3R`q4;<d
zP*SPlTtp1W$HuARH1R4VH3A4Y!6ql^6A}{iiHQk1?P4ABdLjvv33D>08w|!QQ})ue
z`FUn@Ztm(;IV)G@tTyN6uhl$w31v`PCbZN|wLtSeo8rTX`y)0Rr@4RAT}B0%8--Vn
zpKHT54c5bS_z2EDZg7yDNfNjS$;oBl)f57K2%tZ`y8q^iy%mAZ<mQ|HFgRg~K?tz8
z<JvtuA_w4+CLj*HA#fU~osHiizQlH*10*lB$?liz9+s&3LvXY%;uf^I<_e7H_Wl@b
zE1c^{03l)$R0s1(5F#K!j%N+b=i_%Rjm^Z*0oFLMvw`DsVzD`kC_v+|rh=2r0yeOi
zEMT(O$N{i9EEW@G27}3EqX(PKVL&8{%R&ecU^a&c!vGG0#Y8+S5(gHGvle_KHQ*0R
zz!x7#fGr3j$OC4?Mk4WD-^-Ez=X;qnj`*_9^w>oVp5EbZ)i~BZqcb^glEai(3?@#=
zBobsZI4m}c!)1Ugf3B-6E;}lciz#wZ=OXqTH)j|Y>#B?MrY<Ivs0&L#VgQ5v-@2F_
nE^u)bBC)O-H|JzFP8i8%VKCyP6ERts0WK908tdh-0b~6e%95Gl

diff --git a/tests/create-join-reference/create-join.zarr/data/1.0.0.0 b/tests/create-join-reference/create-join.zarr/data/1.0.0.0
deleted file mode 100644
index dd3ac406adf22493494c87505b234e032457c0a5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3340
zcmY)w2~<;8_GVE~v_OfXL=Z$P3RbOz{GWvE8<-9Fh#*T)Hc=@kDk|0>5r4oEmVj8b
zGH$hMU8^FcN<ArqR(nvH>Qo(RwOVUcm~PTqo%^FbGjsoc@7?v?ci+G7{mW*BapgP!
zri~a0fCC(20}L3rT(w(Gdwb)HpyqnhYPFlDk>uLdb~{aD$5q?iT<p1SMAT@(aO`$$
zZmt`7cH|)og(z0L?OYVt-R=+=@L>$?C`B<Us@-UeHYny2FgO-<88+J68F1U_m+PRp
z=*Et#QQLbN7y`k-g5b5-j<H~*0LgGfbr7;P2D}`=VZ&<^!Iuqd$GcfDC76x{J{<r)
zzA%CfobeDq4}(!O3)oYD<qHE|l8z31UpC20Kw>xqbg+Rn4!8jTfdKvhLSO{|3lS}_
zd;s0TPyk=tLjar#`#cyL0ZS#4j<T{2Nd&-DU`+%FW%O7A00KY&1FQ~LuZ!p6QP@)y
z<-&OSqk9pdTs$5lp(q@CdA&SWZ!aoQsM*U549|r&%tLv&E{01nJZveD-HVnymlDoV
zj0xH^su=()84oU;0PQKBN1gG3KG6dY9aE@{Q}HO8?xywL`fj}*2ihAOGhzaC<HyKj
z`RO?@Fj`M{cYAo)qk6YD65SpvTCex^?)JuB@5c@8rX!($Dl-?hZkz&zdc@EOr=^jj
z??yYcM<0wDcKCQBi_xrr7r@}%i~<S}4dSfxVuJ<q6(F7qFxc|*hj$}tF5kM56wkeS
zY+A_e_KI8I?OyTcnTyH$q}R7zQeK|Q7tNOr4=W81ITF~j&vm=8);W8|-WjvJ*wq*O
z28}+Omfz#oYKQaQ-jWc#Npmma%ieiUZw6H#EBNE&Je4>qv^{=SXhZ$|>W8gjuRX=V
z-^bUUytZ}Q?oCwT>uE)ECmvX_Ds+si>ytZvGrNSJE(?ev%Y3o~#@=Z)Tb6xFN0;wP
z=waYh&hjy5o&U{QC*~|a28PuD%agM_YXEc}flg0+7$Kg_qdm|@Bi&gJ;0J8X$muZP
z9}1lo8`#qOpw(t$^d-;=)}^5y!wXO4k#MvV<Q@vZ<`2avvEl2(91Gytc<}R!7wf(n
z74MhY)ipTNH+Uj_?#$Ffs`dA7ddah?a~C=1?zLNjs+-U9c^bc~!wz1J_`Qg~L-+An
zNE!Jk_?y+eqX)~*U+NK6r}>7|o!#nw5Pn=bc`J9aG4RUCCxi1!G6TctnL$64PpTM~
zcB?q*@6L0R-N$o6)>S-@7`Lq>eT}=9isR7gyo>gq%0?M>iX1r~44#jy5B%op>BBWq
z`Tf32g0nvvd)Qa7b@ADE8~w(eI=yzuPV<Ea%e@vp);KtUaUn%F(%yUb#f8SIiJ9YX
z_%}}H^f}57FYFu@b}3UQI%I15_16)DS!?2CBdc$3DV+cL#l+O8oPE#Y@BBc&nYtxv
z7q?^duAaG;XkFV{9dDM&^yK}R9lK0^H-9{OerEXypMqtrc{@j5X#SU^=DdG_@rQ?Z
zr!C4G<SZKfjL%X91%2%A7kW)UDgVaA6Gb}~S^Zf-8yv0gr$tVZCeJDgTi^EJv+cao
z)WUCzZk{=@@j=%w7aQ3=-$q!TNuIWx`0C1d?kL}n0x#{{XTJ5>W9!qsuJV#6{}|I3
ze);!DUn{?8Tlc}(EwyX>XTPBtviqNl_k4HSF_dsj%X;)$%YpKykoy(e=8|VtUmv!h
z%>P)fYfOLm)N#LzY-dm2xcm7>1(p--NA0b9C+07_n~`jMSlp9%=6iU){ii^~fv5b<
zjgA+Mmwp@G8lfc*ObFB-duPSjkTGHA!uSmX&V&HJe6Iq<TX$C8t;yWKa{c=3WB~h>
z^p-t|K?{a@DUaw+U5;hFwZE(Xcz*4v=UTz=I*IXblc<W&ZC6XZ9tT^Gx6AzdO8;D0
zupxvUSyH-5_)+eD(}#cj7?Q@Hz0Lda7yC~9``MQ8@qOF;9v<kB{^mG+nEM95aYFgK
zp5yFdvv;t=TaWBvAKeG;SDE{MFy_ClzZp{(+L-*uW5S8TaVTfNr?0Ekb5i7;a1xtM
zAL_B%*(aX>5YAx@g?`L+JRSgba*qhC`T!__Bl06i;swrT1RjMRSj%N@lgg^9^%ZN%
zSC=j;EnTt3W-D1<Q*B?dI&b+ZyQ7L;Usk@cy2j~rRBy1^H`XCmX{R^UJDXk2uFXx2
z%}q^BjSbaJE>~mo=9XsUHa9awqpPv0sbMpUUC2cxS3^Atob~l}wRKLXvyK60eM24I
ziZt!0b~tKihoia%iB-!=mM$q-zM^tNWo5-X751vinGC$TP1Jas6l1gDje>u`VjTpl
z7r|Kg>pBUq#e?2y#e0N-gCK(d{KrSORxQ-yKS8T_1E3fMEui%AI#>!&x)CY^PuZ+J
z>utBcsQ|uTf#pI=L(6+X=HoMLK}&BQ?92n3pm4<4bhK(63&MyoTftgQHy_A>{RV*R
zR>-z}4UmGHt45X2he7{poldXQsMQ)Rf<~j&>$FJd)EWe2;G;v3>vUSBMx)j!v})AT
zX$hS{r3@c7pwY-?%ze!~FPgOQ{lcPF4Y~S=J8kB0{^&80Q9)oy82zm;S1_MSluajd
z6EpM*y*@53UKgj6DTIc2nM&ENP$<>fxNFKQnk*RjzW?*CPuts0x3?V>6J!i2>P6~|
zVYbm|G0l#TQ>)_)2_}=tkeD#rXr_!BwN9^A>CCub4K7Qk(qbTLm0pFNCjeZ$o{0(p
zX{l5ylgbs?<WiYj7Auu0m2%|C6iTH^EmtTNQW+_X6%u6FC@~?94L@-uHj0o)#3Hfy
zO^HZEiV$H1OQf+9sZ1>v3ki`(AQ7viGObFX)Ebl1t=U=WsdJJn#(1+KKHfmh%gN2L
zrl+P*NeKxSqrqr4QOR>sQj(GqEars7IGskTRw<QAZ6&Lpt<~$53K_00RuU^$sMN|9
z=&#kPlo*Rjslw>+GvOzp5)%><&1Q1~&S6O~n=J|13v%XXXJuw&F_4*_nwma8CpR}|
z!GZ-jx%mYJc{$luD^^W%tvMybnw^=JY>w9`WIC-zZ%E0?%Uxj2m`j-qadBqc*~FxI
z^HNh&=OkM!W{U+GCX?A@G@8sWdy~;H+hFiCqtQ47It^x%0Wn5y#xiLUqA4lKSp6i=
znL{NdCMHrzRC0cPo@X`lt(h4aS-9NXl9IQJmMmGkcu`?t;iAPwZx@%`SFVL<;&pO{
zyl#_eMxzamH0+PLHdn5@^Dtd>+#7DsfDU-44wDwHWj3bS0m}68Qvv#sQ3dT^mQUsM
zo5mf23zBm%h0om=IpPp>Rkbwt@LhWk&bOQhg^y3&Z|mOzuy|MCSPHGiL_x!<?Z?1c
zujzUg)pq~-gz3I@D?)m%9vs-L{I$CiqWe$_o^+niTU}Pa{qiyvz<3o4YJ;UL7%yW%
z;YcS;lVYVCE{evoKm;VCk431&bBskXA~AAEF@a@e3<?M!2$DoXNCGJ$qDe6+AW=vP
zg_!I@QY<2fm}o+RYy$U!#0RZnP=OQ@5X1CB0U^YRtS~BiICDL+%fZR$Ffarr1q%eg
z7yNG}Ex<}z@K+_x_EZ;E-!(AWcM7K#D~-AV6-kP4QuL2N2t_f17)*5L?2&|+6boYr
z5fc%!e!v4D`M-$Bzak<$5rIe|5{k%SoB@`E6uk-vn1Es?LH}g*gp&vf0m<wFx*-U3
XEW}+R5XH$ES^$K{7j6v`u!#5%h3jp%

diff --git a/tests/create-join-reference/create-join.zarr/data/2.0.0.0 b/tests/create-join-reference/create-join.zarr/data/2.0.0.0
deleted file mode 100644
index 4c85f45d0582e325633719f5fc0a69b6f48bd080..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3336
zcmYjT2~?9;7QP9aAWIY%v|yrWjY`Fh<WEA#o<Pis5D+2(6o|5FEl?E=vKRy*q9~(L
z@U+$Fsn)F))GD=BZKaNTZHE!JYOEd$?s}YI?u(r>bN{#8_uc#MyZ4><-hb(|vEJfg
z0Hh8I0>BPB=m7n`;o;66Gq3ag*V)PAb@qgZhd^JTvnSka?&LYms9|=R&CZ@4WUxBJ
zW8hg7YB^Dc0>pZ->miDW_BgS6aAcewhJc=_M8_OH3f@FG!_gp_M<A>?-|58Fas5sn
z(I#&7n6DD@fe4^ML<QD^Gzf~pVI;CD7*oLkUpsiyA$UB5(4m6KqQS(4VzHRTVoAV1
z6avOT7=ukT0;MoqOat0DpasKI9|nWraFB`(X$g;;5(+dg0Dqv-V5lPsXc0gk3@{WR
z1iXD=hyx|j0K)-9K=TD!2ml{wbYgNWi-o39G6cXESPYg7U^39k0sNr9n!#YUF?%h&
zZ5Hg>+K};#+pz0pk`aPP7UX-eL6(Ws#<XB<YwN|rL_JjNMS;bFJ?YS#rPqQm^27x4
z$e<XlBG^VSG-@G+iLQm|nTaqm==*9I*vhn^5UsG?ZnulgcC%epwu^0byOFnAt!%c-
zMG&jY<whafg=$umTG`}dvrz>FRu^)fj;Oj3aamF7wxTXtL(8Ng+6W<wj0**B6cFJ3
zbmVZ14H>&V6>Sr3&vff(uT_3@FyP4rnBfiZrZxTb;zfmy?(7{Ejraf2x})OQI<9wQ
z5iRw4@tPxT%XgoDdit9p{>al;!q%oY*e$!qb?#g-Mm|itC*pK-e)rKo7meN~ntkxf
zarcwPALERNnyxSH9O>e}xz>L)>)wV_?B9m$qt*MY`C@Ba$CFv=qR$@P`<i<w#S-0;
z&N{~2o^+)!r%bW;K{ws;>g^v~U*X+i<HH;3kHm4iPn@beJ?MJ;lLOD!UL^F_`TYRT
zZ`1$IZxf#14>;goPYzHSKo^N?3XHrFNPriEyTDEm5Vw0oUAJtWAp$-CbVzh|#ZPM;
z3{PnYw|2or5aS2nZVd%c1B2lQUUcwQk$d6o&4iZ&163ENq(6>5+&)DVG{DaN`p0+5
zX-gT|zr@ltyIP;b>&0c`hfgbg+Bm5D-*3M=(mSz9SK4un``MjT|FMx<yO$a^SKLgm
zk}k}u`u*JCm(x2Yw4RC8`=n7tXJ?)d9X(X9cyM{osEk?ugICUcbtQGnnSv8<m+Z9j
z6OyAJU#gkfs=t|JE_itV+{>u}WhULe8Sc?v9u2&2IzImSuze5mQ=3LebE}u!d{w;t
z5B=WGfYSjLf0~|8&g-SkEPIewm8bjOux9xa$Nfu;RW+)sGQXLyeB4Oqu-gL-mo(4*
zzArN3%amauFU`G8#?<c}!h$+`L7X(~q{*k*{o==_FHig;S$pPh<*yIAva^k=-}zrK
zo?mca>D|2jcc<NZQGV*(3IE!fW8*6K4=)nlx!$-ZXW42mpOon6rSGB&5C55Ya^ka3
zE|-S|9?l(jdqSBp<X`2gfT@L-j(m4)9$b56bo{*d`GN_Y{Y%8u`5BkqRHU}-d^CU7
zVo}D+{OJ)t$j@}WdN5?(@Nz%@H48Tvm48^L`Qa%n8&{CJ?O|@utB5gWk<`jv_WA(!
zjrLl{Jp29yWq0qp)*X_R`27C*(VK>A!96?g?0WlXfI53LWxIX(g=J@{$edNa{riNy
zdmWD@M}-%v9#UR?w*Kg=yyKTwwXQVl)UR|~-q=4Z)+`!zTl*sGnjq`@yYGIhd%E)T
z+tu`ycgB7CcwwL4*@?{1mk+!f(o<{F4!1;Y+qS%zcK*WVfh{SMbG|EX-%vWqF(e~V
zaMpJ8Nyqbw-_~rZJvLhL`5T|@;~JRd74={E`b{`e5>?1-7-62Ya>sOu#QRCq&^xrn
z6Yu$}FYMi=cO9S7H%T9AKKc5qywrC;y;$g$Z8YvGE?%|m!b^boTX<A$2k8BQsn}=Y
z>1P47BX%Rx21(fCsKn$oOgssE>A<j!fTXJ|#-5T9@GP)v+AUz?HNc6`(}fKwIQccS
z)4hh+wB&ZoDzUYP0%JdeTmqZeMy^l^RC$wEqpD_2b>+$x%a$zpcxl;+PggBo+E8Cx
zTAo+Byt;0U-B!M$rmnuB(N<GwTV31Gw9Zjq*VwqeX=CH24bAJDTaY$1H??fs*o+nV
zmQ9;lklV0vLrY6@^Co0lHzU%ral`u7mW_@0XK8F|YI3Z1G_6Oz*<trk-CF$Yb@lai
zcKavA1%(C0AD33u)>hkWmDRPC(+U0cYRK``D8XvOmjnNLB`S!J=ff!Y@3Y~=*zwtj
z47wczLpugRpI>V})DaM-R5{mTSLsVP0fa-pbla4!4_%i5>OZRi4piSLUA-bqLWPmd
z>rVNqBs@$ucO7&X6n>f24xquYTQ#s547u=lzpXv?ClJ1d?W|VlYYqlDj(G&wOytkn
z|Ie==z_Xw#UZGOT<w~Vep;RbTs(7UW2XX}=0xFa$l|+s$@&ct?B|$x@e3n|7V0_o}
zOr;k@rH;|av~eFXBLb%6b)Wq;Auei`KPOtx3xB{ejPdt{<id<OVIL`KQ%!QMN}Zrq
zYc&a?_-Sfwf>_pdPAriM6~2Kp<SMyBI`dB7w}+1${HCMhkWeh*#(sVFcE@qGMmO7_
zQ70s*R0*^7TAfytXwYa4CJlNWFO$oSYV?*ktW?Mpa=9!%0UH^PiPtI>hVV1=4}7DD
zixZI|juVNnkSdpo;}8-_r822lA`ywi6hD^B<4p|}a0SANM-IhJrecKxzCaKz;PbH)
z2>5-|MFK&rP%PjHg@TxwJh4n9QpqJ!r8e1|nPoB0H6`k`YJ*m-)+D7b$eC}MH#a3|
zjxo`w*P`o5$;l=Y4h?!^VghcdTqc!DmGSW^^gcdbDi%r5N1;$6mC2<diBzJHN=YzM
znNq2W#~o5A5{*V<qF%2vBpMB920u&og8A8*8S^qSGB91pNSm9My&yL?H)sC*`8l~(
zYhF%vW~MMUR;<*eWMpM!q$TMS6cSN_N}-x<%E-;hw#-YJqthg)4F;Xgn3$F}H#OCi
zOguE|khEB|xD0N)R<GA+HSbA_q?wJpPN%`Zbm)<%K{zEPISDBl<D8Q?CuvS{vNbPv
zL3VaFdTGh9WM*a0&sn^<q^Pi<AU}W6qDA=yg+;|B{lSLZi|nvDnE7?L+wrY{s7;gQ
z!7M@4s9P)%uqva!t%6-A=?5}SZY4KlogXWW0ez|P3#V(p>lVPd;W5w#^(On<&F+O1
z18)5aXG$lOM6fGvZo!<gdi1Qp4t!;v#V0`+%m@O$pLct97`)iw_#^arX65dNz-iCB
zuC}{n9lI~$<73UlMSyxVwt1_K{u;{gm(7&XpfMtj222qR76moIrZ@l|Q3D@{6j1))
zMeZ_}53wGQ&!>o#$LCQ1<Kd~2$Kz65geZ!_c>*4uEH01B1>gc#z~c(Id>#)du0X)0
zIH*8zc$lhkDJ)zLnZiW~RguLRKtT-PlaLM!Xa#yDFgYZVj){rka5$bsI%o_8zfYoL
zFp0*@I2dRFB)7;4Sq}7&YnTg9@@mAfqmM;V0zQ{Zp*JyH4sj_)fO$Qi!y`fPfrH}z
z59<`&cd}QXiI2XJc+hQ(hcAfr#1q3oU(wwAczBdRK=C~B;LZqukAd+h4$k9IWI+za
Y=kX~D7stIoA2@&o3GX3~hwH<C0E9uI=>Px#

diff --git a/tests/create-join-reference/create-join.zarr/data/3.0.0.0 b/tests/create-join-reference/create-join.zarr/data/3.0.0.0
deleted file mode 100644
index fcb3d6a1ab551766ebc135485e53a462cfbf49d8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3297
zcmY*b30PBC7Cu3W3kay-ibyNANG(Ps$;%5_AS>{)5O$3ac0QL1iiju>6_6#$Zc(YC
z)s9tM3i_G4rY@t^xX?P*(txznYHK?P&eVqKw9L7|uk+1$@1A?^f6hJkoSXA*t_$mR
zSBVb*Y2SDNU<Zr~z+j@q;cyh?JBA45<C{O^a9Hw*SY#<GqK1Yn7K(D9XrTz;z|KM8
z6oee8D8k1<loki&#2glj!|Ah#BJ<J0Vkx4sfaL=pBScx8FXanXrwVlzbm0!e<8duq
z9Ulr;8XBU|(*r`U!E9i`>=JC9ESSi_;an7z;9tT8w<>UT0nb42a)FXL(^(L-fXx<y
zKYI#6Cb+V}H4-MVCxMF_OrAa-7P5m_khM^!D+Zsz4ttxy;AimjGuRDnZEbdJZGLS;
zw)-Ks+wBGewzf6{+99)}mLPUJvA~H48T{;aKVpGOJ7H+jh9FwliQtEpL^RmZ9<>HU
z{hT8*&yLS4BUGZoj;I|yph#xgZ5|9`W<2mQ+srl_V>T1n#@M`IaFT~@RE0{^;mFLG
zJ+O1y;b0EvZ5|$GTOf=mu|tHx8D_M^86Lz9qBuaG3_=K?5l$dZ2y7$7FvJ!o+ZZ!r
zvk?`s!^vndV@&Bd7tmof0c>;y7;Y_#W7T|l<h<|mT*KoK#n+3Eao(VNx_sR)e|@<A
z*>6GWomX?N=Q?)nJH!i;OmB>Tu;JRc$RtDjjqcBW=)D;HaK+S?lt%{p`2{2WpW1gP
zKe>P7=s@$Vs{@mNo;Uf;bsgbLq&C5tQccX`qhIVEd!>1I4*!MKTITbqd24TPWMf*+
znbM36b9!RC)(PEHu8k3$qAH*MF`@flfc!<<*x|P#|0wXEb)V3yqMC(8RrkN5s>7n1
z#f9;;z;YIq)dB6!+~5>|KMXNeBdi$(WH1O6+=q56enkkGBw#dQ-wrgG@|sQkz=kS>
z@tcF-?<7QTg75p6gykV<4X{EhDv+$WduRknMqn)hZ~=WkSU~3vxUYjFBMmtVwM`KR
zOVn@s`b#6F*HuruPbW0nIvXy0+1M-TE}oRvGML|^t~|K<THUwDM@BqnZ7p1uBucw`
zF!o|$X9By}y#2c_muq64^{!ObFn53kLnae`?ANKQ{jKHp8{0DI`H~CA?=1iDxl-|d
zX32nfU&Zv@#^F7_C#`Ph_m>phy6^RJ;O?1F-TbtJ(kuM|)^B&YBx;@&ysQ23r^62m
zie*8@C*2=<*7JLJlzj3H`xso3otk*#hutp@ojaVrwz~bGvFtCOL|$Cv_0Rdq+<^T@
z-j8drRf+w2x{l8M<4k$=-QzEhYGQbBNwYmGX-(IRJGUNv{YK8OpAU@OKIhnSC|B6_
zi|TgYiq{tY{K1qN2MW^JgYhoy{hpey>V7&~W`1Zo`_VlAEu!8W{fzS;zuft_W<l8Z
zdlT<1?ezLxSh{a>a8;p8Q+aE3*S3(Tr+4Cpt5jnx(~1MaY=r|q?-&!bX7|`VH^zBA
zn5loN=?MrCt=``|x96?y&Pl&{I^I7U(7CqxY)g%3MQP>!;1^4VPLyT+zGu4Ilb474
z<v-ewKDb&R%R6zc-qy2ae_HcLg>ES|zYotm&2ka6Z{lPud!xK^vA6sBhMOm4KXk{Z
zGRvoRw(2&laq|$e>^=?YEqR0WBR^hlW#2eBk7LsGoqX2uTgbYX&3AHqGXK5Q_f#JL
z_|;!-7mk!9`~0Vmd#8RVSDU!_j3-CZb^Xl)>gWD7#}2e?-ZE`pXqxBVjglkR46kSO
zoVa~0zaw?{R$aDqVaBg}`1M|~8wCf%dk<`Iy|-f1f=xc>gmiJz(W_$rG487ppUNK|
zs}TjW3sP3ReIwzk5AJO<^=SWg>f*n0YNXp?C~(=h%RS4+eJFlw<>Q1eO1IrLfhkmi
z_3kWL!m)BylA|sGl72VH18X}`y2Id9cBsk-kKl~6yiI85?LAuQ_hCMkev!!{d1TEV
zGc=QY5J9I+V3nZ$S^wgyymqj%+l}aF0r77FZ@`ZTjwIqy2S!ON0Z|zS3a5ljVnP;x
z4Ys<H!>7EWs&Z?osc7xGg2JNB#ieUEm}@K6Z^+wFw5__L(zJ0?d2Maoj_U0d6_wj-
z>vz;wmRHv|)V<f-(zL6ou?4BAv9_hTxvAy7Rx;Yvf`jI!re;*Ml0hqhTACXhcC|F`
zsB37b-&t2z-%yXNuA#A^icocRO?7Pz^4glJs-m^|d8_ja)^Dq*+*(#@+E%f35usOy
zj|>l=JZvUBfbbtJPYJVSD`5uwdHQrLxk7RSI0T#hVK6P116|JXPv4bQs=cAFd+$F0
zG7#eYa$tF148ZN@p)l=|FS|B#&x|avhWG^j9VV}6DlhCXjYNGUeDXn9YEf?>HvzY{
zjCeZRVFFJRfCfI5;2C!-mw=!!8;jw4fCA?s<e^l_6$+V5K?o(KQX!Yg6*6S<8RUN*
zkEvmKIHin4wKP~BhBgYmKq%EpwXXIr=L^L_A>L8k1W`z^`^+Hy=V$B!4qL)<7tE)<
zr_7BW>ooyntCON<O^;Q?rHT|%MO0LjCOT5|jzY?)7{T?f>x;v2^;=TWJef?UywrF8
z<LQfDoIcYPA{L3mMlO7E;d~UMQEN1jswkBzGD^#6)l9Tjqtzuw%jHTEs4j{Gi?329
zm7)`cQjP*52q}~pHg?$LBC!~KBA19oBJ9XmA`%J3;)n=oxFlSHLy8Wg`23KG0-hj@
zed&ZS7<*cP#HMLJ=>`5QiAX366AA@FBq~fMlZchFhzL0oZ%9iw8j|DnS|%z+6CD*D
zyDTd!J#~3<Qk*_Uuh*&7Y9>BDJ}y2kR<G0Q^eQFBD2>3WDitQ8P^ls?3b6>COCqGQ
z2ywVHLM}x~icFzYMJlAY`7trN7`=|nN5W34U7nVinUR*de0fSrs?nHgNM4$rm6My3
zotcr5nVplHo1KwnBtDhe#FVtOl;l`Vq%2$<rIIV7<5RM;(^Cuyv1%qNTCdZnbumkq
zB_|~&#K-8gF?y{=qsFbr?Z*Yxj9RN@nCR$eyfiQftC6emw;`QIqt&7{u~jD~Cd46O
zrm;?ni_gi;&P*p{OijfQ)6%o@^7GcLUbSk~N<ynvt$ufnZCz713nn-ukAi%yJ&&+Z
z&oxfU3{qCYJ&Pny$JPg2SqXn_ClwsXezAS01ibH`8UScPfSz)wsIMA;H5dw$SpJ_`
z-o8+cN7S;}ZBI?2^^K+pftzPDUFj3g>Z^{pbM==Yu>T}Lm8q{s172SM9P4*`9Z#Oh
z>PwR$<Yx8FyK}wIlhZH&eIL1brOLYwU^5i50Om+puyeME1#`qKSTV5<-Vx#T)k^dE
zXo|!K-faAqX+8)>$27&K1i<I<DGZYsP$;4?pC<r{FW}P@&8H!h;!zX`1QeG-i6@{1
zSZ-Vj8BTI~qIfh=p`kdWIKTnk=z0{NOLNh!^IM+7B^P842QSE+|GOY_MlZ<R@!&d(
zT%1$AY%tYp{+OL8*AF&PBn}Qu@whxbhsVVrvDEo|42aLiV0aXtq{AWkU|b}fus_qG
z>Ctp(UKsFcOb0`zfrCj?p(GvjfSJ-^Bpp;@Ivh-gi}_;u7&y)0^SL}^G_HmVQ37<2
OA@D}Gip#}3dGKGSt-Hnm

diff --git a/tests/create-join-reference/create-join.zarr/data/4.0.0.0 b/tests/create-join-reference/create-join.zarr/data/4.0.0.0
deleted file mode 100644
index 728758fd9d1f0595f4b7f2dc8621db47fa859b42..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3385
zcmYLL30PCd7Cs4zf+8rbB8V)aBBDe~xLL?f79cwy;KC9Cv4BPd;#z_t60js7P@ffr
zxV5M?xCJZPSnB??YM1BJ)`!^VhJHkAO>4Ehb5p<Xoy^=h%YV*2XEHN)sFbi_{PzIJ
z&h!Dm0Zvc>o-3?6Yd`kh-d+(>udY|53lAqb9Q$=TQLi;T+-kMz!gc)!49B4#d$<*C
z*u(o#L~S@KaUgBzh0WUEZ?#SVJv!;|Hy-?~G4RK9omHnpJigG{kEnjck^pi<ll#47
zBKRVd$oG9u(r^d{3IwmmeuM%(bQGr`D}~AH<G`~PhEZWm1o%>6eTanuk<p={90&~c
zhjkY%7E7qbG7UVb;2i_t2b6K(F#@Pw(B*LO9eh6DfulEoFOV`xsC78T03`rC{J{e~
z#sJWPG7%`A0Fy{pDr)3F5dk1i1&TL#jD;|xaTY)ViJ}NdAVmg%VMO38ayXner&F(|
zA?ck?8VWWivNY5=@oBTsXg0mxrl;xAghaEE7(EVpJ&8gEvdAD1VWa{@jDh~hqJU4E
z+eR8mJ1Wq$5}s2Lh{4c`0ZA=Q15ObQ@rbiIZK$%*oGydQZm=6%gv(%e+3g0xZZOyh
zYy`4)7eQK(2!gO9)b+O>8N?w^%5%Yi&ZG*>cGvqb#35tY?dV}AfvCrTE;oQcAcjIu
zWDEpak;uRX425vF)rDyw9K8@X(c`0$R8Zr(1z0o;;PodP2e>;w5eB|8zYV!@aCcqD
z_9GJ}++x02{_Dp#BNJYyPk(Pe-xMG9oktt<$=FuYzM5$T7ifE8SNr_fUcS9_;Mv-j
zr<%(Leb}=c;&oO3n%}fv`st`I#C_KDQscdk3N4RQw^sHZeRB3-#I0@Gfq8wQUG?V9
z#ilneI=g#Pk`8q68;fVwJ>BIsQu@G@b0_N6m&$){zr?xoJa+SqarS*aac3V$uaop{
z?FQl6HUEF@nsMz0#er8nP^jP=0pO22)4zKpI7dKEySqqBGy`nNsezCCHW)2N2t^i2
z4GyrRwnHD^h$eXHIbg|Y4*;-Pz<9#ZM?&&IuSFNWALP(b%mE`{GdlPrqapXh`^Hi5
za%gDN=b^2aF7&+Qj2iM0ZM-^Fx9YLcQ+sC~<ywdLsUN$Z1f4NF@p)BDo&G_Xe#PMW
zO5VVuvJ}?+X9t#M=|7)3HgoblP3Xryt5!s|Zm7(@U3=6pHt4G}pB>o#Fn?DWr)d3&
zXE#rE>>t$cx#9YnVV0;TnPL>Pm6TgAE){<z_AmKmR!o`K?Tf#NPDn=vy{RjlcJ@Z*
zuV>XA&jt56)z_9jSFXRG|2TY=elpeP_w&U&KQLZ!=AD#H@s0Un^7AE24g^kVYVgP(
zSX?k;v@fM8_T|r)_qXg)?i^96JABR_bG3gLfAqSltZCe_HC;swv9YaR{{{<}S1ujm
z&F?)lyM6reV^`ZwLB)9L(8?>XrP{8vj^J>UNA%Bzd*UB|nBlc!jaoVNo7h0zn{%SQ
zVKL`-bXBzY)50@qh-Z7wWxVK_a;L5E<m|sj4du=tTtlUEZKbo{Zu!+m_BiO<&l>_o
z!rw>mDAAHXU8aMBx5^)$Kjw^k6u-MQgT1O__4L!<KfU+GJ!!W}?aD3+n{-}xJa}+-
zh*O+9=i4zS2Z>ZeTSfJ}kfx*+!Ch;Tn)QtvHLk;ZZ+>^p_ejiV%Y1#Z@`p0dSK5LG
zzxlzFn;Ko;HhgUE^_@Yiim8z+Ui?Kw-Otz(f9X}jc<ZYIMac1L)4eMvW-Vxaw%qw=
z)~!*8<Rz;b%17D9HWfwg&L~;@pf{*z+u^8I&6OFohiP*z^p~$%nZ4om+DR#{e|CAY
z`z}rVEpcr}+p}q`6N#@iOE3CGOc0okT;1L|@MetJ|LSiK)80~-)=oTrcTd~Vo03yi
zYf>qFtHQH>K6m%vw=_dw=aH8GD#jf(^dFYVUOKo>f-k=P^6qWJ_o;{UR~jo8to={o
z=1!0M8b2^z*W*$J_>}7Jw~rcXbWlBV!HDa6STJ<u@Uy@ljkqQJAQwVI`9{kqIQta0
z!Xw*t&>J1VgXa-`xXcd%w6}w?jPDNThdbn$J1#=O5t@StxQ?6?a?puy4Do<E58Q_B
zMkB)dutUu-fIqfv+ElxFQ<bs2d}Ue1nsw_oR8(xKuhB1CzEZ!zRI_R0nzftj>bGpE
zsxfZZxVhfkV6LmJZE7?(nOn9tS++K}v@~yRY9OC2%_wYbX>CQJd24fXYfDp8E3(^A
zLE0P9+R)J0XlXFx_sU|%XJcbyt(#1Bb@e8sdQ)xf%4J2xAC{DsSJ!N+t}<3_sIHnr
z(z_ET0Z$k`b|aoL_;u7vAvmE3CW8BjF<?<xGt30*>dQDsB!x|Y7ulZ8+`De+THDTP
zt8r@XQhETsL{(9NYS~-#$+H}Q{qSPQG<)Pv`%WGn{6G(TO)%o8&y1%J?Kq_@{l~uR
zj%X#AK)OPWvkH;bj)2n^fL8oQ>jxlw=f{u&Al75ZvNDJT@b<!JLmeQINyOp=u|)EB
zlSrh}1WAGfRY($4i^Wn&0*5P+N+hDW5&_x}%42Z3>He25ganEsqN!SH0y~=aKE(1K
zbe|nJo<7SrFgAoa+;4Vu;Arn>1#7kPAn#K7qI98{;3g#|D&>jnXbD##mqZWt43~=e
zBC&)T<rSYG7M?wG@`ymdKXdH(Q9g&ui|;&nqFb&|Xq1{{S)w#iCQ~Ys6BWrCm0F!4
z7fX`Fm}#;~M&^nM<Ao^!OTh|6B2=J8idmCJlDIssyYcvZ9+$`Gxrxi=@C5>)h%XQd
z_&gzl!C^3&bNtvud_4W!vAHveNH&|rVzXzlSxk038-;0nE{DU37sitVhY>H}3S?rT
zP@+g%q|@ar%uLg$<Vjk!JSiz<L4LvFtc98BDO!zIt5GQA^3=4n)YOzzty-nl$fP*S
zB7smSmPzqXkjP}1f`G^4#B&5#h>(X*u}CPwVuT`zM3yKKB}gSIl}e*ktCebvMw^0R
z)T)I!i}Uhy*^3rsWo75&WG|dQe{p`{(xprC^YZfYmlPHj<mqy`9IimB%E-#iU6h%k
zOcV?G@<fR=d0tilX1HLURv}N8r)ZRNm1h3@%#8H3G>uxVO;IY93b{h5!lqK8pi(8v
z<;ltThoBZsD3QEe?zSNW^$Nwj^m!?%NNK4lDcF!NC|rU$<l^*Tg1Oih>sRQPmlPMd
zskpefq-1#lH$Yw^sZrP5>h4?}2An>>mdD+Tj>!R<Wc6@s!o}+PZtgMak$|?Tk#lf`
zj~RVA+rydORGNR1`3SGl?VeBwLo=rrB`$=XB_p!y;Be0dznBR|^W7c83%>SFFX`Ky
z3J)PEtdbAK0a50tfYVWUM8StZ3s}<Wd$GT~IxO=k|5fNlJgOi9H%ckJ8XR@VIM-1C
zLPQj32<B2Cghzo7eax_z3lK|3Gr(}LvY2sh7Fhncp&4;ZmRn@9SOgodu1ppK02?q#
zLN9{OLYc*4GFS{I!C){5V55aa#4?!-Cb9%UXRv_@YzDf;q8}R2kBPrH1~8b|Q6exT
zQ3li;@(vslN8Y>XbPj~j@su7N0eD#(9RuU&0`k5+!i&6}=Z<b82T$J_9t}8JoLgD=
zT1+C2h>Ii0u=F?<rb`eEch<-;*>OxP0*S?nC*rXj#J?+tbjHdVbTU^Ko5et10!tw1
vWI1d`ELkBN<1yZqLsr2g5opB%=xjR1Cno_FSTuvd0*pu2g&3STWa9n@Nz~gf

diff --git a/tests/create-join-reference/create-join.zarr/data/5.0.0.0 b/tests/create-join-reference/create-join.zarr/data/5.0.0.0
deleted file mode 100644
index 42524d3b2656f4fb3835a499b7e27d0af99e6cef..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3366
zcmYio3tZFX_H2ma6a_)1Q(y`{Fkc{h!`Rr9Z49;nlPT|qL>ic(B9aT_HNp)AhBJvW
z%hU=LHI;IS{<Nj}$S$zVhMHHew=Sq@*1dAihxgI>{@;0j=YP)s|GlV`Sz|<j0Hn|L
z1waB_RDj_ox5?yon4-J8qp^2)JEFTCCR2AbHn)S}M#6BI+-?&pO(urh?Ql3u4m6k;
z4u*+=79>a=3?eazk!aiu;=yo}QOFU01UIp{k#(T<1vuQJF-bHx0|f>H-%Orp2ZP}b
zfkp1?6bJ_FmDmqcz?X$$7_tfosbqt%6+*6u!(=K{h7M6+-pd}3XUH=o1_%J~1L|1t
zp~AF{zA%;wL7oLbrGeL2@J@i?2_Db#<^7}2asvJR_+ZCI1E2!c2bjzIBf!fCs1v~p
z!*NJB3KJ-(nF17yJ_#sufJz~}9z%6?%?C&U7z0%Nw2=UzWCV^I^6#h9`|L)#u@4`k
zv5#~*-QGth*lwrOX_%0)&ps;H31g(w(Z_B_S37+&P$J-=unz~J(P%`BS|eig#2~0Z
zJz<P?42DBVLya9r8_}PRED42UP(>I!ej+10%A?!+>=qTxqH?(`7S#)m=2BTKG?mJN
zI+u#(qFHD(6%BRxpb7;SX-H9~5#n;8y9@1z(7}QdjfP{95DjreJz7;3T0RU>i9j6C
zs-js`Dwj8mj7NVAhE6WxO02ZeQ8T>E`1g`=#sXxH0eIHVz89-sduesq<7q<=-b?FU
zdg!v>qZyy;9xpoZ!SYRq??$YBvUuIf*&7P3JpF2t|3k^$hQgVGET4A=Z=N<)Oqz22
zBT7kX_0c1#gJ&YjB{|RUW@fy8yAJk+?_}=y<M`Q2e$|0R6)bkb{2yz=_sc6bp6Y()
z>pJyO)bl1yK<$x7U)CF*w}0ix3p;7M%ANkhq8{gh>)(&`%Yh*hH6OQ)PuY`KXSzbb
z`^C(ifiu_o|CwvWnLC3G;~RlOg-PMS=nsHli4EYQXS77~%%W1l^e_?m5Y7zX!A`)L
z4dy~!XQvrN-~^{xR0^B>!OsRBIMZeJU=&pQEhVnmkVE2RLlBvt5)3064-=>`JTlTe
z=JJDpeW$kw{{g<s)-BfPQ>R<iy`O(QM{rxgIaP4^+rREz()p|G(H~#tX0iItj|c;%
zTkn=6Sbz09#Gn3saQyvy@!$3S9<J^fjz|mIcg25J+~qee*KB`Q&Ql*+)3UPqa?dUE
z@as1#st!EkAIeU>`Rz-Qp@I2rQ@e^wdqma;kHYHSI&iu7>v@^4WG<Py?#m}P?%!Cp
z%3JyEj{_6-&x<P2jVwEQCeOnBallg0Rr`dky(28$)0XOt`_O*SHKPG)^^ICut#z+W
zBwf?=uxr;f+tJyT4Ug-*dp@1>%O4pTkABn7D#(#)PnK4_Q=K`1aVWDxZCD)nMP;~r
z;L+OP$rE=CNw?nnDf5IxWa4FgcWv!c_E#QTtl{j<BA?%T_6ZKQJGOsx`czTjd|`E0
z0juey?+(s>e86w$uyWOziEsb5-u{>K35kNe_mAux*zn|sYrV{X?vK-57jCz{+h6#8
z%NVa6ORuOyALR63>woju<k_4Lij7R?!N5;?LVqg0daCbQX2QyoJOTTc*v>xhNz>y}
zYj(HnnfmGnQNi7}=11<`_cxxl{O1ns1%qlU_ch<F#_&9I!<{`(ye|IoL;ipz<osDf
z*{%=GnrdNq%+*QOkY|_HHP@!h{NnKKO;el8C{LpeTlR*<Bt_|)ZtNYb>IL!6hd(XN
zoEcY9`MK5=*!-KJrtIp1WAn4d|2_O0b!*zgvZRZ<wd&;G7v>bN9~k7vEPubGWwAE$
zsP%Ap)YESk(4%g7Rox!wqfP4Q^e^1EEWNX%E2|q0+TJ*O;>fX(qsP1rjCmc#f!)t~
z#5?Cq&YH5MJ<xM**T}q`Cdrh_TmRS{z?w7ev(1W)!mF=czxPFD|CZh+`!VlL7hNrN
z<G45}%}z4EN0+w~89%er`7zb6b1amaaW><IIstz5>11|w4WZWsC!*2Wjf|72^BM{=
zXPRwHGseMK9$X#@oB0Y8ZBu4~xeLAV%zNK20bspXX*LQ=f$0m*A~PO;;PiCb%;c1y
z0#dS6P`hOe*<osH>g#IFZ){j!R$gAdsj_-=#aouT*VmM+TT|6gTT{8dveweLZCgWK
zP4(8g#wP2wnwolRv$btU>kj*lmR5uv+Z$S2TDG^^+uK@OTH6q{wCreUX=~k1%q{J$
zZG>!ZMr5^aYqnXfHtQ(1HQSo&2{bfV8X6lA8XM~CH?A!$eRb`+vg)nX)n>D~s%C2>
zf&c6<Qruxmv72$1!LzYc0m0H&U^@JFiwWz@2n7Ey9~jR0CxA5=>jXGgpB~bLG-VE+
zQTQj|?cA!x%=7pe%%8wH@pSQ(cl+08Jl^Z9*?wnrw@ug}p3>0KS>r#rV9C@c(@rzL
zhV~4ohi-DtZm{F{<4)&N*ah$&`L=L!MLEDXP(}{mp^=f15T!z)kV)lonfyhUD-=?h
zR3?+l5KtwPDa2BlSgKIS<q|Pnrie#JiF{EKE9d#03-ntGzl9SL!x*uV;ezpDDMxn=
z2g96*sC#qg#!L)NH3U+}!`$*jHD_#1-ioXkwS=iws}fZb;X;K<sZmBwIQQ8Mr9>i=
z(<RaOrE=K<|Dy}UVsZSZM~)v7v$&$*W2cU*HJW6tL7SjZD3zMTBy~c9GAT)`Gh)gr
z%v`2YD=}XLsRSFALQ0UJh;_&@>roqu!njcoiNvE+C=|wv#ll!2d7nfqLJ}v6jfvs#
zcu@=iCnhHPU)_wCIDUKqPaqJ42>3ifj6lGT3K7P}#>B=5g>gubjT6NyWfF-zF*PGM
zFDGMpicYIh>60}om3~D*VSZLddYVD6)9ZEGM2$KnHPv7+r09~9bULMiq%4k)$1Ig{
znIb_ci5H3DNLaB%B8`tjER&*HB8ivC6$uG)sZ1eH)FvkDl9Q8?bvnI4mxSDk%-sC^
z-0aNEjEt=8?5vCxro6(Ji;G__EXdC<SXETCsvs{nP8b)Tpi9rn&CN_VBr2uxqC}NU
zp-#&z#0*!a>NOg5qCuai(&|j6^mJotD!xE(z|X8vYqi=$Z4$BsYt#rD1QgJwNyIiP
z5szf_!Sqtn(o#~e(6m%TN{T_RPcak}6&2*><mBe&WM}7K&(19=EiGBI`W1xLqp;>x
z^U`&L^QOHs(+o?r9cK)Z3!?oGvNoIBkHO;8H=y3E>;khb8s{@WknJ^n8`6@kxI{{j
zc^j%O@xI61%C8&7sgte@_Gauq;WuN`9?#Fuqi%_(<Q|Fmxbu_8Ss4|4<_QMgQ{Rc5
z-wFH75GZ=fo(xd9nGs++J9yS7+v&9N{Tr*?dOp7DS|`TteDnE2xC2lMWfZ(RN+_@`
zSV)1;I0_WwHFuE^Z^2=FZUFcT$U^1;@UbKC_y|Y^JY2<u;qW+I9@^M!4sdx~u0X)!
zaXDNr7iS#G0zO*VNCil_92O5F02>?LRJkN1FPaP--4s}eNm$N{g&fKzcWf4m#b&Yp
zbjQZ)w_gzJPq*uFv&fY^V{!!qO^)!|hP-Vk2$SOSIb05#&F6726&{Pv<?~o<F7O0g
zz&!ZC;`dR9im?hlUqC8}`CmymaI_>22g@cU5lqs@I6Q6)iNxmdVz4AWm&@X_c>+Eg
jW8%Z*VP$*{pUXxYkIUwBkf1-8l!Yc1i;F?Y=i>hx<W!`v

diff --git a/tests/create-join-reference/create-join.zarr/data/6.0.0.0 b/tests/create-join-reference/create-join.zarr/data/6.0.0.0
deleted file mode 100644
index f0e6ab1ed46a75ebb89c1161708335ada7a72405..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3399
zcmYjT2~?BE7QP8V5EnK@kzJ_^pn^6#OGy4C5Rwp9l_&%eMB;+h6}O;~uqPl<3bjRA
zwTfETN^9}On!1(R9C0n^!{XM)BA~uk+hcuqQqSqT`RBj)e)HWscjnAL89FV5As7Qd
znsgKZD!5Ds7+h_o^j1CP;^Ja37+egf8G1^O%EiF2>KP1!fzne9iowt$&5FVx8eMD*
z%1T)Y3zZe~5QMN%^m;3rr0Mk}Awn2xC9sv4Rx}uBIg&@iYY)sqQOLpyNF4@@BoA_L
z5OIb8ph3W9Y$s?iiiz$dG^G%@nFX$uK&Qj#P;jTi=AcFzOj!~c8QB=w7!TkLt{#w!
zHUi$@A$Nw6bQl%{3=TNCf_J10UU?wdVi5=g<YjUEh)EEKxIwOk_?Ag<9sy#ZE3k|N
z+BBfM0_{C;b^%}meLT>E!OKDeMlh_h=yZhb4>VloXrNUAxR5#r6ePI0yPG^H-G2CV
z`=O|t)oQh$A3;g4A9kpl{Sd^4y-4(HZgx=B-H0}mhM<Fw4Ako8sW9M!By{{R7<p)s
zc;pdAH)$lQy9k1tyA?1{f(`=R$neATn6()he%)<$yH=|Ww}+#)x1p&GZwp7a&8`jC
z+U=y-?QQ6_p^Z3px3z^kAVk^_#t=eiLyUxK?cpdGjh@!7wUZQDj3JrB5!5b$0XhcR
z3876JPONZy5WIEOYB3!WFaSqD7%7<7->KA@4l3Nq00j(y*NyuxM6}*-Y!J@=Z}0S)
ztJhfHjEWW&7Tv30O_O$}T~F@sWM9>U%r5`AzE|*~Cdqg>bNb2#&+Ojv3ujhG^{#l<
z64jIcp!2rZxnIxr^f{NEc<xm*%BsG9DTlG@?cs_wKC7!&U33ddGQuI9RK0TZY{rPk
zZN?^J$Nl#Lz7HEe*i?UF;TQHj4@!<?30J&1>oR=8XIlq3`&#R#|Dp-$zZ3cW)0{3F
zk$0Qf4>z;%pUrH<&Fsg5;Z=?e6AB&{XTbSQnyT{vFM&4<xCnYd=a?--Zn}k5C>%o9
z9|ni%z@`&8x{i@|u*PCL)II^%Gk5|7!zn~C)H*!mW$;+cgG+Xq(C%RC)e#F12ABZF
z1E2xW^^SnibQpU3cJKL~34P-2fd_lP`Hk=Ee{}Kl(4~b<!$<m89xwjRHcm6<<^7T_
zuUAohpI`Z_=)@Rz)3GZT{e=M$kKH|2{urRm+<a4DYpp&i_Ib4Yi1<*~Cv68tXD-O-
zJdxBfvbIBbHS?f>xw!v_h#fmeOzIvzt8@J?5npikNB*1Ba%%?FHSgffz$5oo?YnRO
z$lcdz{HGtQ*UPV1GS<hR&2$eKE4}TvyfQE3_>(^ZvqiR_c9(?aeSYHy#`UIaFHgR&
zkNWm%`^@<V;(CHF@AP~e?h~C-<SF<{mtXQc-@S8(>_};J*OhH&TYhc7n;*Gx$=$-t
zW^d<quLfDJZ>C}TGV_CVxt~~-LV?>kg{<Xa`|p7Rce47HM6W#h$<axvwU<rQ!Rs?s
zgOD(n-&y4)UlnlOEul+(BU&*3{7KP;_kVlS&%4)rI=5%>^2WlxS(5%&z0cX|VZZy-
zExoAiO*MIaH=+2&@0Ayg2S(qft@`@M9NV)G|0>q_C*S=kX8xL{PUX2<y}n)}j%Vy&
ze=fgt;&{KlON{V^X={T<j@`ZPUyqIF%d2Is9d~b~eqH!@d0S1AulY%SPGDVO`Noht
zubTEg-T&j9{}i1v2lg29XYTFhPoCHHdf%Kq?#7fyZ#Jy^?8u(p1uuPnp@sGzSohP2
z%yOr)o~|IvpAmB^zuNuK$+Z2<y{suW_fC{;vc0Tm8Sb)S^%dLkf~Q~cj8owJGi~~s
zG^l><)5Y3oje4d(x#sq*qch@s-<mq6*IpQ1QASgSzkVYr*pvGF&Jd_y$X=}O9p`h$
z=2~^uJy+rrA9igQtuNo)xN=@v+|Pe@j2YMd=~J!sb;p%IHa_Z<M1=BU?N6$IyDzIh
zd+23$q5teP1J(VHb`_rc7WbzGcP5b*#~W0FU>$DNza0x`B6Xk{2|F#Otq8e+={h}4
zXA*#JkUlsTSsEzT*-Q?$KnE7E;TM74=Q-mY-wWu(Pz<^jgzz&Oq@$jS$!!Se2GEK0
z1{XYObZEj>203ZU%E}F8rCZjmUB6|^hLSB?KhRZG8rCe)eo(ZnqO43;QnI6>x~jTN
zuhVTWtE#OvR#sHi)EVn)%{%LjJIyGl%?%Cp=A8}ZT?p(lqepCWeLb-o3CG;9vu?M!
z!Dy<judA&!8S6|&qsdfnsw;O$MMXtrCBCXEE6P_dEn2Q!wsP(E9flpHrQ1sN+oFiP
zJ8399X^OGw@U+3d&0;A8P+IT>$5FGc0pPA1KsdDCm~SRi1h_N~2Gy!!c;fK*J-S3E
zz`xwU8X!vtzSlld-{zqa{2|N+n7At5k~z8WdT{F^Rq)GiVh@8Z!xQEov$vj>O*^v(
z4g!d$UItUtUjVc-t^mXE)H-Z~4)_?jEt|H;SCBvDA&+dSBta^cNW>@-)DnqQN}+)+
zB@#<~VW^&>D4|q}J@H(LI6hh;;fkV?!m|1wd<or&<D)nUys!jTM98ENX=T?X@7cp5
zVrWyMgySSBE9cl;|9G68B~#AYut%MfLdkfdq{Kw2RKOM|s$vpnd_V5=MNy(qB$l3!
z^cRXH)T|S+0)bHAdGgEC$K#@=3r?IpC6}iptJ0Gb5+u?Dr6O5|;VPvfb*_Yx$i$Rb
znvf_x00VDtNkx>1qQnxZgcL4D(_xc&p-^#gv5ucWfL3f=TuiJGtr&qYK3+%(1@S_G
zKorB`vw6Iz06r(0&u3mAie!iJ`D{L)KatPlqT=&rPl%0)iAIS<iH?ck#|ngr5{i;4
zGqV>i%3UxwU8R&KrKTukvec|a1^Jrnd6}uHDflQA@}#u%jP$hB^pxb}l+?rovcT~|
zA%*jjh^2`MA}j$bijEQBJcM!aBAk#=C=!Xt`Xu5gVyRRvS0t-cs$^A)isV6+wIDYy
zZ=oh9J9|NnMw7E(zIsu?k|OPr!h%JM@(T(J7Z)tb!z+;p>(S&b%$b*_NDzzS6-g3l
zQf5xU;(X2gj1;*%Ns*qakSSDIS@Y)2&B(yascFedWs*#xR4SCo$++T*<m99zWG2ZK
zazt^O#KVFVgg``zxFRhhGd&$Anwg%OmVqrjEe}Uon5)Ts*EE{k6~)CXmMz6Qh*rDQ
zA<Nt*hg6RI(u31$`@!=-ab=AK9yP^VaF+vX(5?DmnNQKsR&df;=mUi_9#m|{<A4TX
zMu5rW!!Yitc_>38umv~(Q(jaI)0f}tww$`t8r3p+E&~sha9De@>5D5RVeKsV86Zcl
z14bNvNlxv-0JQ+e#?X5Ad3Osc44aM=4?jIuqXAfL!IXFw0hHp&<hGUu5G0~OZ9psy
zg5qdUG|C94Vp;G$o9iGtLo1K_w+S3?5TVT-4nry)A3;9wu{1Ukxm+%Xi#Bk$q*^W)
zcNUM$VY1m=K8wrd0Sf~;WUp~KY&KvlkHrEeA7i;37QXOdb1)UAXR^86Vi-Y(nH(0m
zfHRp)7LyhK?gCzfHUfAV|NqN3QwB~h<kFt!&8#7Ml>0QNTGXb&P!0#n;c(b27AiKE
z!vr3dhXXRPW-O4yLyyPA;+fbH&1FJ#n3D%5%*lrr<^(QIL!Mk7kB=D~vtjetd|+V&
rhac^j4F|YoP6HuaTnfzf_gwg7NFJ9A$i)sW8)4E##$b}kvbp~Q(Z{{D

diff --git a/tests/create-join-reference/create-join.zarr/data/7.0.0.0 b/tests/create-join-reference/create-join.zarr/data/7.0.0.0
deleted file mode 100644
index 0947f368fbcc7e6333131282de07f7922ac490ce..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3386
zcmY*b2~?BE7QRVjaY0bgQjtXss0bRwtUw4^fh2^4ut}7FAt;N2wk}v~Fd|@NHNoXs
zAKDasR;{&Gebxm{TeU@P)mPC{<<OAY9%}*XJ7TrGJJEC6cmIE8?)~nPxsyBd50x@~
zl)xW=G@cIt4)75bV32M!hIhC+x)fbqUB<311&*$c4rIC%#&DMlnQ)X{u5co`j7FEy
z6+Q+ABM=F9yArL@MJ(J}V~4Q=JxFIXcHoFg^0-F=#6qa6BM=5XjYgLX0Sw!LF!C5#
zVS!<)LJ@An&<GgAZxix>AfP}{6}ICP@L{4j6ImGqSFvEU1t?S)PX}KrRE6xMz^vsA
z24g2<XClBPn6;BJ5ds)KFrEq%7}3O(HU~y|!KkTF!C>_0dU|?1V3kgXoUX?cJSjjK
z2QV2R3^WMobfW>r0Qdl94uBU>1TdpVHx?+KKnVa!3{X7)yn!|wfC_%6Jzzr54Dut8
z0@MVcq2~e^$m-G2>2#;TX*bZFWWeTh+Ua(b?FIsnfi$PxZpVhkbb|qT0@LkIy4wOJ
zWb6hzF*9I8Bd6W1GT1S^0W@|*oC+B7#TZTlosKxh!A7UMoiG4e(1QWuVK5gVRMO36
zhe#x%IYcy(!{P4DX0ymaqlsu{qA=6U2qWTf&>Rl8u|woAi)cvbgElmj@WI}J=C2xR
zP(c71%^U@TQypfsB>~XEf#&E%LmPyM8Tw)X2hD88wY-69nj1$&0Ig~?71FR^0dhwH
z4DalnwEw=TRB$+GntJX>?ORr@eRz%5Jufx;!rAwqvmXr)SPIuXueoFG&OCRs>_J_<
zNBPC>yZ;GZ*}@u#%`T|UZS=QPK54zS?)tbENckkvBV1H*zkz-wbe!*UzvR!i>8b{M
zTlE#HqQ~W?ZLx=j0%piJMi$#nHH1A@|N7*s!9~|BxwR9!hRSFQ&XwHh-E9xP+5heL
zci-9tme02|57a-s-Qf2Pq1R;_h-KUKSJ^gU*#`ctY%emP*_|5*0D@t{fG21GOy3Ab
zgP#VAiwYuBJDA4ep9;{4I~dy0+++gp>{nwubeg*Nv<qecjSA_{2sj?fJ42BXnD_v(
zWM@zYMn){N4nJA#dui_%zY5L|JyNZzQO{myHs^IutW4^AI{$25hS$iwGpY+~bk#>K
zAOE+quGSSOe*Z_P>fU55@p&1uL}|YC4)2!b#UC-kb{pl>cPm=*lvPspJ5zSAKl4-+
zkYfwv=;p-xU)yv^`(cMB{G!Kal#I-KZ3^be@heKasW$cZ4N)CwlZV28$@Ogf?~SPH
zsGLuj31fOM*PP1ARc!Ei`-go&57fC<S)~IOpL!?kVZ`~<YjeM4tyF&?Ej_b(Z@<v5
zL>~9Zt1D8ZI{9@}Rf=U-o{!%Uv#nSdGGTgQ#9?-=>gCarujCOnXUN7IrT&82-Ic$d
z-1D_@cT#8k#~n+qzD=LoQE>Zgz+=z;n?FB|Tp$X$A~h_ixSH_C0jprT>)$`G34Z>9
zF-MU+#=myS+_%&GetIy0Ii|Yf@#UC|p}}H)WN%5=gL!iU-f&7M&Of^TjBwXT*6c$8
zCzR3e91`x^^JVjo*<8lfyDW`T`A1C07x`6@j&mo)+qlyXd^+#Dp*x-HkINpO9M|Xy
z^hvBMx?bgY@!Mys$Eo!ndq)^M{j^0x>}T`N&7b?juJwKI-Rt-ElAU=GcYDV4us>~g
zgimQJ{RJ&5L)6BxSHvxaOOGi>)3Y`$ywhgAtKa*tXQ2lM3tCSHm^AS{mD<9dDWQ{x
zcU-#89(-^)%wy!xvfVF>Z0V89D<?mj_;PpDmHitgOXK49#U6Nal(FpCSMPOOVirCO
z`KQV`=B?|jtGA-R%*r{k%S(MeV^((C@F(IhuRF!LIls4zit3}UZJVI(q*|9ouUchX
zaCUia-;ELO2koUtj_jMDuF2@HS^LZLtAoz*eM@SKSHG`vz0s^+{EhK~_|%6&X|?x-
z1lD<3G8QI4d#`&4B~^cd8Y(D1P95;l>j9DgLK(Ot{D{6?Uk;u9RKGPLWFrg&_UcLH
zgGoTP0`|v%1EFXK(+nm(;=nK+1I@uK69myF^{%0X>P-@WMYMq_#L0vK1<ybr=tWBu
z{!Z;gs2fF&mzvtzEw$#Vwd*##Yg%7fS+(k2%hoOHi&s=EH`iC2H?H2WdF!@@#`-N=
z%r&)^hQ{W)8cSnylg-+;eaCjI4QYF8oy}@(ZQHr4t<Bbk&1$uxrp>k;o6Wj|*x0PC
zEjw)1rsk%mmL}|5nwuM&np>M%>Il^%SxC2aYhB&Cit_TZiq&h(n>TN&HdR+|-W*Nn
z_3<Lc<E0#%36B{3DwfM2NL&JMg8Q?$maK^hU@q)0ydOYP1n7GMLYGE(!jnZP3=8-9
z)e)4Z2OwvMq`tK;v<G19K^N43|4h!!?0-O6_pN@Ix^@=#&)Ld2bNRF!<>t%${Z>7!
zT!k+6p#Zd7xVC+85JedEo&)sN2&RDdJv^<xgEH{Me{W=DBv^(dkxC?_iODFFx<!dp
zBt}q*l2jrR3(!m|Nlb`J6!OJVv6!b=s2aYzm-0X}lPOQ0!i|`}AS^OM@$H%6@EN~P
zT{vE`FgAu2UAoNq1jYmuDCF}>;}q#C$#n5-NlLOpE>-#`NMqu~{Z~$&@RTJB#WME&
zc_N84ap5sRf<PclJpSp4<0rWD!$hY~B&VuXT5XafNhVXKtCY&rWLa{mQj;weOHw50
zlawUG8NJFtBo>qTip3(7351iD5FPgX1U{dikbp0CL`V<_1$@3BF)>jnOcV+d`2u0Q
zXAqCan-{?2#KrR1Q-`A2vlqn1a(J;!gkzC->?!ea@q|b$!tw<P$ug-lS(UkDsUd%H
zwl+<vkZaURxm>fzU@XjEl9QFLQEN2nRD{yC8CtDIt4>pC(vxIlp7`4$i7Z(vmZc<%
zgk)|QR3MUwg#w{SBoPTwFLIkCVUoB6a)nBjrcT3GldeHdovYIqEG@{_=Pg;Bm!F@v
zc(JaqXn9FV@v@>qL!q%~S&^~OfGJ88T3x<jY2KoAWs*dcsFX`(iY$FmQDMF=Q>{?Q
zm0FD=S(&zIQBF>FMn;-SrO~FPDpOLFsj2v?kXNcy3VDiLu0T?zs#2ANK~bqdjS4*w
zN>wS<+RO|sCYqU%o}QuAq-)a)3X6<O^Yim@lKBM%dc8h>#me%PD@sd{N=wViO0j!5
z$=9>FZdtQmyXDAKwjHasxN3v}@CfX`b%GE&=cpG0L?a$CP884p+3+LqXS+`)kOYl?
znE5@{`@uk{fGl0PdCvn#DY`?^&8)xhXRtm;5Y{!#u&Ix>egEN=aMp%Flp6uGw?J^f
z`7jC3JN?!-ecC^Fax!2Ge};F~#c&zglwt!o1E6<=0_@mK0SFOMpfQL~fsh0W6#F#6
z3H+Wf#D|av?B}t0ZWg$cU=r6C29&_zaXD-P#<2kK1umD%=5hcyTsCl6sN|v(m(Ae-
zRvPj|%j06LaoKDRS`ioYi{$_l1Ca<Ag2QBSxn+>gLloGn0WV}BEOOOmGMOwU>(y18
z#l)*N^VL<G`TtjK-$WRPw-7n4fm}v>eCN;_acmxlU~xDcoFJQx@yQJFAmZ>aDGu<s
zOo(OSUO<Uti7<~?a3K!O*gsJRGI_k$nQ?H+7=<hc596^(W|$NkV_{}-e`SXAf1Mc%
c6U84zBQ|a|;PQB60;mS}>X9B-3fP?g0iPO|KL7v#

diff --git a/tests/create-join-reference/create-join.zarr/data/8.0.0.0 b/tests/create-join-reference/create-join.zarr/data/8.0.0.0
deleted file mode 100644
index 7a88dcd2e9fee8c1f32914c95df7ca79646ec55b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3367
zcmY*b30Tv|5}$-afr4nf0XbB>5RB#yN61YeAt9ikf(e%hs6^vYlr}0Th(a(#u~nNs
z@px8|0DWRxEUib2wO%N#ZMAsTqXlB4&)YYf_I>Yr^X=};&ir=fKVNnxOQnRm3x)%b
zIms6Q8JwU3IFuHxmex(P><<qo182CFW^oA?Z8r@C3(aEb?zYgnyS0RO(`Ymc9<k80
zG;KGYMyr+x7S!YDM#6)r*J?3}H$0z(jw~)9jhG06_#`TXBaV1P1**F-9{GfD3vrJK
zOZOE*9uNc+2r9$bLIGbUZpWi2gK=dn7`z#%RPdVsL#R*|+(?1Rd2)H9ys=RTL*>JO
zIvCuCg9jBxT=oVpD(Yquyr(x%Mguhf9G-IdeL@BI3kqBec|m~(01ZM5?o)y41C+rK
za=%~(P&|O@57a^676#OD1mPA15CYUuBvXAs!NrRO_oYA$0T=~TKEO=yTnTQYiQrg)
zAq)n)!9h3J?Km6`y4_*Gt=&#{*y$)ZP>n{q!C;^}=%_&qigp8?j$q^t+z_k7Ky)|=
zLyc>1FyL9l8xZTT)6tGb^oEh}II0n6_W*yp5C&>oNCYC7;4uKA=yXP#(aCT!jLtSE
zN^M56#nIN*Mwrv+G%`?XLk)xBbSYg5gd!G=PN$RD2^JaA7|EVtBqtf@1Az#2I*q6#
z7AFo=qPYz*2x&vaWO(c4L=b@)894C_hD3nT8Q^B}q=FjnFTi|vfaeW9>QUCl6TK#3
z*{izXO~LvxBVSp|!dpxQ`<u#>I*glpE=Rt$=e*(0a$Cs=)EwErB|N5df^<Ru)yLts
zQh%@Qv~zn`{&arPttUCRBfmd5EPHFqxDyqWsvTqAYh|=AAG7>GUdNU3?R{~dd8<wy
z56Sx4xb*8glrMH&l-f#y8V+l>o%W}mPkOewb}xk)UjDBG%r}E~Mo;*>|A*7xU*F)q
z=T!@xf05ANcQ^p=aP9x@a4p{902X*v1I2ZR@tzK|dBeb<831@~kj$%1#Y7rzHUR{J
zV~Q?apaX#b0Qc}MKtU&R0Zx-(HLiET6oM?t?2oIU6U@j6*<t{wAc`=VI(<M3oe*O4
z0dFcie*3oW{*gJOt|c}dGJkyZ?xS@jN59<Ccipe2m$x={+;Hb{fo9|zo0mPx&(YPI
zAu_Mscg9zC;8Djf!PW&wYMbBh@;$bzWbeT-Q@g^t7o^rsIX`{k;^+1Z<+mbdR?$Z3
z!!K4up8OGZ4}><!+Os~)zVWS3b>6ez$L7cb+VmNnYuCnCtVr`R*UY?i=Z!)*@Qv`=
zuo>FlCa#WqmV3P7n?H`cWDaq+KB^jHx_*1OWb3-z=4)4J1G>r!A*-6NO#Ce4(aS6A
z_xarPo2MHZv2=*f*>|R2E4-X(j4ZszFK=4??zD@{(h&{J^?RT$?gS&U^@^bC)C149
zM=AH7tSfe`XrEtee=oh|+?`9$F7yO;7ImE-l{xbF^XJ~9eoDXB6x<z^{XB4jzBTPl
z!|D<A1o^`SV><$Kzne-CyGP~g@-FQ-8yKo8@9#}-o$D@H?VrefZKxQ}_-s>G&driv
zk63QUW_li5*;i?u<h&D+yyEicO)o6Ve*Lt|u;`DButY`Gz>8vq{YCEJ50>e>tj8Z#
z<Ze`daxPD@UKbkac`)aK@T0pGVGkIugQGvup1yQVcVp+#oq?}AmNxASI(z8M(^a>&
zo{9LCf0yC8aLa?@T_65aT~s){W5?rJ-w)n1;orR)_ZRb<{w)7NzoqlQj{f&QxM&+6
z+qA(uX7bm)8>Y?m2^O9`x|^T=re*Bf4G}+?GGhGv+^zi@*q5nbmu5|UI{RGTf$L{3
zpUfTM_(rzo&+cVzk6-RKZ)Ju)`NP)nbK&vNo1b3o@H!L2kn+z*eRvn<Tsk#O-8g;U
zE78o9l^%5g>sAT!tjFIo7D9TQxc@~I#rER6P-~C6tnkcNUtMV4aX)I`^U_5VPQ9wT
zeR-HI`>EeCRs6YT6JFF1vbZiZ*+7WyHTGc8>->8G8g5em(J25^E}Ejv0nh@bPMwKd
za}a3Up1c$Ci>oTo+H5lc^Z@A4<WFP`EE53q0>Bg)7+Gi1kpdD_Vtv?brcS_u_?rZT
z>u{RZxRd@;xpC9RjTNTScT3m2U%I+<ZAn>m)y8)h8IAcHs>;jP7|S+n-cr4FWBK}Z
z<(1VnH8s_ntGAhJ%{5lbwwikDb}RBaYkmDToZGFI?b}gPUte#<&9-esZop-`wcb+K
zV71iNnQQ85YiiAPW^-+=*-~q+bWv4RRdrR>W~9o><;8`Ii<cNyR8&-yZ&-&>*JI<R
z)Bo--V(c#^I8E4N@ISId20`K?7z3^bvx;Qa*$dMkEbO!|g#C1OFt(bPjj*U0EfcnR
z&D%K*^S-)eJ83nKrIet6fHlI200H&z(Rg5uo(0o-w{~*O(-MY9dH?$`%!}F$H9uav
z(RbUlR{;kBw%`v}FfgaHA>x<`yH;OMOX}p{66`*(X9a|R0H!dQ2ZN~4DwWElVyT3X
z2RNFg5~)<|nj&0rClg8H<HRzVOezxagi>(~D&yrMmDjn4zIRi?qovU_A)C$`(a#xw
zWxzXNe1AC4sf=bsj$-9x!i=Gju?poZZY?t-Jw?Iui;&CYiiCv3B=Mw4^2F&~_daDx
z6C`4}G${NnhT}5g1cJD@_?Ba>&8O~859Ri>BqVE+(^4c7nM{&ATcgsbRZ_VsDJ4TD
z65~OMRGxqzF2<1{7K_B<6VNIa{iVUgdSKwKO(KpZ6c>jhE*@9p7$qhqP7oh26bj;n
z0s*Suw)1(s2yYfAn$PFQz%2H(F?>D`WeyLC&*HPk#Sn@n2V-NSqhq24u?o3NrbwQX
zm6NN_Oixv-k`hyJNlKZUlb@HJl`%(~qS2y8nW#ufP1B}oQ#I;jjW$6h7E44JTPTsq
zr4m^}f=EcdDK<JLP9zcICy9g-5t@a#5VPwSQ6(m+RLN?!TBA-ur)rI6{`>{{>}<V$
zL00Ah9a%H;3l<d>6)swspO=@va3NvnL#CRYm6MmNo13aiK)1?7sa%<^FIbqT&zzm2
zOiEOyYLXI^>bdhWGSbu1)XB*ysmaPDg+i%Rsg$bZWR+5xoUBkN5)+dMsZdKe0VJWt
zMFgnSq@>MGOVQ$znwpl1LaII|e_^g(r_UynEc(SuOG=g$7Zny36%`jRE=HPlNIv(s
zm|HtcHk;>~S07UcMovDnDPyvgX<Gi!1o;F0?xUITa~{^k;a{c}cG)-(-jlV1G>>#}
zdteUSN$Mi>7F?MO<x8VGX}wmHz@6(hrTtd;ZOA+&hCZ()qkf-fYAl6eDyW2+rjns^
zbPZ9*Ozi+;eQ(}qFBA72u~ByIohbOY^-BYGu)`P(_G7iE8w7Ci@e&F^u!sURL9rAF
zj-x=KZ!MgRMGT9<Bv9~xFdo|r!Z=(~$6OZhiJr~nad|ud29L}6tBz5@<#4$GTsE+I
zIQYO}ahNO?hs)!z*|<X_8$oO~gTn;`vVa3DE|<aPa2PBA9-d~Qe+HAqMmrZ6oWJTg
zm_^>(OePDR;d`6;-}g4Y#eIh||No^uD1<!GHA9v{*pMl1H7J|khVwW`JRXO|VsSZa
zk`xcOYz`N{o(UW_76BLdm<LA1wD?@$@Oe|++y<gJSRLe}SZs2h%^_KEfsN(F=8!CS
pm;vyZEFKpTt}LR_1xDfW**wlFH-GLPx1r2KZZ^ydb7cXG^B<f`*%|-<

diff --git a/tests/create-join-reference/create-join.zarr/data/9.0.0.0 b/tests/create-join-reference/create-join.zarr/data/9.0.0.0
deleted file mode 100644
index 3b6725d3d23dc5d2887da7b597d1c99821bbe6fd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 3377
zcmYLL30Tv|5}zD^;DshCYQcyU4?IxG9Z5*YNkUFActC=3C<UU$1C$aCa#|8ez)CH}
zR;|x^VAX1^Rx4I*t$kSNXVtckYV8ACYqaRARq@Tne(mHxyEDI;o!!}QcC%E<M7o3t
zKzR000PNrh6~G_DK=PUy-rY#C0cLbDV#DB3Jj2Xj#O{tIRtCfDHG9!uHk-S!4Ty-F
zQSR~*3?VO~gfLzg2Al$q*@&Zymw^Th5@1vqBpQBX5E(^cba}~K<csw(FtXR{?P7Ej
z3W7+WKx8@c2NW2}Ms*U3N{A}wKu|RVQDImN45vc*gnkN4$?rw#@0S8497gogfJ%jt
zy&;g$-(MCAVdyI(P;dmM-YY^n`8T6<5-AL*6rc<ON)WJ4ib7$eXb4cKKpg^*1Rw@r
z0Puh^3}}Oa!U3QHEga~BfgS;rFreY)P6D6<cz}`))Cu5UBobMe7N5n!?C$n4v9nlw
z%x?SzKp#RV`?`Gy5$;C2#pmnpw%~^<I-};p9<gp5?z5o6L@Npi`j|L8U}rLYGr&I{
zJ<;m31SVjTfn)&DefTlS5Q_yPFfBHK$>2}GpdB=t%te!RbjUhfE?P%NhpPis7pgQF
z4GAqommv`Fk;z=d4;^U4qeBG#5i}W55SGzgWCRWmxT70NG+Ge&hhT?JILswO2Mj8s
z5d=eb=wJl_Tr}DQcn&XycgS!e5@j5%B7_PCtXhC<+}y`+oX&Co{^0Vqm*P%++_L2*
z{hI@8I>Qfy^=3c0I%Dev|Fa7_zxn*japs3Nid$3(OS~SnXUuJV^NKI}W_MPUe>s0t
zM&I?CggN&Xj-<W*sp+hHSi8R9e>U~EocqHz{Jyuf!M3z7BxvW*>RH1-+b4MVEcU$p
zUxvQ6w)3js<&M?2vR-BsohzOkxBj=Ozr@{=fAH>Irs{s<gYJ{cwc6&}r{1sXTTzt4
z?j-bA^^V8tb^lYnZmiz%90;u^m4(%dHO!ENfuH3BRUk)Jr*q0z0MmgLQ1RgY;XFgc
zz!rqV8K|=vSv?%wb_t*bN424~%@YCO?12ayDo$i{j3M>jk6J*NAQJ8b))>J3^gn-I
zQTfyM!rAXfr0v@|s49Gqd&`l%b9~tc+ZOH1ewsb)Dp+f5bF#IC$F#NYzF9R;QZLV>
z&!Q~4y@oed`Bvqud&<xw*V}L1J#n^zUjD3C6?N!Y>yW$8=AYj1lQpaQ%0SotkdI~u
zH{{>=o)JT1${o1}f<~zu^QwL~&d;bAb|t#3dD-L7^hcMo&b5BmUeNs2A=R!QT<o`^
z%7QjbI$XFeJUcw4v#Dz3?(%`1RmVD`*Phn49~|>2^yf)&K?inx`p416<7Gda1@!mD
z4Bfe~Tj_t6yq2)Gybx~n)<>_oI{nY6%ZDx<S=um7d2oMJ^f7Dan_YK)%zmzSXIM?U
z7nMqrKlyLc+r4k6m4y!^U-XRSGprubu?wHv?)mU&^y9u+ly#1%MQizK84G@yOxNia
z6*&KJ{$~8o#lDvJwzWpsx5Uoq8HnDI6-xa$=2Gwak|*Q+t12G8^Qm|@t0K*~XA}Fw
z&PU>9XKQjt*&4r~FxPPwRMRfaySMvl{S}9C`^dTX4Lg{Q@a=O)3`81pcPy+=y~ti#
zT9iLNq<MOb>O@<}rs9OCMn&D^*c}b`cGkYMp8xRCK;81AdrNl}x6n>1zWSzjT>I$J
zDW;VRl<&ruR6R|LyEb#?fxG)&)vuUx*qUGW?JfmOj5xmfd})_C>$R{*XG(eE?uK8*
zqhEWaX3Ucx6Po1T#LR)cubyA!8W}$BJe(CR3~^`|PA#!bs~B^W_U~I;@3mS-{3)-K
zJ`EF$I<W0r?$-<IrsQzm8@@gz(l!uve5fe*=G>w;?1>F$A{r`-FGpXbD+;Hke?GOV
zr8&#$y1n>jJ+G7EFSk|HvsL>}9jh7*`|n>an`O9Z|9)Bb<Iw}c9T!hbx=?iG{40#m
zRc#WmouX3Osdys#nNDXXi`4!ZEMbomY}h|V5x^Gi?Dqn+J0;VgpH*WECuQA+2JFtj
zuQ2V!iCGI03G{$J9$~QcI7!*J!BVi`F_1tqIGnH<pff%m{|21jdTi+R5A~qPs-Tlw
zrn0)aYSX3-&UK|_<z=f&OC9U$YpctON>}E&YFw3NC1smxH#fLzsyA({s;zUs?yjw_
zb9>xPzGm-MpRcL8$>(jXYx4QL&0DuMH~X3if|?JdM&u^nRz!%`maR=*ccaJS_B6OX
zTRdCbZnxLtsUcKbTU%FGSA$J$ZOxh$#g0P9%2kz>u8kX<8#YwBW)u4BmPy82#)0g_
zn+E?Q9ZHByE`YHRxMtR1RuQ0qIABg60wW)V!UOvg2W}am#8sKU#R1e?wfkON9CzBL
zfawOncToVQrqpY!<l=Y2E=e|5emxROS{?&5XDWbYy%wC~pQDD@i|`A59LCw)3yZ-T
zPeN{lpPj?@ps)+}=-uV8_ddXC2&Tdrxk4eA%N6ou<S=+}8f8T?LJGN*U}!~FNM#bG
zQX!WqBy$xCnL;j_tCpIAuiqHB9y&83E@HG~G-oCRO%&d{L7i^68$7n3C*+QbPb*l~
z6GWSyt~5_qIYq`S_G~p*qLHgKDJrdY-pu$!#rU&VKKw`)r;#b;$rENKE0hwkc=8O1
zSaRt2k%Pk}V<){VJgC(fjq{CU9dc8;!C=&<%2jH;F+(j=XmIIDl@?c@oGee);2#vO
zS*}3FkMIxVPRE-yAvrNIQH-BhjKL*Dl3*uJOiD_U%A}H{B#9(RBAyz;<_Xy0vw0$+
zC@vA^@?xTdLZL_~<Ol=;zHppy<|xs;7bKR>OPq&_c%DY3RHYcKnM;=17h5fcRBejM
zkgC&~GM41#WM?gyZ#J3C0Yqamr<pBgi_u^(nbnxFT$Uu2O64k*LatQf>dDeDrdTSM
zOC`80d9n=c$Y@q!RtkBlR-2ltH(&$|VZzNYE?%5vx7%%YTV`gK&6bs!nVFZryr5wD
zvi!W<+}yl8(n@7=wZUr3$;+`Xw4|yPGO1RhRHtTS=jZ3x7pI$aTCLV>(5iKY1q&Bs
zSku#tdcDc4*J;%nolb`n>h-BQoxzZz(P&c8mn;GSoldLK5QW$Yr|MI+Mx!M?-E2uq
z$3!g_6Xt7QlAD*4ZL??NDziyrFDxo{6crXM$92B)g7!8)c)Rt4M9P|4BuK3!RiD1*
z#x;rgvp3Ez1E?HaQ!}^~=oY+PB!^d*&|*%*kRK|so;_I7cbFR!vhj4>Vd8VJX{nOF
zetUZwm;TO~!3TN<Q6lF7-_Qop1wk`v$ECaqi9gkkVc|J~LC>M(aGIzE#pomPCsqCX
zw^P37e<Hr@ZtW8wc%TLwLOoNQ;NQ8H0x&^FfriLL3QQ1FVEIrtbR?pklfVu51p$LU
z2<V~EZ|C!XgaF_R_yQq-fWybfDH{PE0z4iXa14*n<*|i)A)n9TaJT~C@wh_ZqlL>S
zPCPyc_`v0&H*k3P%;g2<K>!D0EFSQAe10HO1<<K5fkR&0Y$Oin#fzK$;>FDlAusaa
zzaQJfr7#o$$MC7N2JAie6L3<D#o_ZYG?|5s%MkFn0v?Btd7y;r0U;VV2=Ij>RL~t&
zl94cwQ4ba0J$@FK<Rjp6Ie3mRoDlHbVA4VX1}6DnjwBy!$V^-*KsorsF`5t^$!-bA
P#&JntLTo_-aQOcRZ~T5J

diff --git a/tests/create-join-reference/create-join.zarr/dates/.zarray b/tests/create-join-reference/create-join.zarr/dates/.zarray
deleted file mode 100644
index 9a6e5f3..0000000
--- a/tests/create-join-reference/create-join.zarr/dates/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        10
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<M8[s]",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/dates/0 b/tests/create-join-reference/create-join.zarr/dates/0
deleted file mode 100644
index c04549e88081fd35feac96ed26ced48006882a1f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1R!QO{5l>)9mswIrW-`wf@z1n5Sk(F9Ymh*Jw$#xgmy^y0Om7r
KeuT(xgwO!$(iTVn

diff --git a/tests/create-join-reference/create-join.zarr/latitudes/.zarray b/tests/create-join-reference/create-join.zarr/latitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-join-reference/create-join.zarr/latitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/latitudes/0 b/tests/create-join-reference/create-join.zarr/latitudes/0
deleted file mode 100644
index bc43c5131b3b14758aa08ae329fc209b28364f0d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 179
zcmZQ#ROAp~Wnci|%?u0-B0$Uy#PSS`44nZ$235J5K;vXX7#U>!fY=3yO@J6^6R$Fm
j=?)aiW{~vjX9x%Z0gzMV9T*u*Kq}-9Ff!-|8~_0TnG_Rp

diff --git a/tests/create-join-reference/create-join.zarr/longitudes/.zarray b/tests/create-join-reference/create-join.zarr/longitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-join-reference/create-join.zarr/longitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/longitudes/0 b/tests/create-join-reference/create-join.zarr/longitudes/0
deleted file mode 100644
index 3fa39c0fa3bb6f35f29e18b16de9bf93759c73e1..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 199
zcmZQ#ROAp~Wnci|;|vT8B0$Uy#PSS`44nZ$24!41pnf5sQGyLj3`q_R2N)a}gn^bZ
z@Cz~I27vkcKo$Q*8BARKLL%c5Q_?cC^9l-!N=h+BR2b3%z*77`EjkPij384g0vtd9
E0Dg2I-v9sr

diff --git a/tests/create-join-reference/create-join.zarr/maximum/.zarray b/tests/create-join-reference/create-join.zarr/maximum/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/maximum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/maximum/0 b/tests/create-join-reference/create-join.zarr/maximum/0
deleted file mode 100644
index 732cf2baeb9eb9099f1e1bb7531ff26821d0a62e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03Zf}1S5-L2QZ7_gFTowoCu;DGM8Mo1Mz*$PlITMd!@M!K-wYE
NBM(F~IE2o(2LQ}(4iNwV

diff --git a/tests/create-join-reference/create-join.zarr/mean/.zarray b/tests/create-join-reference/create-join.zarr/mean/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/mean/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/mean/0 b/tests/create-join-reference/create-join.zarr/mean/0
deleted file mode 100644
index b4499fbe5136f6d03c167d7c0f9de7f8b8e50658..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03hc1DR=o;e4)b<9-F&|DxcUJe<;beyJ2YmT5@;AOYdWLe?PN5
ioWpX$?$VQeT)XwM9D>;9<+1M0a`?Z;(ZYL&vpoP_IUo)I

diff --git a/tests/create-join-reference/create-join.zarr/minimum/.zarray b/tests/create-join-reference/create-join.zarr/minimum/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/minimum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/minimum/0 b/tests/create-join-reference/create-join.zarr/minimum/0
deleted file mode 100644
index 88a384089510ed3c35c50c526b3d26791653213b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03Zf}1*d&-9DsC#l+`hNkN^xk3*Kl4VxPLT6+|~oYE5?l(heuH
K!8DM`zyJUZ7Y<_p

diff --git a/tests/create-join-reference/create-join.zarr/squares/.zarray b/tests/create-join-reference/create-join.zarr/squares/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/squares/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/squares/0 b/tests/create-join-reference/create-join.zarr/squares/0
deleted file mode 100644
index d4dca4823d0d8fd031e8d522476dae466092047d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03Zee8==2*9l@-CP6r_Uvz|H59!U3WoaAW-63<xW1EN=4Vw?z4
PC-!SQn7+67N3}fwF{u)c

diff --git a/tests/create-join-reference/create-join.zarr/stdev/.zarray b/tests/create-join-reference/create-join.zarr/stdev/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/stdev/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/stdev/0 b/tests/create-join-reference/create-join.zarr/stdev/0
deleted file mode 100644
index be7cfba17a23be2207e85d8716f2ef8e22f20731..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03f~<s90C4V(c)x`}eDRPM7VswS)^+Uk|YVrL2*1@zX-PiTY9D
h$!*<sF;0fE)>fJhYk%&4H(x=+!Ex<@1uH}n>;W019tZ#c

diff --git a/tests/create-join-reference/create-join.zarr/sums/.zarray b/tests/create-join-reference/create-join.zarr/sums/.zarray
deleted file mode 100644
index 2e7a914..0000000
--- a/tests/create-join-reference/create-join.zarr/sums/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        8
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        8
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-join-reference/create-join.zarr/sums/0 b/tests/create-join-reference/create-join.zarr/sums/0
deleted file mode 100644
index f45f5c9e61f242eb180b85ff0e4f940debb478a8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 80
zcmZQ#H0E$%U|;~@03Zf}p4J*^M<Bf<z-WR4kUk(~ea9Y1>+viu1kn=9{uhDhFW<I`
SfYhCNuObSf8TyUh+XDbn<`OXg

diff --git a/tests/create-pipe-reference/create-pipe.zarr/.zattrs b/tests/create-pipe-reference/create-pipe.zarr/.zattrs
deleted file mode 100644
index 03abd51..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/.zattrs
+++ /dev/null
@@ -1,354 +0,0 @@
-{
-    "_create_yaml_config": {
-        "common": {
-            "mars_request": {
-                "class": "ea",
-                "date": "$datetime_format($dates,%Y%m%d)",
-                "expver": "0001",
-                "grid": "20./20.",
-                "name": "mars",
-                "stream": "oper",
-                "time": "$datetime_format($dates,%H%M)",
-                "type": "an"
-            }
-        },
-        "config_path": "/etc/ecmwf/nfs/dh1_home_a/mafp/dev/ecml-tools/tests/create-pipe.yaml",
-        "copyright": "ecmwf",
-        "dataset_status": "testing",
-        "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-        "input": {
-            "dates": {
-                "end": "2021-01-03T12:00:00",
-                "frequency": "12h",
-                "join": [
-                    {
-                        "label": {
-                            "name": "previous_data",
-                            "source": {
-                                "class": "ea",
-                                "date": "$datetime_format($dates,%Y%m%d)",
-                                "expver": "0001",
-                                "grid": "20./20.",
-                                "levtype": "sfc",
-                                "name": "mars",
-                                "param": [
-                                    "2t"
-                                ],
-                                "stream": "oper",
-                                "time": "$datetime_format($dates,%H%M)",
-                                "type": "an"
-                            }
-                        }
-                    },
-                    {
-                        "pipe": [
-                            {
-                                "source": {
-                                    "class": "ea",
-                                    "date": "$datetime_format($dates,%Y%m%d)",
-                                    "expver": "0001",
-                                    "grid": "20./20.",
-                                    "level": [
-                                        50,
-                                        100
-                                    ],
-                                    "levtype": "pl",
-                                    "name": "mars",
-                                    "param": [
-                                        "q",
-                                        "t"
-                                    ],
-                                    "stream": "oper",
-                                    "time": "$datetime_format($dates,%H%M)",
-                                    "type": "an"
-                                }
-                            },
-                            {
-                                "filter": {
-                                    "param": [
-                                        "q"
-                                    ]
-                                }
-                            },
-                            {
-                                "filter": {
-                                    "level": [
-                                        50
-                                    ]
-                                }
-                            }
-                        ]
-                    },
-                    {
-                        "source": {
-                            "class": "ea",
-                            "date": "$datetime_format($dates,%Y%m%d)",
-                            "expver": "0001",
-                            "grid": "20./20.",
-                            "name": "era5-accumulations",
-                            "param": [
-                                "cp",
-                                "tp"
-                            ],
-                            "stream": "oper",
-                            "time": "$datetime_format($dates,%H)",
-                            "type": "an"
-                        }
-                    },
-                    {
-                        "source": {
-                            "date": "$dates",
-                            "name": "constants",
-                            "param": [
-                                "cos_latitude"
-                            ],
-                            "source_or_dataset": "$previous_data"
-                        }
-                    }
-                ],
-                "start": "2020-12-30T00:00:00"
-            }
-        },
-        "licence": "CC-BY-4.0",
-        "loop": [
-            {
-                "dates": {
-                    "end": "2021-01-03T12:00:00",
-                    "frequency": "12h",
-                    "group_by": "monthly",
-                    "start": "2020-12-30T00:00:00"
-                }
-            }
-        ],
-        "name": "test-small",
-        "output": {
-            "append_axis": 0,
-            "chunking": {
-                "dates": 1
-            },
-            "dtype": "float32",
-            "ensemble_dimension": 2,
-            "flatten_grid": true,
-            "order_by": [
-                {
-                    "valid_datetime": "ascending"
-                },
-                {
-                    "param_level": "ascending"
-                },
-                {
-                    "number": "ascending"
-                }
-            ],
-            "remapping": {
-                "param_level": "{param}_{levelist}"
-            },
-            "statistics": "param_level",
-            "statistics_end": 2021
-        },
-        "purpose": "aifs",
-        "reading_chunks": null,
-        "statistics_axis": 1,
-        "statistics_names": "ascending"
-    },
-    "copyright": "ecmwf",
-    "data_request": {
-        "area": [
-            80.0,
-            0.0,
-            -80.0,
-            340.0
-        ],
-        "grid": [
-            20.0,
-            20.0
-        ],
-        "param_level": {
-            "pl": [
-                [
-                    "q",
-                    50
-                ]
-            ],
-            "sfc": [
-                "2t",
-                "cp",
-                "tp"
-            ]
-        },
-        "param_step": {
-            "sfc": [
-                [
-                    "cp",
-                    6
-                ],
-                [
-                    "tp",
-                    6
-                ]
-            ]
-        }
-    },
-    "description": "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset",
-    "end_date": "2021-01-03T12:00:00",
-    "ensemble_dimension": 1,
-    "flatten_grid": true,
-    "frequency": 12,
-    "history": [
-        {
-            "action": "initialised",
-            "timestamp": "2024-01-30T14:54:49.417554"
-        },
-        {
-            "action": "statistics_registry_initialised",
-            "timestamp": "2024-01-30T14:54:49.423223",
-            "version": 2
-        },
-        {
-            "action": "init finished",
-            "timestamp": "2024-01-30T14:54:49.428172"
-        },
-        {
-            "action": "loading_data_start",
-            "parts": null,
-            "timestamp": "2024-01-30T14:54:49.462681"
-        },
-        {
-            "action": "loading_data_end",
-            "parts": null,
-            "timestamp": "2024-01-30T14:54:49.765826"
-        },
-        {
-            "action": "compute_statistics_end",
-            "end": "2021-01-03T12:00:00",
-            "i_end": 9,
-            "i_start": 0,
-            "start": "2020-12-30T00:00:00",
-            "timestamp": "2024-01-30T14:54:50.685090"
-        }
-    ],
-    "latest_write_timestamp": "2024-01-30T14:54:49.756384",
-    "licence": "CC-BY-4.0",
-    "order_by": [
-        {
-            "valid_datetime": "ascending"
-        },
-        {
-            "param_level": "ascending"
-        },
-        {
-            "number": "ascending"
-        }
-    ],
-    "provenance_load": {
-        "git_versions": {
-            "__main__": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 16
-                }
-            },
-            "__mp_main__": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 16
-                }
-            },
-            "climetlab": {
-                "git": {
-                    "modified_files": 7,
-                    "sha1": "0c12855c1e2c8ec669714d4b873eb2f3971d6e24",
-                    "untracked_files": 0
-                }
-            },
-            "ecml_tools": {
-                "git": {
-                    "modified_files": 1,
-                    "sha1": "a8cb15befdd8c9544ea38deb780cb319bde6e9f6",
-                    "untracked_files": 16
-                }
-            },
-            "prepml": {
-                "git": {
-                    "modified_files": 0,
-                    "sha1": "b9fce5cff2ec951899b13b9d97a4bcdb3e0fe4ef",
-                    "untracked_files": 4
-                }
-            }
-        },
-        "module_versions": {
-            "__main__": ".../test_create.py",
-            "__mp_main__": ".../test_create.py",
-            "_cffi_backend": "1.15.1",
-            "_csv": "1.0",
-            "_ctypes": "1.1.0",
-            "_decimal": "1.70",
-            "argparse": "1.1",
-            "certifi": "2022.12.07",
-            "cffi": "1.15.1",
-            "cgi": "2.6",
-            "chardet": "4.0.0",
-            "charset_normalizer": "3.1.0",
-            "climetlab": "0.20.1",
-            "csv": "1.0",
-            "ctypes": "1.1.0",
-            "dateutil": "2.8.2",
-            "decimal": "1.70",
-            "eccodes": "2.30.0",
-            "ecml_tools": "0.3.0",
-            "ecmwf.opendata": "0.2.0",
-            "ecmwfapi": "1.6.3",
-            "ecmwflibs": "0.5.3",
-            "entrypoints": "0.4",
-            "fasteners": "0.18",
-            "filelock": "3.12.0",
-            "findlibs": "0.0.5",
-            "fsspec": "2023.9.2",
-            "gribapi": "2.30.0",
-            "idna": "2.10",
-            "ipaddress": "1.0",
-            "json": "2.0.9",
-            "logging": "0.5.1.2",
-            "markdown": "3.4.3",
-            "multiurl": "0.2.1",
-            "numcodecs": "0.11.0",
-            "numpy": "1.23.5",
-            "pandas": "2.0.1",
-            "platform": "1.0.8",
-            "prepml": "0.52.0",
-            "pycparser": "2.21",
-            "pytz": "2023.3",
-            "re": "2.2.1",
-            "requests": "2.31.0",
-            "six": "1.16.0",
-            "socks": "1.7.1",
-            "tqdm": "4.65.0",
-            "urllib3": "1.26.15",
-            "yaml": "6.0",
-            "zarr": "2.16.1",
-            "zlib": "1.0"
-        },
-        "python": "3.10.10",
-        "time": "2024-01-30T14:54:50.246362"
-    },
-    "remapping": {
-        "param_level": "{param}_{levelist}"
-    },
-    "resolution": 20.0,
-    "start_date": "2020-12-30T00:00:00",
-    "statistics_end_date": "2021-01-03T12:00:00",
-    "statistics_start_date": "2020-12-30T00:00:00",
-    "total_number_of_files": 48,
-    "total_size": 38817,
-    "uuid": "13ea8d8a-33be-4a00-8ea4-7c80ce23a435",
-    "variables": [
-        "2t",
-        "cos_latitude",
-        "cp",
-        "q_50",
-        "tp"
-    ],
-    "version": "0.20"
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/.zgroup b/tests/create-pipe-reference/create-pipe.zarr/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/.zgroup b/tests/create-pipe-reference/create-pipe.zarr/_build/.zgroup
deleted file mode 100644
index 3b7daf2..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/.zgroup
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/count/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/count/.zarray
deleted file mode 100644
index 4ee32a5..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/count/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i8",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/count/0.0 b/tests/create-pipe-reference/create-pipe.zarr/_build/count/0.0
deleted file mode 100644
index e377e98d016e616b7342cd88647fbc23d7a17e95..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 41
pcmZQ#G~}4T$iM)?nhXpKB0ww%#PW+68RX;{7#aS%1pv7W3;<K`1VR7+

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/flags/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/flags/.zarray
deleted file mode 100644
index e07a6bc..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/flags/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "|b1",
-    "fill_value": false,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/flags/0 b/tests/create-pipe-reference/create-pipe.zarr/_build/flags/0
deleted file mode 100644
index e86af411a0cf2fad9169e2fae28ded98597aecb7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18
VcmZQ#G-hOCU|;}YAs}XC1ONmj08jt`

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/lengths/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/lengths/.zarray
deleted file mode 100644
index 76820ba..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/lengths/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        2
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i4",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        2
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/lengths/0 b/tests/create-pipe-reference/create-pipe.zarr/_build/lengths/0
deleted file mode 100644
index 9d719554d8dadd511f1953deae146050e52ab286..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 24
YcmZQ#G-lyoU|;}Y2_R+xVm2TK00)f#Z2$lO

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/maximum/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/maximum/.zarray
deleted file mode 100644
index 315ad30..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/maximum/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/maximum/0.0 b/tests/create-pipe-reference/create-pipe.zarr/_build/maximum/0.0
deleted file mode 100644
index 1c86bcb4311aeda5fef3855017701c54e7fd03f8..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 229
zcmZQ#G~}4T$iM)?PZ<~(M1c4#5X&<#GBhP0U|?uqXkcYvW?~R<;9y{6We{lKVqjEe
z5J=zxVj+hAJ$o4*$SG(rIHWsVW?=Z_pw5uss*uBw@FPK)!J(UB0)v9b0~Q8{-3LA~
zEbvt@Vo*>|Fk>jK_7i4^Z1^I}V0b1}g5lWrRw0JHJWBiw7GHk~G9*fPD=>J8oD*S~
zV;*F|;AJwY_(Sq3AklYfTJeX8r{))bs5#YE{GsGDkf=G;Tl}H+^yK0X?Wfy|Ka`$s
S21&Fy*xTFLvoeSVfEfS*&PwM1

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/minimum/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/minimum/.zarray
deleted file mode 100644
index 315ad30..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/minimum/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/minimum/0.0 b/tests/create-pipe-reference/create-pipe.zarr/_build/minimum/0.0
deleted file mode 100644
index 8084353e90db2dd05a9efb746a519badab150b29..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 217
zcmWlOAxHyZ7>1uyevvC)&^z{(EiKO^e1n@dgJFGeEG%kv@mSMjIRw+?fl-BnjT^{?
znFQh3(z<mpSkeCv&+|U;9(d=+PaZf{z^P?nY2q<y<>@mR&RT|G=o6%NHwMpU>9b49
zl$0-F#<y@nB#elKC1W9=ErjGkhX*0zQ+Q!3)Z8dn)XFQr$~}d$W}rMWRWiCt!i93p
kkJ4bT9CBOnZpiK`-mtR$cT~8VyX(WL<g!O>2>WjN2hT_{NB{r;

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/squares/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/squares/.zarray
deleted file mode 100644
index 315ad30..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/squares/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/squares/0.0 b/tests/create-pipe-reference/create-pipe.zarr/_build/squares/0.0
deleted file mode 100644
index 5d32229665ca0e610af2e61075075710fe9a7569..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 246
zcmZQ#G~}4T$iM)?-xwGeM1c4n5X&<#GPL~P-{6pNfT4lG;Q+${2L=TOhK2?p>%amA
z2B7c)h6N4>fK&oVLBoOt3<(Jdx(t7Kqgokax9~bMT%5Fe5yNUO7g2`pM^%d%xR(4+
zVp#XZYc7Mu?>HX@w~ebmGqCJj{?g&VqaId={D-GLIjB8a#O+}Jdc{YFTYsjrIHbLe
z`0OCfw42jG=kw2B4vDX3@;R7(l}QizE#{CB;vg209%3ODkrARU=8zsDE@qJtqAKQ^
n9-=Dloe`odW|kfzAZC&g!Y5|v=wN4O@91F9${-W~r0nbgoLf}*

diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/sums/.zarray b/tests/create-pipe-reference/create-pipe.zarr/_build/sums/.zarray
deleted file mode 100644
index 315ad30..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/_build/sums/.zarray
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-    "chunks": [
-        10,
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/_build/sums/0.0 b/tests/create-pipe-reference/create-pipe.zarr/_build/sums/0.0
deleted file mode 100644
index 0b98eca403c237f0fb75812a220e7d76c4eca922..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 245
zcmZQ#G~}4T$iM)?Ul|w}M1c4%5X&<#GBo|)_h13T0)_(%7!(*97BD1$hzAS~ATE$N
z0Fr26U;v2$Sqc!MPKxJA!Dgw7#v5m(8g4AuB-OA|BwniKddXg?8FL#}NU^wb9Fcn9
z+_6q-#VZphDUFG5KEJvYBhT>aOw_6Gue>8>Gru|?U-JFci9A+@SL~_jKVBu5CosM8
z&rA9J$|`Xt>nrm-mZy=^?6y9O++cU`Y2<3Vz0V@o+U<E7xx{Y!v&a>8JD*0bwA=kG
ba=G1>r;+pQHa`Oq4i5JA_N)w|0bm9IJ)vd@

diff --git a/tests/create-pipe-reference/create-pipe.zarr/count/.zarray b/tests/create-pipe-reference/create-pipe.zarr/count/.zarray
deleted file mode 100644
index 2da665c..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/count/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<i8",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/count/0 b/tests/create-pipe-reference/create-pipe.zarr/count/0
deleted file mode 100644
index e168f64600062d64ce540460c7a4ff6fa585f68e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
YcmZQ#H0ID?U|;}Y3m^_*0}@2g08pC(i2wiq

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/.zarray b/tests/create-pipe-reference/create-pipe.zarr/data/.zarray
deleted file mode 100644
index f2d1d66..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/data/.zarray
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-    "chunks": [
-        1,
-        5,
-        1,
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f4",
-    "fill_value": "NaN",
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10,
-        5,
-        1,
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/0.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/0.0.0.0
deleted file mode 100644
index 8e352ee887c614e5abc459113834927706924da5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1937
zcmYLK3sjSJ6#sp5h@kS=pb#W5#7EXDWZU<BgE3$Ze1P*ks}+{VV?KtDDS8C`K#&3E
zHnwrJ;{z!(k2ncm9XsYMkH=G4h^F{ha?<j>hGJ@Ys{2i=```cj?&Ejw|K7X%-z3Zs
z>8Aq#aXZ6-H~>5Y^i*3E3We7n@caE{T7I+HtOzK?m*1-hSS)6<g=DYA8!&sl3NLL-
zKoJOd$&Ks*g@Ra1AYdT`0WG4^ODjM;ZL*09dMFfP6k;rv5J&DJ&`U5cS;<H^LI&vI
z^<V~Jm)T3v&x#a=WH3mspxw@3ppwKiVhiv>g$m(y=p#YYP(({man;&uwOV_7=_!7~
zFs!$?FQie3wZ<dHN`*^-{^&zOZzB3gA*DvlKzIT$2p9?(xnLM%H^w4N0wxg>iHOH*
z6o5f=22&9sfsuL#yh)bF8-K{Ye6GM{+R<|*V#v63naptd{B-Q^KS%Q8t^V`hxBv2M
z%h6S>)pJJtnKXX*&-{u8`)yCf;TMHTZTMqbSo+7apX|A>b{|@NrMYFg<m}YZt@|#-
zZ^-OoXYEZsakIsD&^q&cqFQm`;mCalif+yPsUc}$%2f9cl~I!2>DehU_N3VS@#VAU
zFDSd)cxqJV{gVsR;x4cH&7E**@xaumX)ph6jId^Oi1e(3Nz_66{~feb2a{BYXaE!H
zV4NU_yd(lWslI1L3c<HRqtew}5-J+0v;YCn^+U7BHHj7#dSdp*7;)mH%|S}t3?Wst
zp&MP-&_*n}IY{PVN;eV&LT3^t0+1kPQ5bqQBIC@klf~1b<EHy$ANF>23_ZE=`VIMl
zZ%=G~$DDrk@RXSYw(g$gUlaDI<@@l!@s<yde*Q_COxw4?aJ*l9>&VAj7A}>Kn6l!g
zcUkVBOY>TOKdm_WU}Eslb^V@|*6eRS+FCZ_N8`0r_Tp!)vuih*lIyaMUEe8tyt;!=
zYabYYz<9)7@+9Np`45z5BfmLz>aX|8F9_MKn!(R)k11~m9J_e%cGsO9tHplo7H1+=
zS5n9zj!8Anjl=_*CUlAQA*fr7x%77UUWLpT4rwVP4@To2sn+|t;b<0`i51*(z`(?I
zm>_*s?0UO^4go&umCY9`#$VLtBOI_PG+s5eHG)tfth8E}FJHc@qGIWahWe@{tKKau
zt**D(Dp#&vS6AP#p=zB_UTIsu{sTu{ZLQs5cW&^wX*YR1ZdapIe0ba>xtf|pmD@#<
zr?Iig?RIY@O%nl}n>;SN!)YgH+D@m#VRtn;Y$DYW)z{hT>gi!CD_v-rU%IrcdR<M;
z+BH?x)m5*H^lZHJG+v8n3p8SMK^GMuSwEjH*1z+WW?mi#xvL-U&&o%AsLU_@N!<sC
z`}%fcNXBEl4gKVG7<}#lT|}Vr_}RMJyv=CtG$xH(flCK(j%E>TWpZ6Gp%OdMdIxl^
z+c2!k^9A8M+s65E>ZMV`r_+p)yf$>RQHCG~<dFiHFs`V$*jPNiFke5mXv#!mQBkpR
z{Dg@`6Ab!-Lc^G`MPu}|3<g8K!Jyac^J(ezV?;6-3JVGhkq9m$z!+_aa)|PBb96eb
zHYYDHN1LtH=H=$)=j(EEb9Fg<W~NHbMW$(3O?E=NmK(`xvS=l7Ssc&tJRjRfqY*uG
zvX!G>&kB)Nt25qugc?-W=mck?fadM)!p-qJGFF!imVR}lCKVW!Me``@#4$8oX~Yn0
z#kY4|F$kKVM<-0C8bmdv;5gN)xOaZX<+)pI2|-iyWjU_&A7s`6liM%cp!V&;9b?=C
zL?dA^ii4i-2w&;m)eil`NhawTUzm{9xEK4K8Qo|l6OfQ`Bzha%bAvhb&ykYPU_-K&
zK?+Ip2HNq8mVTtuI5qu-u>IiUz=xbzj?aul8qX%uSHOY;t7f5QIpP40XIVALDwSHp
zkq5`|DkxbEOAv@)ju*qIz^hm_;aMfUSe9RcSfvqiyn$F+n2I$75-AL`Qk2UOe_x^I
zpB_ZvS(TdH)#72Cif1{N*QgN6MH5O<!(}Nobex8CA>#B*Fr5+)<;XwHQLPqpq;!Z>
tP;vj|sOB}$P(@10aTgPOo}&Ym981A?6-O>CCCsX*Y%w(D&U2tx{{S`BbtwP<

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/1.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/1.0.0.0
deleted file mode 100644
index e04513b91485c32ad64007562f0bd9bda11dac5c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1973
zcmY*a4^Wfm8UKAD6&0LNum}i-LN}^daVGixl7!?VL4<&vy;|$pl<P77v>ozqsL=b6
zub2c%0tpf7+HunLnB^SWs?^an-KE^*+Fo3})0=j%v#VR(t#H@#%FW&Oe9_)+Prm29
z=g;qX-{*P0=X;}L3iRWtfX3&OflL6k46s<O*UK^4yL>r4M6!OlSC(U5FR6M}R#mT;
z7QL*-^t9?@B*+w@SyYucrjL<LB^#N@Np3l&>dB$TV)<B1(E?TSlAC~XjDjhK+<Fd6
z*@V^16t8L&Q@!`|@apLpm9ml5s~SCoJTS=XBtFMrm5J0s5?#peG$YxEBprTGgw;B9
zE+$LU)AZbD={}LN5*gDvFzcYt0J4B|fCZBQh6E>=6wsDf2CN3CMUR7_wE@eD(IiRJ
z?d{W2F|Ywl5n#}8j0xZX6V^}vnCV=hLmf2~sMi5Y!{7e;%i`WEU)=afmHzW{oAT!_
zb$#*Ix7z;wyHB1Pw_HE@sr|}^T%q2wvY@3fe=;jR9-a&J2ey~KUAi?{=buems!8c<
z{VKQLxia%-BRh<J6<-y9cJ<lsZsz#U?fch-XB}dR;ZoIB!{ESe|D91W`B#m3f2-=h
zaP8zT-s;<V;Njgb?<#ty?XY2Oc;>ga(zeX7A2er_@a-wjnQE?X>K$qRz-#On+Y#63
zK~r<7se%7BH9$?xHDg6Dm_$=EJsF5jBATcVsG`w18^@HF^k@f=1|79^GZxbgh}J1k
z_RM2cQ8c`SQFxjRiNM+KY#VVl3R^-0lyo$bN1-Q0dvC~8GVyI_wdv%GAHEh!TX*sO
z&X-Qr&i=hMdH+8vd`Vf`@?XBO`}NoUGaKqD+PnTndT4Xfys!N?`=e_LKHcjQ&bX)U
z-c4P4?&vn_D*xQbf%=a>dGeW(7sl^Z-TK=5#4{r$WBTbw#^Sr`jIOCpSLRl?`<pk-
zzZ`R?-TcSd%Ue29Q}#8FHk?{D8~L-;dpUhy&DVFn+;p&EDe2%N_i`CWPR@JjX@+Z+
zkNxyU(P^z-ybr_Z<TBJ#HRFh_Kmp!N#z%2wAwm^likQYqholnfkAYcCnV%U=bkG*X
z1!ASrKoH>6yzNn-jA52xfp(>~0YMurM4rkF9Kxh+l4P1ro#sdY;us<9(r|ZA&#|tf
z9Y<Q4TUy$VD$1eOUccIQq@ne&>g(|yYwtMj?+pZe{#O<Cc#x!S)!R1^h=e2I6GNfM
z(9lq5&_5IohaxA2BV>(4G$j-c4Gj&RAa|Io1PTuhkRvcK5bO^I0)e1Lfq}sw-4I@{
z&+qf~dVM~BFDZwc4>i4XsI{&8)$Z=DSGv@m?kyTUm@=8BtdUruNuy7<(S<zuAhPlO
zm=*TZ07N}BUm8uoiVV7(Hjeh}uLOSXEdL?UNRDCHQ<5iIfR^Lv&bp|0;>VP^zjPrt
zZC~B&@Zj+4IkoSWDmhIzC!!5dOb1f4_fXVmHrV2|C*kpXBk#O`)78Lr51v>41Uya6
zl?n3k3M^c$-d<Bv=iXkmO_sM+?{K@_)lcr&UQ@fXrb2dA$_`hpELT*>PN&P^tdM2d
zQR$$COhissrPG;^NVixlR*S8i*k-ZXtYsFf-EJeBwcKuZ$hLBOxy8z}Wh}=RtPwe}
ztnmCF%St#&5(QCwLJ|aCAc4t~EM=0#DvKh^34%!y9Tw|tdnb(C!+fbN*yk<{DVQ8Q
zZN9e4=DKxfkMQnt%#~squLP+eT-zJktpe?pRU3c>GW6imXB{%L;9hqIv(oSJc&>hY
zW9k{q^bAMhx#71b>g&!M@ZQDSQwt-&i(^^YJ1J^y2?h_paSon=ikW*QQ@5`_x_Nc5
zEkFMG#NrA2-9JQOoF^2%iC%6v(mwFU6&L-htamWz&$BRCZ)I>`RREhTbbS{JM!HG_
z@EV?OUn%j51+yTMg%>%xPtD}uz;QfJ3d@5RI3q9eCZ3!;%ZeP&@}j_TW+Nw&nWJ9t
z^rI*<0eF^!SzB2X$I?a~tTC?CJ{?^L0vbl6hw%Jnlc@=riw>R_2uOPpajalAnW^<0
zPg`&zFS2G%(6ZEW0=*LH|FY!2&yq`I2|*HAfiGkh8HpDjBnd4^^J7@}5uHSvu$+n4
cDx+gKjt*w2a*jmWT+>b95@%7lS`h;GU!Hwig8%>k

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/2.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/2.0.0.0
deleted file mode 100644
index 910571a0231973319f71efb89142f5eb5f65ef75..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1954
zcmYjS3viRi5&nK*%NYC+*aVSCSm3nrfLoJ*^uMeJmSh=Y6FXrHEt<w*3T}DXNgU&X
z-TcBY*)ozWTSh4gF-_9iOdbZ36omvd#FJ?nPfG$NwAD0DVG@^gz+fnbVTSH0>9qIX
z-tB$2d%L%{cfX2Tk}{kDF#2^GkPD!a1Foxex@f{J#=iYxF;R>qbh>OzWkeG?w>u_A
z-6Y|Ty4}%4f*7Kx&WPzkl8VwCO%Rr#F2Mu|O+<;3I5E*A!@#5}lX2@7;f_A4qXn5B
zgNR5!8YSywKPED1=2pUeo|RPS!9m|jbc{o$k%sxiy0Ey_1hp3_Dr7y3Y!zB>q-r#p
zV2}|ttYtiHHV51+ARQcs`N2YPdZ^|C^MP!ns4*`{Gey7xzzR+cE*p@*sla7niNUZG
z9L>xF)G%lac3>H}W*`kk8qH%|Yq|<m6i&dY0`5dM%{;ShXK;M;;a2}we~TP$J-MGx
z$=}6oyiniu?#p{eKDjaWyIs=4v2)s<O}*ZyMwZ5Y)3Vr+TX{r37TP}k-nDHHzGPi{
z?A-g4SN$Jbs*eXQG{zPVOLzW!{k^9z_n#^F>%2EPUuxHD2Q4SAt}5U4$~Twa;g4^4
zswlk4a5C@E`g1$C>~_BK^|&f{`|cH9ZN5}r{m+Z#Kd@OwPMz_L&Aw1}_2|u>v#fj<
z)Q2djf&T?HKtX-TgmfR<{ca%6_O%=fFJ>^{*4#L}41sMZITfE!l0yPg0TrHz#>-Yj
z=HdpW79w$+g^fO7GLi$7L!(K<EEQ5*?66W&@^JgW?fOGk9)BY?XKYUEwVIpD>b~Gs
z?*6*2t!~|iRb6|p2EY1L^FoK~ygh9t_AXr*&HeIz@23^he|j@t|N4g9?0>tz2vl$U
zFlgS<?cHH1(VnhJ4NczqIPm(Z&sX<+^kVbOvEj{|tM}bY|6}zhKRw!bsc!Vrip#f}
z&)hqe-q~?-sb_S-F7rPw_>XMavu{@FhN7azdxbmSzE*qsk?H3@Yu0AGz4iXxQ>8%J
z#8J%9@na*P)KX9kzz_{D)+$&}J&hU=H*#tfUR5X*IB(D#X?OtB8SxbpC}0Q_B;wSt
zO1d)`(s2uv54cqg%1{pGk6KuG6`R7M+JM6<mPl<KUG1J<w(NQK+2<N}xBTLTXBvBb
zolVVkO?%tBySz$sOGme_*ROPVlzpAOf&D>Wx8L6v81N7Fhx$TcD*d5AcwitzM0|K~
zFic$kKz}$K3Jnq)IY3BwpuaB?9`Msm@dpBdU|%rMM|>#gO_uH+`n=sfU$@u$eEp7{
zJL;cnYU}K5R}@csr{_^tzDpzrB~k;CLdit8v%v+uV>=e%znK)JjZ-G&XMUN9`6p&$
zDlM`d-3-L?wrCG^9<{0l(BXQ?OUvWmAN~yRz0v^_+AlWkYtgP2v@CMt8{TTrY3)DS
zj}ukSwA%I{y$mNOI&c70Tk)?^WvKWMFwfwSA%dw;7VthW8%hm(Bc|?K-ol)u#qx@E
zYpW{C*Q{~5)~tQJa$RM`6IB(JRW%h3M`@Ye;ixWmIGt3TrB1uk;jovjp~g;QhqKh>
za+MO7EEbEERhz|XrOL=rVzUrrEh({=*jBH$T5W<<%!}d=b7WpNKl1Kz%QB(ZEK9Pi
zlO>5dS(c_AwaRj_*(Qr-vus=`+U!>A^{lF`XA87XY6>tu<fR-!;oveymO(CDG-0s9
z;3@i`4Zlrp<|n5QvheLsGyFI?#^<K^=<o#a`vpe43}21+?t#gl3mQ!P1s^qiuR&kX
z`uAU9&f@k5SIrI5rJXKY4=o<cgp`&tv{{Q=hl5vgZq|B6dNWqsjGrHxw4WF`OR4#@
zTDH9K&63bTMKy!9bjS1T9Q=9<hde8XZJ7ZKS^$xWAr<+ukd9gGq<IO&Nl=mmHY-Y^
z06dHvrJ~3SJVAmWke)13V0e+|0Uo?8@-i=pA_QKRdBH>if=Q%@nHQ+?CZ@s@M54r!
z1_T&E39|_rM4<8@&%|L%k<qw^-uYZ`QwBkhC7u_^f{`~dGe(*IIFd<Zj!7`l^#8k>
zfZD-L)kw5H^O0O3A0@ds>7&s^_G!6qeG~;*7R02Fv|AYxIV%b#(h~({&?HErBnV`j
R_D4RLK$QxmhA5JK{0GSLV%Y!y

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/3.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/3.0.0.0
deleted file mode 100644
index 7e9efc10c3ed795ce0fde2acbe0af91b82830867..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1915
zcmYLJ3vd)w7Ck+F5V9c=5;cYSC_y8#I6!9lb<eLMGx-sQkd*~@QR9b73d<saKt{}`
zg=Cn?Kqkq|U{KUiqefs?G*Yy%#sNQ|uo_lNV=!SX1ub=P5=(7rYhmx3Snln9@7;Ir
z>wC|+eY*|9O;`5+0pR1&DL@8*^E48xG+oy{_4+uMdb;b!bzQ6Hsz>v9*!Z}nF{YEM
zF>cUl(HT8LCY>A}x^(W;be6Q~nx-dXdKiD6PH38kt%Z<*SINY*<jpb>OS;IXQHT^=
zoJzSUJ6()2jgK>m^a+=%kPQLZ%`{#UkZPjsY*I^*(`?4oK3r+Q^n9cl(41Q!VC@~1
zm0OUJwoquEYCt7%1Ud}B#opFCg^thqk7WK>yXNft;xh~PnQoRx1~OCLI`cx;cmF9^
z*7<hLyS4hRr~AYL<%XV>pESO6P1))-E03Rk|J%_w=YG2Gy2w4lYrZ%#?{fU;7rXB|
zKXiQWh48GmFU<I1&Wu|&?q9s9=#*uBi+jb{y&vzMaxA>NMw*DlS~HIB*g85|)^lIY
z!IlRab4UI(u+f%s&pTJx4>0dHm!_S5F3&ZwXUfG}%PuwK%o^hI8{Jt%H~4?LL83d$
zjHv+#NxCr|(Iht<0CF&15v#`fNrMv!Fbp9YqxVZDRYA2%YZNlny099D5k4{l3g&*T
zaKDOk@kQ1;GRE>^=p_e9jH*Y6u(}$;0>FT(I5bqI;2HxaUO9jI#pzwr=+5RhzBKN`
zpz}cL@o#re^c{YoeuF>yTy^V9Z<M{cAnm8y?lR{+)Bn_+-KTtx8%GBA&c1Z8&Hv%w
ze%b3@A!5+IeeLS?16lw4VED|<HNXDr!sU~P^~d^ZZF_z$Jvp}SsylvoerDFQ4fh)p
zD-F^3bob}M?+>-^_*6ag@|>K<?4vbRSw~*|rSojZJnOk9SAF!C*V6uLYk9h9u5Ys;
z)Yj`)p%yAsha?p~6QfdvuSz@(Go!f}fU&T&8pAjUMMq*zdio^uA>2l#@Ti(H9E<Yv
zREL7sRm2ohzl$&Q)kP6AMyo0Kyc{8st0Em#RP^)UK(*4FT!&yqaZ;m6QEAGffjtJ^
zjcx5d@75O8vteVyW>3?WmJN+N0_~d`>l!`V{Ow-#kw@DCf#6R6_V#w~_CVLpE^nK^
zt2_93I1<_w>WPqqdIFJfI23ujm+y8(XcG>F!sO`Xn_g~;gnPPoMZ!CS-Q8WC!C+T+
z7fr$Lo^Br(zrVvD=%6{!;q!Sm)Ym;+->_+0yLW4Ai@L3S>jEynb)%EIv5tmH9Z4Tw
zT?w+C4<QS`x8wdr$2d2j4^25p+*fO&>T>!1>sIfwbc~&T^54J%WH9~w?A>E4fRjHK
z;`+f%W8lFjveqItKO_Gu%vcv{+q_@BT>i4{{PWg(J)`;NKT~a6d7wl4RZLd_H;yXQ
z^dHo6BZ_xaZ^1dBfp=x1w7kN-%<V2KEiWxCD_>r*d|AcP<?iK`tCqT4CB;scv$EXf
zbn&~ySybe7I*UtOq>zP7#U;hXE?Q`~><)*+PM1S**zI=5G$ckGirsE=IF>9aTC6Nq
z$irkSlcf2n7SUog4*uOXmsZ(AVw7cxua+D^vD<7`o6TY)VOFP8ah%#1S}b6it_&As
z#zuyT!r`84A1o;G;v-F|t29>S9eW7xMLD;8<HYt(1?fWvE&vfS$TRyqV}3wL6yjPT
z=Y8$=fB30&|C+t$8`Zw4N1c}6lwC1!f9jzsUw?|Z>Mi8QUjcmT*oYfx9|Qa1SKmZE
z>GcoJ!2A>b6Cch_Kf*tdJc`|a;+QWz2sB}-0OS@4=*+eY$aM%<mm0($?eq_h$&y4z
zNhFBb^p<4_mdU-$BxZpmN{nFg1B_HUEQuB{$s);2mShw%kug{-%*;p=EwY75Wo9(d
zL&-=?l)(xMY0FG7L7dFTB(rR$u*tWaYvzA>lZpQFi4fz#O)?YBqGS@yl$mNTNt8vB
zD1*o(j?u(HQZkOw`a4Ehp2R4NR!A~oBpfnK#Do=cj1+<3%T|t&oP^Ot7|le2I1w7z
aBuQqGCYiEPB4(lR1VNnSZ#ENn5&r|1`-`gp

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/4.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/4.0.0.0
deleted file mode 100644
index d8ff558258f1b7e888f2d125add55c027c01826d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2000
zcmYLKeNa<Z7XQ7VASx0USI`!PK-J<0+9r7)3CV*b1QZahUENN%jj3NNR<nF{vARAG
z0wI=!K&W*EW9mmoZ57wuLdBV9S#8&ycIbX|-LZ9O{^+*FPLZ|F*0IZ;n{j4O?s@lo
z{La1SoOg4dhRdGv)=Yr$ND?pyKq&{@EswgRG2-FjVH?q~d)VgI>6k=w%<Z-fM|HYr
zH0sv5V+7RE6eHF}$w#b<kxEt_8EIlZilPyX#iG$1cqqtCzq!yx^YO!Cchv1BJZ%__
z5h_Ml24FL};`_2hXd}|tet%;$6`A0Wxu5tVhZF;;IV3u<V80QGZJ46LtX!mOu)kkb
zWmT4CmFDR{DwxiQtg1Q-Tm}--ke~%O3owA22QCqK9GYZk$WjK*29T@)HysJHk*z4p
z0Xsu+g$&4*0JMw^`KtO9w?A2fN@^ugqX8bgesIjv_j=L%`+*-9esb>3_Txt`rl;Q&
zAMW}0Ykyf{f3R`!oYS(;ocGs+Uh$jRJ-$=kMcY1Dc|zZi@@b^*NcGsauim@dUH4sz
zX}Rjy*Z7a;LZ_X-o$-!iG<w~seDRwd!LK)T9T@)IH&@T)-aNd1>}R7{{r<q2mwXQ=
z&I}Aav-<3DxpUXj_B+Rtr#Zj$xo+j%f3M=7NB&fB>#n}_lb^hMD#dv9OXmoqN9}x!
z+8OwNI|J0t$Ban!gVP{27f6%mAZ=h8#*^TR#ET3r6L5i~O2~kmNgf4RBob9ZMQ}p|
zqq0IS+)Eq_x*{3Cg&>qasG|(B1)lX3B5UDVNvaD9Xeg@8gmp;N;HSzAe1BN?YtI5r
z%73o!>U&!GV0_D9Nlt41I}7e^-+p#}PN*Yc>)1=%o}8J=h4lA6zi~QztfDXJfcyNl
zck@4r9g}D7YkXmmWwy7!sza~udH;JnzvsX&C#}y7zr7+d_xG1R>b;EmxthtEKR<A;
z?|=4qrp}kJ?DJQ?aD4jrCzFrvt*m(BkNWxUhu3T;v-7VX?XM4~t<-Ju^534gwrOH0
z=T`5I!4?0VF}Ya_WyC{`&>&5#B1=&p=Qcy3YNyOYze|1rX;dHiwb+cTELjQ8z|}j{
zpsy~Bz@w<J;%=^%ia!TLB2a2&58(+d;}R1<P^eihLSTfXakwZ*RzwabIC?n|MIp>6
zG5R#X4~JV?+FDy0mAbl`+WNiw_BGYlAM$%WyZ6+1nta}tgL_|XZTBBK)aX^34z~IO
z9f9_?woqpv6bN^Pf?eI=aCcXzgFVCDq;!RQdPwQ+>hA6dheACh9wr0xcapcGqq8&E
z5eO0m=-JuX*%l{Xd%NF9<oC6;)$Fd?we#ibx@K=nbEDGO)ZDm=(W7%`r*r2aR_N@}
z-`L|sro9UD5I=aYP*GGfmLl44gL=d$I~@}*CU#qH$F+}p`xZ4&Yi@Hxz<Xq>+O~H0
zj})^dcowHIG3i?|?Xy#Z=f8W}gOffaefE}e<?W-FE2`f(HKH!7fDg`JRk9W@X;w3?
z1c4s9Ogv-I^}U8?0EdT`tgVF}n4XM@L6nu3ud7(MrlizaT2fL`zNWN%&APRfm7B^O
z#j73mV#k`bB@QCH!$G&E-R>xMI_)+a8OY*v78f%oBa77%A68kmTCB1)P8LgnTv%9S
zlM9OqWowZj6bPcYN^9cHX2Z2hPc7w_m`svnGA%brqRDI`Wsz(tC@3%&nb}kzm<uh1
zc1uQ?x7b_hy*Y5EAsYpw+VIx{zqnKegi8Flnf_t3f57rFE@t#Tv1AohK5OQU25x*~
zsCw(5_!WH%B8k|6$)$^{N}tEj_M{ivaek;to1d-(ZXcbx?fvN+Umk7UfPZ0i_5m5Y
zGV%g>8CUY?=wc^UW^6x``eCfDIs4~#<oj6%=?Ej28dz;;hT2ZT_<H)vEVOax$h2@+
zXyvdoC4iF_KyM&7FcncrG{!|pY4oB6qbS8yQIdEQ-O{2Y05l*nqA1=VkuFK1APFKb
z2m%iic_dyhih@WY&l?02M3@8$(^DKdC{CoGQGg&4lg87Md@VFCcHJ*Da@d-0Fzm&g
z@h*uLqos|!(a1Bf!6;D@JTJs^AxShDMM{lGlFYoBk|g}2Bv~-kBN$i)l1UOMmZ!vc
u1545*=vm$-T2FYCB+E@?2=Y=|29tr-XQh#WN)ZGJv>wZrFw_nbjQ<1Gih`a1

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/5.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/5.0.0.0
deleted file mode 100644
index 91e117cc6655f7e20e9d8d12f91f3c070451af6b..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1970
zcmYLK4^)%a75{yywpc_1QjdNJM!_>}CsLMt`Tm5Em=K^?RHz%v24WqKqjiS)uZY?C
zMhGNO$j5&cdo->6qt<q*rt|n`Go|*lc5`DM-J$8$!sa|Zr)N8$%(m;sc3<3|``&%`
z-gn>oefQq?@}lEb%sDY1V0`8wAQOO-1E#wZZg(Q=HpXH`;#e$fjD_9qn2}l{%qK|j
zVRs_orb)M(Pb9+Ousci+Hy`HRJb6fv4D-z5nUl>Vc(#I1uwA6F^+XBgOOP6-+4~eu
zBrzFUf+vIL>C3De!#tnJ!P>+P4%r~~5})CaW+F3})KcVl1*C<Lb0ZH6b@1MKq^4#(
z*=rb$k5ebU-$+eb<C)Ocq=L(bZa%nFU@qo72+ja51-cAydgvB_TZIR{2W}}mH8s&F
zVH)tj9O!hIn+Dx3pb`rUIqy6jHc=IUeLCRF5#iIKs-|nt?4MaO_1TNxe5c^e-{|jV
z{e1Jx+SgvGIdtasnx=dCd%wH#V8htwR~Mw;aoi5>Sf*@Ed13P7%kGv188=RHd$#!A
zI=f|ZbWO8k+wASS+I@fOz=^z{82Z0DcjcPim)Y2263SP9;?FzfYI*iz?8~$dE}kr$
z?cT7c{p{Vh1J$!5SHs(LFKE9Nm;PbxrTC*aKKj?V3%t$Ndrq78)bZ^d?lDGl?^iR6
zsu}u!HA7U*ECKVn!0E6c5B&HdOgl8-R};w;)n$zyrd%@6kis+-m>^~puZOy0b0ng|
zhB)G?&4a_^&}*2$XjGjb6q?o-u%-3b#&GJfn3dW=Ft&L})nRI6bbF`alMg!^qV4yD
z%3D^?@u4m8;#Y@He~=YKdsSEE!|kD$HJfvH_nqjG>)Kl@y}_A|xtD&r>aSnb*53W1
zYDL2~XXOP?>oH$lD*tBP(TeK)$A96?bA5QXDSP4kBU8?{PybwZ-eGe~TW?-(`dqj=
zp%qnM`M5FV@0U&}r$@qlCof-Y+_Bo~i#C|LAG>*a<;-jPsWWBYAD#dF7YBxa_g?vv
z3I(gi)40*(smE17uf{3O$TfIW%5gQ2ioc~q4AJNmrTBiFhGfp9#0|T+sl{rYsu@&5
zA*2Z^TB)Xv8OWEh3~H2C1Gbm+<$%U{>d7d8Ar0|H6^J4}5z$o8<<x;BD<IAGChuLl
zKhV*x{^;O={msqIhrGVSEzfm!?AyI(@9x%MyWe}j+uqsL(-Z9Q``S9Xx<fsFe<0Kw
z8tNbHAMPI*B<k-A4h{_T4GxbC4Gs(r(J(O3KQJ&f*vH%hBZEUs^!3sZ3ib4Ap^z3z
zj#{tQ8(<U+b_TnGL|wr^;MpdR=Lb!D_xsv>K2=p){cVpk`p><iHz(UetkT`1U$LhY
z+0I>9ivM1`+(>OE(sNTVz3m@BD4X&G-VJ;^r<>$cs~!YIQl5XhHQ(?al|g+2@7kBz
z$6gpeP&@N-+~0TWnV4n`+B1SjBmVTsN7p@aZ^>oDFELV!0Ah4xI5<rE&&A^fI0C%L
zYKsfsX$F3c{YcT_&NnJHY}iy;U0GgQT2{8<$&D4|<z*W;R&M^5tHkw0iPKf`L`9ji
zgpt!h&FONvN=uy%GM(ftb-7APa<IZ$%wc+`&1NrVWVKq?+wImOtKDv2@37lQ6x)g{
z7PBN3@`~AFG5#~gKW5g;ilit?jv`BnMN#C!9BWaLrO0Bn7Ly>g*tT9R*gN^PCC8Sj
zSXX&;wA%5$?bK&m533_TL;mI8BcPT=p=!pPm}*?C{jh3?a^2HLMcc!QhSqD+N5Ce1
z3=dUo9GiTn_SAWO)}iAQf0-@(qdnv4vuj?DyfL%2wna9a=jn|-R<t^T6Dl%o&kb(^
zb{ysxX;&t%q}0dbnw;L%ny8YE!1V}WM}9o}b=(3xD4{=<<qi%#*;WqAsSG>miC$}^
z7d%%M7a?6?M=Jp`F_9z_kyIo)SEiUHvnY{I5X=xIQB)L363wD0QnAQZWbz6m6_TRa
zBoP7xYGBJ5ppk7%UK10I4Ay-AXqO8d8rGRihmbk7m_8!PX3;DNvScP`$s~)iWD-P3
ziU`7$VUnjYwUeTfWram%`Co{%F&U!SOlhzX*~r2n97(h=Bteoa6rwDOCRvaaSs+a6
fqC}C(W?2--Cy9b=CPC{(7ACn&CXql{`trX4XzYn4

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/6.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/6.0.0.0
deleted file mode 100644
index 768f921b4f1de6589b608868251a50e5eb0e5667..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1992
zcmYLJ4Nz3q75?7(a0NyAu_R)_B~oKFRtWCCzh!ysK3F8E#I%eP9bMblBGw3q+7iKs
zuDdMA?y~GAgv5qn@?)$fj?qfSB(teI4Kc*oLEC78&Khmf=Ep3=nKqqC+jBRW>3Q$o
z``z!Jd+xd4oc9b|5r1qBz}lD*m=B<V1E$(ynmgv!j7Fo^>ot05<lUN^2BVjcxq05}
z)!Z7d@w}VdVq`oUX-bXPVp@!O(Ga6JjA7nnZg-3zYi>6SNEB_2F*?Rzi~vt472Jw%
zvnYzDkr!pa^6=75_LA=vW@#t@hk_1jr#WPrX}Xx86NMcX%<6%`fVst(XFx}CPBf~j
zD(j-Dhe5_Ln-4=ec1IZ#RTl!Jh5!@-JYWDf3s?ZGg@F_zemyt?vZ8a@K(5?O6zTed
zKo+>gFc4S>j%ZpGFz+F*Bi(>{(iPZ4)VG5#U3hTh#$ec9_RqwMzKfSEZ)RE@jZN2m
zmIs{Uk4~(+GcH_o6_s^=5=yAw^gSFn(XgT=oV_{Gec@bNX=2xx!=;nkZjOJL^Y-WG
zCvT;BPJf-#ml@l5<J~QM>wi!9_T;r4Y5jA?qK5-Gt~)ojbd>S4Km9{sDDc*epA@{m
z?B1!N(CJ^i_Qlbg?I$0z@A~e%F+Kl<15@IQBcT=la24ITTJrwiw_J`h`$5`;ByHgT
z(gsM{g%+gurtVivGNd8RfW=GoEaa%UNE+1y^we!pvj4AyigBE#KgQHd9dTVvEuCV$
z$rX(sAI(S7V9JLHXNaM#vGk(5rXNupl5C8*U8R=L83I(~qd5r|peJTyt^w17pZ&XV
zvm^e|)9uB-Ja**+eq!j--Pc#SOaF9nbmgX(D<(~21KFQ0&$B+#l&!v@Z)^X0+r06^
zwI{o*m)|>he)#jzYuiem+<vXG;o01@(VTGY)&&jT6+0t0_icSXw$`p@yuGG&`1a_(
z3X@kKyS3fg^6K-iE?(~+3u>=StgN3x%?4$>H>a+(U?QXDa@`fHy6K(Q9T!%8_1zu$
z`m<-ZPX79d!Nyw;QQlG48^{?pqP1Q%>6CmXTB2~3=x}A?ooF<8fP69#1hVm{9#o;v
z=8}2j=t8p|Cz?#v5k;IdB7AJiy{8yQLhmF(Z$!3UNy}JDOnjL5<O>{fFpCOLCpSEo
zDm;&;+w1A-+_(3)ot?jH?>w+u@9pvK*}h|U(?Ordqqnyo_8sXx;&JQxAy2R0AL#M*
z_Vow){gHuCU?4(9V<a37MFzr=r-*neLNkUVp%BA^%qJ2a=zlsA4g~$7e!o8$=nn=0
z!C)xZ-<>j_&)3sKU2l)CyX`kkPwd#)viHzo@8PblgI(@}rObXPJdFyknVL?;N8fa_
z69w81`VLc-w=WD_%K(;R<jST9yCU%JJ(${9-;B>vGi6d=Jp=SVDC`5%4GS(kxA8-n
zfaw{O0r$05MjMvgnlOz#UT?blrtJjupJ!v!sV_#();@6VC|&`omX0A<`X7KhM}L5s
zRKzFZcniOWG`zpFZWrblkUYP+uI`a_^*>uzQ&a7%S-WOk?P{m9e(jp|8>+SH+A6Ke
zS+m+%U0p@ybZ8DutEzTZS80x_DuOA@mdLb<3R~(?RYJCkigKHsP`PTath8%(wbHJt
zj&h44$Z~0cB3c#2JTYA&EK?LgQIz`>S)xIa%kpjI<yJBqnYFxJv8ncqB}F}R{+J~u
z;vZzc+}zU_#ovZ1qa*<=QyE8Yd-9s54`7D=5t5BRz3Dr|(}l4t04!mu1&;RJt|d@7
z3=z2h8{bTK_w|pXXReNv4lmijQ^76A-oFgJ_Fnt4Q42l+w$$n1D@f)Uqb9%)oSM%i
zUd;Vi^?CQ7Y)(JB!Q}$lq7;(VFS@8|Gs-z&k%NQ3z{X)w1&5~00M6JfSS3g)$25${
z()SQzE*ylVbWGRF3XuvFN{xQZkR(Zz2t$-uwvt4u$bx7V1WB<-f(#376q!zvC<vgv
zvc&?kLVG3ALLI$=NTDdaS&*b=%r;=9XyNdym1gsk$ek({MT#nlg2h6EAc<zkbPDY=
z)6sN}DAP<fQx0ZYuu5iFH7-lkxICFEx`><3mSkC>2&tq5Sx{i19in1QB_&dEamgZy
dZbY%ZPf1}%$`U(Jq6JAHnk}(2%q(X?`X3$CX9fTO

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/7.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/7.0.0.0
deleted file mode 100644
index 95bf5ed3965e053f914c683834f33afa6131edb7..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1991
zcmYjS3vg4_7F{=`0Scu-I#?-f!!Kg#(^75jCndejr)f(|ZAGaPH7Z=RrILbDC<bor
zzoAXqCPfh6<n{4l$HDJ&(C6T@49wssyr+Otm4OaItTTvd8D|7PxKCt8_suzH@3r^Y
z=bUx+z0q(BrVP&nSUWxyNCi;F0TYWA#So5$Bju4uM2SSoX^e)$q(sUULo`Z?f$V71
zz^tgEM3tx^2@~0b7~)fzSBbKSxK{})VVc1@MG4bLPWHq{2AH6ahSM>jRg`Fy07@Gs
zm~_fq?!`4tdAUKM)C4Go(@YYP0S*}+YHx8!&L?vbsSad%jF|2Pr@^enNYTJEnRwQD
zmV|XqCk^o@6$u(7=rP-wgla;Z&gnoBkPPm2U@C5LE`eqSI4!s|aCbwK2uwr5Qa}UU
zyJT!<2~Y-|lc<{llmdCU>#Q@6^Gw&^Ug8pXh9a)LFg9oCoVrmym@&U<+3TV9&5h$H
z5=QS_RrA5&e_rP=UyFIyZ@u1jCNTQI`zPzqcXlPZkBol#dFG}*W6V_Zc=OucRBzMO
z{uA5&H?t3A|ISG?SR2muEdEeGGvz5=>9O5T&%{{2tD$nk6}P(E^vYz~!j(I7YJ>0e
zEV)wk{nd{r9y;M&+i^=|vOXd2{kk(_`wnEDy!6@EUp})J-ec|lSl5rIdvu>Nxyfic
zG3xu1Q6Djy{uf4%<ia068)P68vtwFR1L~*pbm*#y77eUw2<i+>rT`<Ls1Sv#DyG%^
zF@`Xrj_wc13mJd+{lvgoaF6IokySHbVwc9n1~Sf9tqnMuj^g2Wo?9}W{oi*R*M4T)
zRQ0dojqh#w`z4F6Ze_vcsgWFO<)Kgh<|*^;eKc7&nLk)-$(p@jefB}Vqw?3+>ONkX
z9lV&e^Zy%D<&J$#-yholiL$R`WW}4|M?QLX@v`varw^xH(Ox?F-H$nW)~pYU*X1>Q
zRQU7D0eM06Ki_T5y#7n>9p$A-sU43jd-h)4x94Z)CpCw!9KZYi$%$HX&RE?c=tE;#
zI;5U)!A>i?b%;%MxquQtpG!pPm?PwJW8?;OTeBEHvGg$)BLu3<=1~6z1j<1&f1T<g
z9J%Tns2*OUg6E(BeopUF?Nm-&Y%ZIoQcC$J$nqG)lJQpwF(ag5b=TI>(caPGY23DB
zm%6>F$+LNvx4V6N?b8iUwRAPN?A)@WwR?9@Z&!PJOIwGxr`O-v=I!<Sf`P$-Jp+Lt
z$v}T+Fc9b;d|~h4U~rIHAP^wWU~qt1FtCTk1Oxqjdx8O<-{<S|QQznH_xk+(zP?T-
zT_j%Cb$54mZfkJ6>l?OgZE0<NzFBQ<Zf#x8<YwL3sP5d<RH{9Clidzv*y?Z_;;(We
zV?_m)VQBriG<HPb!mZFhk*&qmhsd~Q`G>BPp^^(=MP={mzo0)0%ovWM4XKNSlQl1*
ze)QBO%xk<|{B3DPL5q9-gB7jE%|ihfHf^RUU3wtl6tQ;!!(?L0OGn{qCa?_CzM?Yz
z0`<_+{oOE@uBxhBy}HC+;&4>lS6NZ9s?<@ss-n84*k&)YTkXXqB@P?uc6+hiZnfF$
z#l?1;%}O?d90W2S_2xpe*<4siKkArKC|hK+SuQFnvRH~Na*<iKtk7mil5}U9Bovq=
ze%|DAercY`BuJ)wf=wh6KX*mJ3MMQUQ_XVW_?#52zw=4IF64c69)EyNQ|q~r3tYy5
zA4Xuo$Q@M=1Ir1|o@q_%K@Glvd1<_EVb7Q~{=nD7!*Dtpu*T_b*?%5o8_safMO`2K
zmg#y>UJ{wV?)eJ|149Ql;c$?WRO|#+pMw1I#qkp0ZCCeg$)RHxh5ONNUN~*d5rczb
zHc|uL16)V-z#cpO(_~pW^k$ejWEFC#P4?k!`T*QR50gX;NxT$SA<n@ZF$J*-2$Cr9
z3@k7L^n)mhyeI&I$U`uaQ=~~EF9;x(NM~M2B%VZ`7buENp?M|&`ILxdpcF#BQ55U(
zm_#V}Eub%WmXX7Tto$8Fjm;GVfmY#p3S!Gpl?f8<On@Zj!(^n(Aj`pB)e<bR2oVJo
z#v(;YvT<&K)}#!(xR^zvTs+$v?TV*7w6%giw?-Sdxius0loq5Qp6;3DVT+SHo;Kdo
IQNatp13%q(#Q*>R

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/8.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/8.0.0.0
deleted file mode 100644
index 49a810879cf725801e6c7840b866166780de87df..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1981
zcmYLK3s98T75@I^p+GRJ>x=MO5qFGOsA2bi?E=5B3(}$Z*rsZz$!gSDiB&RUg4ngc
zx*!X>4}KQcHnrQ)8mB`X0!>1aYPX4Ysx^jWk=8WH5Ve*(29ZvtakQrAu9@kXd+s^+
z+;h%7_q*qxU&k#-KK%&5`0O-bHh?+~xbs8J=`_R*+9{KX4Pukipd}Q|88?uk88j^(
z*9`HvlgYTjV9+Q;GdK;-I7QR0lPMb6X~c;rknBz;osy2brIbiZI2za%^I$odiQLJL
zWh4Wc<8&S?!KAVD<e|k!7^NT=9CAIx0~|6fv|LDP4e~rZrguW8!;C^as)I*A&f!j~
zd))0FA0H=n6D211csig1^1x*Snb4JhONDL*Fb_!u;Pe3DQV-pH=t$oL%z|-v*bEc{
z<h6uFb2QIEQYw<ks}T9_Cpb@<4yy=FpoOyEjbyBv7e0AzOs(|X59J@q-<UP`e%NCg
zP~E4_c<atKcaDBo{NGofeIS)4y<pB+{mQAsrn1(;>Su0WoiasR^=Z9^%O_s==*qV1
zUq5@J_`TD!HXRwrJJ-hfj%Mu}G!N~{+V%35vm*<KCMw=auYdPV<J0dn?|kPb_lK8;
ztD{Z1kv?bl1-<S{-8Y?q6P(55ef?$2gXza?g>T>f+lBW&IhcI>{=hQpFr)7f%_c+x
z|3@@Hh-ULh>jIZRG)0G5(R55s`#vx~amiqdx}9mWgDQ}NJCD28DK01qKx_O3aK}h0
z#H!Ll^FN^)Nz?@iQz$V6Nid3*F+8i5s%k6)PQ+l0W*}XMfA{&a)K6~Ab{yHix&PW#
z!=$U@edCM$Ba7Zz|K+`r1HaAqY(|4?=F**yW?XvYd)FF2+|XRyc+2WN)3T>%*wQ*D
zVmWXep^9_n;=vKccmDI#-+#I0*4O*D-`PF%lfbL{)(u>~`N22ukLJXlyLe&VhPhu|
zxx7#J+Ok_`^5Z3&?&cJ39DMRYq-D;s+7(|s^Vr#(&3|3O*^^7Qy0(09^is}(dhhMA
zb%ReO_p5XPBbyQzWeXTxbmNQI=+ch?k$d=4Oo6(Ywrq6cJWQpiF_+2!NQ4la#*d+9
zV@lT_Re@2!MO%8NySbStf_^LwdNl=;b3>|&fva>7cG0LxZZsZ}!i0-h{iTleUPt?(
z_VzZlbx-TwpSQNO{;bK<<!j%wt+{#YL7&&Vx7l;B^KjRZcJG1x-i|K6-`~~Q)g253
z{b8-!-xKZ)lMID>db){w!&+}IS$cYU!nEk_W_pCC-f)i=iiEX5DCiFb{DEL77z_l0
zS|HewAfM0I<@0qCb#(07-uTn)JD%U&*4E}dxSvivu#C}ny=SN1Ya&*u|LALOszI*(
zIb<Qxd@r(_#l{GUf`SW~DER2obZWzUC$X)cb_%=G8jcmwy{{hbWo<ZB&E;li@2y+}
zWcT3L3&GDXMbX%im>4Xoon4atpFT8{{0jcRU;A|8hI--$I1L=8mvB$6jYXEep;AXr
zj1K&0aefnZCXVlh=@(E7(12+=OkJ#9xq9W3YaEW68pp~XudZLcs=j(fechV%HC6Un
zhuu-VqSoOc>9E(@?NxR%RNEc)s)x4f>KalKWG`n_Q9+|(In5*$rJ}5?LRr4NvQk-I
zsVHRrcE~ErOVfGLX0=+&P%0E<S*<eZqD*AvtwLTIBO41WFSpssY)X0Q?^ZnZud?e$
z)o3(z?|lLYi~oFR{o=4i-Svfvt&{rX`4;?R3)%ZVUeb6mD#A4S^wFD`T$}WHFy|P<
zZNYVnEQWWdEoK-Ct4gwz^!U(q(+zAmZ^uO1j(J}-s3%)7s~#PALT#G4))gsvLmdJh
z%lz!qA&-6Hl_>YpiABn92LJdx^>`nv;S`~{DxzR`N~T{;eia9QZaIhi3J#5#0lZsI
zKD^n&T#<qTSx7^HD6w~zcvzWTkYq`g0kbTL58s&#k|;@lBtVd9SRwMFh37>{76pM;
z<SCGoAeco7<j6w=FG*%W6wN#!Q#4Qc%@$sueF<jq;XCK^9M+duEIVNQ)+CEWvMlmE
zFNp#xP^P6IN>qsjqCig}!Adnz1}f1iLA1(tE@`qvq&H#*@B)h$L{^amfgV*5Sw%8c
l1KGmM5_u#lvQY{;MY0OA7~=HO5I56uo{LhAR5uS^{2z=FcGds@

diff --git a/tests/create-pipe-reference/create-pipe.zarr/data/9.0.0.0 b/tests/create-pipe-reference/create-pipe.zarr/data/9.0.0.0
deleted file mode 100644
index fc69f5954c4354e3f9393f61268d2df1e7fec799..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1981
zcmYLK4RllG8Gde>Qrc1s+%hWKk`=5#VH9j`a_>!(+cxPB7WxyxvM~(|k7sg(f|dSA
z<}HLaX|Xr$O;f3?RAmfNk@+(`TA(@=Hf5*Bx}%70*<m1sjinVB;Do(j;LJDo{qnxg
z`+m>;zUO{7F>pUN?wbTKKb!=l18{LbEQu%6J-oh`EGO>#2%nRJD|x(!=X3Vvuqw}c
zbdOF2kH<4YV>~3>L-B~tFoJYKnGxRz5zfIi3*pqF^Hd;`xKqM2HoDj(Hj8nL=<HtX
z%F%gZ)^&Y^|BP88GQlC!NB%m8Gz+D(C@e*mPe5WD5)GI<8+RDs8=qt+8~ndhG)P7W
zjVYKlIG6^LJpohfQ(!Q_Kw1<6#Y_!D5x5L+so?B@0SP7;Q(+_|1qo@O8^Q>Lz^#B`
zCKlT5i#cDi0ac_TP;UUPJ$v*K?c4DaJMYdt{BB>@-IdSXXc#sfNIAFWyU!Q&9*cc{
zY|mf*_{j{<<x?B_%JP4qhss0K&s0V>{%Lg4@YB9e9-Ovv^vgj0;(z~aYQi(`)x57w
zK3KW#Ki={$gt5sj-|p}4^sXIEO5Bsyw(yP*Uy&|<pYze`m#Rhw20kjA@L1jE(+}Un
zum51fy}4U1&b^R(+Wq>A|4J_#>mL7Xu(Yuva^~>Mt)m;)-)|Xac3aK6NKNerH8oQ6
zE&<6M;NohMro57Zm{~<xe2?Z~b<UrG`7p;*K)I?8u`n)y2}5Z2R-2;&Vp9}Qg=P&<
zbs&@hs8M8iDN)I(aW>Qb5@qpRA;hG@JRL~}T;A2XX|M0bp4LOd*^Ngl4!$=1O7huR
zxrqm!fA8vB-ETLa^+?8-o&3@dQckb9UjM87M&IK&eXb+B;q&>|vrhc(<2ToK&MSTG
z)vWA8wZqSiod5fpo0Zzi+M2zqnu=UUzA4;&ZujzL(+$`0(48V*8?qldcH~U-_itri
z8(qjfRhPA@QCz<Arx)%uE?v56oqAQhSpB!-qkX^K*`KkxD`!FUM)va$CmY@!P~c_z
zP!C~znyL<)nIuDmgj9Ix`@a+dUXyxB2M($V7GRAU@S32`<HmlZvfh7U-Yp#)p(uzV
zmPatWQEDC{*;IiEovsdWF#}W;JAmOl)~J3F@|xLsuZmK?`PT+^?)+_St=qS^s82OD
z`<gd5HPvnD2(~q^Z`$<ew!k)jbA9vn;Eql$(6)VRYp`8=MhmvJYay*C9MOBj;hsoO
zSnqD{35WGa@6Jdh9AOA$VT!uRd&0ehu&%D&9$o7Wg|two7U~LhX_}^oLIGyMV6eTt
zJwQV+7--nIq3-dzO;7s$+qSl-EiL|Si<sR$cl4W+Sx2tY*`v3(t`wQBb+{ApgV#X2
z%D@ujB7OcOOudqf@zvkekzC9iPd;=p7wD+1c;(hnIzKQ6HK%~DvVfYNWhc$94KJj8
ztnBcAIu-SO*MP`FB{0{XobX6&aXE61<AQMp@$5K<pQAR9QEtUqb@Dz6euI6Lnh*QO
zfX$d-K<uVav9!AS!RiusNr}5=MO9UG<uZ3!d1dv=^5T*uZnwL%tfIv2a=F~DCB?<X
zZkO9#LQYrQY0_<TIUEirT~4RsR1_8|^yPFE78Vv27b%5>ic+XJ=O$TflEt*hW|!@`
z4%}y(oh8e%U6ut&l0<oi{F7<+f?MV+DsU7~;w;!78Q<Oij#6ZvyIxx6XPT^NICWC-
zeE979zXbfLfk0|Mj1SODsJyYcK4JDzO!}*zb{nD%pHJVKzlO9spAPUnYfFt=b{$-9
zHNG&EdLTNH%PfFcHGu4Oi3@@=?tcP~e{@VYlggNQ6Mb(ixA#$){bt@^*`-%^abJr^
zoF}yYQHj6<0UDS^=ctJ7VSg|)i#c>=IylU9a(FCF!yg<}7xJy~zC`fiL>QBCX|-qr
zqX0xnlw?2>M0yV_1lS0$*{DEsY@*d>kwsY)1wpV%u-U9KM5<Ut)?^bQiLhF!8-fk=
ziQk6+nn+wWh&EA-Gd&HX0W$?!-B>;m5H5%!wXqo%qLf6dWD`VM6Gg-gnF;~{qHL#x
z+9_qLljEzq4*JcDnXPPXlGQ5E^CUVM^!#W8vP8seZM0stHX7KStVMtiF%z-L)W~)&
PF%hhcn3+Y&hamnB(*<g{

diff --git a/tests/create-pipe-reference/create-pipe.zarr/dates/.zarray b/tests/create-pipe-reference/create-pipe.zarr/dates/.zarray
deleted file mode 100644
index 9a6e5f3..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/dates/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        10
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<M8[s]",
-    "fill_value": 0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        10
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/dates/0 b/tests/create-pipe-reference/create-pipe.zarr/dates/0
deleted file mode 100644
index c04549e88081fd35feac96ed26ced48006882a1f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 96
zcmZQ#H0B6kU|;~@1R!QO{5l>)9mswIrW-`wf@z1n5Sk(F9Ymh*Jw$#xgmy^y0Om7r
KeuT(xgwO!$(iTVn

diff --git a/tests/create-pipe-reference/create-pipe.zarr/latitudes/.zarray b/tests/create-pipe-reference/create-pipe.zarr/latitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/latitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/latitudes/0 b/tests/create-pipe-reference/create-pipe.zarr/latitudes/0
deleted file mode 100644
index bc43c5131b3b14758aa08ae329fc209b28364f0d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 179
zcmZQ#ROAp~Wnci|%?u0-B0$Uy#PSS`44nZ$235J5K;vXX7#U>!fY=3yO@J6^6R$Fm
j=?)aiW{~vjX9x%Z0gzMV9T*u*Kq}-9Ff!-|8~_0TnG_Rp

diff --git a/tests/create-pipe-reference/create-pipe.zarr/longitudes/.zarray b/tests/create-pipe-reference/create-pipe.zarr/longitudes/.zarray
deleted file mode 100644
index 0bdb72a..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/longitudes/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        162
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        162
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/longitudes/0 b/tests/create-pipe-reference/create-pipe.zarr/longitudes/0
deleted file mode 100644
index 3fa39c0fa3bb6f35f29e18b16de9bf93759c73e1..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 199
zcmZQ#ROAp~Wnci|;|vT8B0$Uy#PSS`44nZ$24!41pnf5sQGyLj3`q_R2N)a}gn^bZ
z@Cz~I27vkcKo$Q*8BARKLL%c5Q_?cC^9l-!N=h+BR2b3%z*77`EjkPij384g0vtd9
E0Dg2I-v9sr

diff --git a/tests/create-pipe-reference/create-pipe.zarr/maximum/.zarray b/tests/create-pipe-reference/create-pipe.zarr/maximum/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/maximum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/maximum/0 b/tests/create-pipe-reference/create-pipe.zarr/maximum/0
deleted file mode 100644
index 62c70405847f523eff5fee1ae1eb110ccf332f06..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
tcmZQ#H0ID?U|;}Y3m^u91S5-L2QZ7_gFTowoCv0U%}?8b_zt1-?E#*<2Xp`c

diff --git a/tests/create-pipe-reference/create-pipe.zarr/mean/.zarray b/tests/create-pipe-reference/create-pipe.zarr/mean/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/mean/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/mean/0 b/tests/create-pipe-reference/create-pipe.zarr/mean/0
deleted file mode 100644
index 8ce0ca2f74499a2f835603a8372a5709f3e5c5fe..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
zcmZQ#H0ID?U|;}Y3n1qCDR=o;e4)b<9-F&|DxcUJe<;beyJ2Ym_cPnWIV>ma{_k<L
J@ZRBU4*<V&6nFpt

diff --git a/tests/create-pipe-reference/create-pipe.zarr/minimum/.zarray b/tests/create-pipe-reference/create-pipe.zarr/minimum/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/minimum/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/minimum/0 b/tests/create-pipe-reference/create-pipe.zarr/minimum/0
deleted file mode 100644
index 5d0dcfd8d08973f11295b50c9ed76c478f8bf77f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
qcmZQ#H0ID?U|;}Y3m^u91*d&-9DsC#l+`hNkN^yvy0z5~#s&b%00)`?

diff --git a/tests/create-pipe-reference/create-pipe.zarr/squares/.zarray b/tests/create-pipe-reference/create-pipe.zarr/squares/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/squares/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/squares/0 b/tests/create-pipe-reference/create-pipe.zarr/squares/0
deleted file mode 100644
index 4bc168a54f59500d720cfe49175c59b58c7157d4..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
ycmZQ#H0ID?U|;}Y3m^sp8==2*9l@-CP6r_Uvz|H59>mXB<zok=?(O|iZ4Us{K?@21

diff --git a/tests/create-pipe-reference/create-pipe.zarr/stdev/.zarray b/tests/create-pipe-reference/create-pipe.zarr/stdev/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/stdev/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/stdev/0 b/tests/create-pipe-reference/create-pipe.zarr/stdev/0
deleted file mode 100644
index 645ebb6f95999db67a4064a95fa5338fc39e1a7e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
zcmZQ#H0ID?U|;}Y3n0D~s90C4V(c)x`}eDRPM7VswS)^+Uk|XKs2>%c+}3U9xc0z;
I6(R}t0IMhyF#rGn

diff --git a/tests/create-pipe-reference/create-pipe.zarr/sums/.zarray b/tests/create-pipe-reference/create-pipe.zarr/sums/.zarray
deleted file mode 100644
index 68f8ea8..0000000
--- a/tests/create-pipe-reference/create-pipe.zarr/sums/.zarray
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-    "chunks": [
-        5
-    ],
-    "compressor": {
-        "blocksize": 0,
-        "clevel": 5,
-        "cname": "lz4",
-        "id": "blosc",
-        "shuffle": 1
-    },
-    "dtype": "<f8",
-    "fill_value": 0.0,
-    "filters": null,
-    "order": "C",
-    "shape": [
-        5
-    ],
-    "zarr_format": 2
-}
\ No newline at end of file
diff --git a/tests/create-pipe-reference/create-pipe.zarr/sums/0 b/tests/create-pipe-reference/create-pipe.zarr/sums/0
deleted file mode 100644
index 978d02f4e91cd971aa8f231b9dd40e873a37ebbf..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 56
zcmZQ#H0ID?U|;}Y3m^u9p4J*^M<Bf<z-WR4kUk(~ea9Y1ODy|e1fm)Gjo#Y>0O>so
AGXMYp

diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 90fd3d8..37c8670 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -5,51 +5,59 @@
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
-import json
+import glob
 import os
 
 import numpy as np
+import pytest
+import zarr
 
 from ecml_tools.create import Creator
 from ecml_tools.data import open_dataset
 
-TEST_DATA_ROOT = "/s3/ml-tests/test-data/anemoi-datasets/create/"
-
-
-class Reference:
-    def __init__(self, name):
-        self.name = name
-        self.path = os.path.join(TEST_DATA_ROOT, name)
-
-    def compare_dot_zattrs(other):
-        a = json.load(open(self.path)).attrs
-        if isinstance(a, dict):
-            a_keys = list(a.keys())
-            b_keys = list(b.keys())
-            for k in set(a_keys) & set(b_keys):
-                if k in ["timestamp", "uuid", "latest_write_timestamp", "yaml_config"]:
-                    assert type(a[k]) == type(b[k]), (  # noqa: E721
-                        type(a[k]),
-                        type(b[k]),
-                        a[k],
-                        b[k],
-                    )
-                assert k in a_keys, (k, a_keys)
-                assert k in b_keys, (k, b_keys)
-                return compare_dot_zattrs(a[k], b[k])
-
-        if isinstance(a, list):
-            assert len(a) == len(b), (a, b)
-            for v, w in zip(a, b):
-                return compare_dot_zattrs(v, w)
-
-        assert type(a) == type(b), (type(a), type(b), a, b)  # noqa: E721
-        return a == b, (a, b)
-
-
-def compare_zarr(dir1, dir2):
-    a = open_dataset(dir1)
-    b = open_dataset(dir2)
+HERE = os.path.dirname(__file__)
+# find_yamls
+NAMES = [os.path.basename(path).split(".")[0] for path in glob.glob(os.path.join(HERE, "*.yaml"))]
+NAMES = [
+    name
+    for name in NAMES
+    if name
+    not in [
+        "missing",
+        "perturbations",
+    ]
+]
+assert NAMES, "No yaml files found in " + HERE
+
+TEST_DATA_ROOT = "s3://ml-tests/test-data/anemoi-datasets/create/"
+
+
+def compare_dot_zattrs(a, b):
+    if isinstance(a, dict):
+        a_keys = list(a.keys())
+        b_keys = list(b.keys())
+        for k in set(a_keys) & set(b_keys):
+            if k in ["timestamp", "uuid", "latest_write_timestamp", "yaml_config"]:
+                assert type(a[k]) == type(b[k]), (  # noqa: E721
+                    type(a[k]),
+                    type(b[k]),
+                    a[k],
+                    b[k],
+                )
+            assert k in a_keys, (k, a_keys)
+            assert k in b_keys, (k, b_keys)
+            return compare_dot_zattrs(a[k], b[k])
+
+    if isinstance(a, list):
+        assert len(a) == len(b), (a, b)
+        for v, w in zip(a, b):
+            return compare_dot_zattrs(v, w)
+
+    assert type(a) == type(b), (type(a), type(b), a, b)  # noqa: E721
+    return a == b, (a, b)
+
+
+def compare_datasets(a, b):
     assert a.shape == b.shape, (a.shape, b.shape)
     assert (a.dates == b.dates).all(), (a.dates, b.dates)
     for a_, b_ in zip(a.variables, b.variables):
@@ -66,47 +74,44 @@ def compare_zarr(dir1, dir2):
             a_ = a[i_date, i_param]
             b_ = b[i_date, i_param]
             assert a.shape == b.shape, (date, param, a.shape, b.shape)
-            delta = a_ - b_
-            max_delta = np.max(np.abs(delta))
-            assert max_delta == 0.0, (date, param, a_, b_, a_ - b_, max_delta)
-    compare(dir1, dir2)
-
 
-def compare(dir1, dir2):
-    """Compare two directories recursively."""
-    files1 = set(os.listdir(dir1))
-    files2 = set(os.listdir(dir2))
-    files = files1.union(files2)
+            a_nans = np.isnan(a_)
+            b_nans = np.isnan(b_)
+            assert np.all(a_nans == b_nans), (date, param, "nans are different")
 
-    for f in files:
-        path1 = os.path.join(dir1, f)
-        path2 = os.path.join(dir2, f)
+            a_ = np.where(a_nans, 0, a_)
+            b_ = np.where(b_nans, 0, b_)
 
-        if not os.path.isfile(path1):
-            assert os.path.isdir(path2), f"Directory {path2} does not exist"
-            compare(path1, path2)
-            continue
+            delta = a_ - b_
+            max_delta = np.max(np.abs(delta))
+            assert max_delta == 0.0, (date, param, a_, b_, a_ - b_, max_delta)
 
-        assert os.path.isfile(path2), f"File {path2} does not exist"
 
-        content1 = open(path1, "rb").read()
-        content2 = open(path2, "rb").read()
+class Comparer:
+    def __init__(self, name, output_path=None, reference_path=None):
+        self.name = name
+        self.reference = reference_path or os.path.join(TEST_DATA_ROOT, name + ".zarr")
+        self.output = output_path or os.path.join(name + ".zarr")
+        print(f"Comparing {self.reference} and {self.output}")
 
-        if f == ".zattrs":
-            compare_dot_zattrs(json.loads(content1), json.loads(content2))
-            continue
+        self.z_reference = zarr.open(self.reference)
+        self.z_output = zarr.open(self.output)
 
-        if f == "provenance_load.json":
-            assert False, "test not implemented to compare temporary statistics"
+        self.ds_reference = open_dataset(self.reference)
+        self.ds_output = open_dataset(self.output)
 
-        assert content1 == content2, f"{path1} != {path2}"
+    def compare(self):
+        compare_dot_zattrs(self.z_output.attrs, self.z_reference.attrs)
+        compare_datasets(self.ds_output, self.ds_reference)
+        # not implemented :
+        # compare_statistics(self.z_output, self.z_reference)
 
 
-def _test_create(name):
-    here = os.path.dirname(__file__)
-    config = os.path.join(here, name + ".yaml")
-    output = os.path.join(here, name + "-output", name + ".zarr")
-    reference = os.path.join(here, name + "-reference", name + ".zarr")
+@pytest.mark.parametrize("name", NAMES)
+def test_run(name):
+    config = os.path.join(HERE, name + ".yaml")
+    output = os.path.join(HERE, name + ".zarr")
+    comparer = Comparer(name, output_path=output)
 
     # cache=None is using the default cache
     c = Creator(
@@ -117,35 +122,15 @@ def _test_create(name):
     )
     c.create()
 
-    compare_zarr(reference, output)
-
-
-def test_create_concat():
-    _test_create("create-concat")
-
-
-def test_create_join():
-    _test_create("create-join")
-
-
-def test_create_pipe():
-    _test_create("create-pipe")
-
-
-def test_create_perturbations():
-    _test_create("create-perturbations")
+    comparer.compare()
 
 
 if __name__ == "__main__":
     import argparse
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--name", help="Name of the test case")
-    parser.add_argument("--compare", nargs=2, help="Compare two directories")
+    parser.add_argument("name", help="Name of the test case")
 
     args = parser.parse_args()
 
-    if args.compare:
-        compare_zarr(args.compare[0], args.compare[1])
-    else:
-        _test_create(args.name)
+    test_run(args.name)

From ae0ec9e3a3fd5c7ac00a9a88570211fb0429910a Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 16:46:26 +0000
Subject: [PATCH 15/25] test missing passes

---
 tests/create/test_create.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 37c8670..0c190c0 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -23,7 +23,6 @@
     for name in NAMES
     if name
     not in [
-        "missing",
         "perturbations",
     ]
 ]
@@ -62,7 +61,11 @@ def compare_datasets(a, b):
     assert (a.dates == b.dates).all(), (a.dates, b.dates)
     for a_, b_ in zip(a.variables, b.variables):
         assert a_ == b_, (a, b)
+    assert a.missing == b.missing, "Missing are different"
+
     for i_date, date in zip(range(a.shape[0]), a.dates):
+        if i_date in a.missing:
+            continue
         for i_param in range(a.shape[1]):
             param = a.variables[i_param]
             assert param == b.variables[i_param], (

From 6f6777cf75b44e8d70761fce95ce844089abaf54 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 17:28:44 +0000
Subject: [PATCH 16/25] test on statistics

---
 tests/create/test_create.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 0c190c0..1e73585 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -90,6 +90,19 @@ def compare_datasets(a, b):
             assert max_delta == 0.0, (date, param, a_, b_, a_ - b_, max_delta)
 
 
+def compare_statistics(ds1, ds2):
+    vars1 = ds1.variables
+    vars2 = ds2.variables
+    assert len(vars1) == len(vars2)
+    for v1, v2 in zip(vars1, vars2):
+        idx1 = ds1.name_to_index[v1]
+        idx2 = ds2.name_to_index[v2]
+        assert (ds1.statistics["mean"][idx1] == ds2.statistics["mean"][idx2]).all()
+        assert (ds1.statistics["stdev"][idx1] == ds2.statistics["stdev"][idx2]).all()
+        assert (ds1.statistics["maximum"][idx1] == ds2.statistics["maximum"][idx2]).all()
+        assert (ds1.statistics["minimum"][idx1] == ds2.statistics["minimum"][idx2]).all()
+
+
 class Comparer:
     def __init__(self, name, output_path=None, reference_path=None):
         self.name = name
@@ -106,8 +119,7 @@ def __init__(self, name, output_path=None, reference_path=None):
     def compare(self):
         compare_dot_zattrs(self.z_output.attrs, self.z_reference.attrs)
         compare_datasets(self.ds_output, self.ds_reference)
-        # not implemented :
-        # compare_statistics(self.z_output, self.z_reference)
+        compare_statistics(self.ds_output, self.ds_reference)
 
 
 @pytest.mark.parametrize("name", NAMES)

From 5d3a093a7bd196df4788cad5bc09d870578c6544 Mon Sep 17 00:00:00 2001
From: Mariana Clare <momc@ac6-101.bullx>
Date: Mon, 11 Mar 2024 17:51:53 +0000
Subject: [PATCH 17/25] fixing perturbations

---
 .../actions/ensemble_perturbations.py         | 33 +++++++++++++------
 ecml_tools/data.py                            |  1 +
 tests/create-perturbations.yaml               |  9 +++--
 3 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/ecml_tools/create/functions/actions/ensemble_perturbations.py b/ecml_tools/create/functions/actions/ensemble_perturbations.py
index dc599c7..d6e1999 100644
--- a/ecml_tools/create/functions/actions/ensemble_perturbations.py
+++ b/ecml_tools/create/functions/actions/ensemble_perturbations.py
@@ -59,7 +59,6 @@ def load_if_needed(context, dates, dict_or_dataset):
 def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}, patches={}):
     ensembles = load_if_needed(context, dates, ensembles)
     center = load_if_needed(context, dates, center)
-    mean = load_if_needed(context, dates, mean)
 
     keys = ["param", "level", "valid_datetime", "date", "time", "step", "number"]
 
@@ -69,10 +68,10 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
 
     ensembles = ensembles.order_by(*keys)
     center = center.order_by(*keys)
-    mean = mean.order_by(*keys)
 
     number_list = ensembles.unique_values("number")["number"]
     n_numbers = len(number_list)
+    date_numbers = len(center)
 
     assert len(mean) == len(center), (len(mean), len(center))
     if len(center) * n_numbers != len(ensembles):
@@ -83,6 +82,25 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
             print("Center: ", f)
         raise ValueError(f"Inconsistent number of fields: {len(center)} * {n_numbers} != {len(ensembles)}")
 
+    mean = np.zeros((center.to_numpy().shape))
+
+    for date in range(date_numbers):
+
+        ens_list = ensembles[date*n_numbers:(date+1)*n_numbers]
+
+        for i in range(len(ens_list)):
+            for k in keys + ["grid", "shape"]:
+                if k == "number":
+                    continue
+                assert center[date].metadata(k) == ens_list[i].metadata(k), (
+                    k,
+                    center[date].metadata(k),
+                    ens_list[i].metadata(k),
+                )
+        
+        ensembles_np = ensembles[date*n_numbers:(date+1)*n_numbers].to_numpy()
+        mean[date] = ensembles_np.mean(axis = 0)
+
     # prepare output tmp file so we can read it back
     tmp = temp_file()
     path = tmp.path
@@ -107,16 +125,11 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
                 center_field.metadata(k),
                 field.metadata(k),
             )
-            assert mean_field.metadata(k) == field.metadata(k), (
-                k,
-                mean_field.metadata(k),
-                field.metadata(k),
-            )
 
         e = field.to_numpy()
-        m = mean_field.to_numpy()
         c = center_field.to_numpy()
-        assert m.shape == c.shape, (m.shape, c.shape)
+        
+        assert mean_field.shape == c.shape, (mean_field.shape, c.shape)
 
         FORCED_POSITIVE = [
             "q",
@@ -126,7 +139,7 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
         ]  # add "swl4", "swl3", "swl2", "swl1", "swl0", and more ?
         #################################
         # Actual computation happens here
-        x = c - m + e
+        x = c - mean_field + e
         if param in FORCED_POSITIVE:
             warnings.warn(f"Clipping {param} to be positive")
             x = np.maximum(x, 0)
diff --git a/ecml_tools/data.py b/ecml_tools/data.py
index 7e17e32..2928f22 100644
--- a/ecml_tools/data.py
+++ b/ecml_tools/data.py
@@ -367,6 +367,7 @@ def __contains__(self, key):
 
 
 def open_zarr(path):
+    print(path)
     try:
         store = path
 
diff --git a/tests/create-perturbations.yaml b/tests/create-perturbations.yaml
index c4873f4..5c907c9 100644
--- a/tests/create-perturbations.yaml
+++ b/tests/create-perturbations.yaml
@@ -8,21 +8,20 @@ common:
   mars_request: &common
     class: ea
     expver: "0001"
-    grid: 20.0/20.0
+    grid: 2.0/2.0
     levtype: sfc
     param: [2t]
   mars_request_acc: &common_acc
     class: ea
     expver: "0001"
-    grid: 20.0/20.0
+    grid: 2.0/2.0
     levtype: sfc
     param: [tp]
 
   ensembles: &ensembles
     stream: enda
     type: an
-    number: [0, 2, 4]
-    # number: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    number: [1, 2, 3, 4, 5, 6, 7, 8,  9]
   center: &center
     stream: oper
     type: an
@@ -32,7 +31,7 @@ common:
 
 dates:
   start: 2020-12-30 00:00:00
-  end: 2021-01-03 12:00:00
+  end: 2021-01-31 12:00:00
   frequency: 12h
   group_by: monthly
 

From 5f270904e7b2f3250ccb10e4b85c32be0ea6ab13 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Mon, 11 Mar 2024 20:07:11 +0000
Subject: [PATCH 18/25] tests

---
 tests/create/test_create.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 1e73585..27f8836 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -28,7 +28,7 @@
 ]
 assert NAMES, "No yaml files found in " + HERE
 
-TEST_DATA_ROOT = "s3://ml-tests/test-data/anemoi-datasets/create/"
+TEST_DATA_ROOT = "https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create/"
 
 
 def compare_dot_zattrs(a, b):
@@ -126,7 +126,6 @@ def compare(self):
 def test_run(name):
     config = os.path.join(HERE, name + ".yaml")
     output = os.path.join(HERE, name + ".zarr")
-    comparer = Comparer(name, output_path=output)
 
     # cache=None is using the default cache
     c = Creator(
@@ -137,6 +136,7 @@ def test_run(name):
     )
     c.create()
 
+    comparer = Comparer(name, output_path=output)
     comparer.compare()
 
 

From 89a7fb16f555b6b0ac8557a18d66778b03a5140d Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 07:42:50 +0000
Subject: [PATCH 19/25] tests

---
 .../perturbations.yaml}                                     | 0
 tests/create/test_create.py                                 | 6 +++---
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename tests/{create-perturbations.yaml => create/perturbations.yaml} (100%)

diff --git a/tests/create-perturbations.yaml b/tests/create/perturbations.yaml
similarity index 100%
rename from tests/create-perturbations.yaml
rename to tests/create/perturbations.yaml
diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 1e73585..7f41806 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -10,10 +10,10 @@
 
 import numpy as np
 import pytest
-import zarr
 
 from ecml_tools.create import Creator
 from ecml_tools.data import open_dataset
+from ecml_tools.data import open_zarr
 
 HERE = os.path.dirname(__file__)
 # find_yamls
@@ -110,8 +110,8 @@ def __init__(self, name, output_path=None, reference_path=None):
         self.output = output_path or os.path.join(name + ".zarr")
         print(f"Comparing {self.reference} and {self.output}")
 
-        self.z_reference = zarr.open(self.reference)
-        self.z_output = zarr.open(self.output)
+        self.z_reference = open_zarr(self.reference)
+        self.z_output = open_zarr(self.output)
 
         self.ds_reference = open_dataset(self.reference)
         self.ds_output = open_dataset(self.output)

From 1bd928df38c59a63012d505d0ce5a242ee9caa63 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 10:08:35 +0000
Subject: [PATCH 20/25] building zar with recentered perturbations changed

---
 .../actions/ensemble_perturbations.py         | 158 +++++-------------
 tests/create/perturbations.yaml               |  17 +-
 2 files changed, 48 insertions(+), 127 deletions(-)

diff --git a/ecml_tools/create/functions/actions/ensemble_perturbations.py b/ecml_tools/create/functions/actions/ensemble_perturbations.py
index d6e1999..83acbfe 100644
--- a/ecml_tools/create/functions/actions/ensemble_perturbations.py
+++ b/ecml_tools/create/functions/actions/ensemble_perturbations.py
@@ -10,7 +10,6 @@
 from copy import deepcopy
 
 import numpy as np
-import tqdm
 from climetlab.core.temporary import temp_file
 from climetlab.readers.grib.output import new_grib_output
 
@@ -56,24 +55,27 @@ def load_if_needed(context, dates, dict_or_dataset):
     return dict_or_dataset
 
 
-def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}, patches={}):
+def ensemble_perturbations(context, dates, ensembles, center, remapping={}, patches={}):
     ensembles = load_if_needed(context, dates, ensembles)
     center = load_if_needed(context, dates, center)
 
     keys = ["param", "level", "valid_datetime", "date", "time", "step", "number"]
 
+    def check_compatible(f1, f2, ignore=["number"]):
+        for k in keys + ["grid", "shape"]:
+            if k in ignore:
+                continue
+            assert f1.metadata(k) == f2.metadata(k), (k, f1.metadata(k), f2.metadata(k))
+
     print(f"Retrieving ensemble data with {ensembles}")
     print(f"Retrieving center data with {center}")
-    print(f"Retrieving mean data with {mean}")
 
     ensembles = ensembles.order_by(*keys)
     center = center.order_by(*keys)
 
     number_list = ensembles.unique_values("number")["number"]
     n_numbers = len(number_list)
-    date_numbers = len(center)
 
-    assert len(mean) == len(center), (len(mean), len(center))
     if len(center) * n_numbers != len(ensembles):
         print(len(center), n_numbers, len(ensembles))
         for f in ensembles:
@@ -82,73 +84,53 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
             print("Center: ", f)
         raise ValueError(f"Inconsistent number of fields: {len(center)} * {n_numbers} != {len(ensembles)}")
 
-    mean = np.zeros((center.to_numpy().shape))
-
-    for date in range(date_numbers):
-
-        ens_list = ensembles[date*n_numbers:(date+1)*n_numbers]
-
-        for i in range(len(ens_list)):
-            for k in keys + ["grid", "shape"]:
-                if k == "number":
-                    continue
-                assert center[date].metadata(k) == ens_list[i].metadata(k), (
-                    k,
-                    center[date].metadata(k),
-                    ens_list[i].metadata(k),
-                )
-        
-        ensembles_np = ensembles[date*n_numbers:(date+1)*n_numbers].to_numpy()
-        mean[date] = ensembles_np.mean(axis = 0)
-
     # prepare output tmp file so we can read it back
     tmp = temp_file()
     path = tmp.path
     out = new_grib_output(path)
 
-    for i, field in tqdm.tqdm(enumerate(ensembles)):
-        param = field.metadata("param")
-        number = field.metadata("number")
-        ii = i // n_numbers
+    for i, center_field in enumerate(center):
+        param = center_field.metadata("param")
 
-        i_number = number_list.index(number)
-        assert i == ii * n_numbers + i_number, (i, ii, n_numbers, i_number, number_list)
+        # load the center field
+        center_np = center_field.to_numpy()
 
-        center_field = center[ii]
-        mean_field = mean[ii]
+        # load the ensemble fields and compute the mean
+        ensembles_np = np.zeros((n_numbers, *center_np.shape))
 
-        for k in keys + ["grid", "shape"]:
-            if k == "number":
-                continue
-            assert center_field.metadata(k) == field.metadata(k), (
-                k,
-                center_field.metadata(k),
-                field.metadata(k),
-            )
-
-        e = field.to_numpy()
-        c = center_field.to_numpy()
-        
-        assert mean_field.shape == c.shape, (mean_field.shape, c.shape)
-
-        FORCED_POSITIVE = [
-            "q",
-            "cp",
-            "lsp",
-            "tp",
-        ]  # add "swl4", "swl3", "swl2", "swl1", "swl0", and more ?
-        #################################
-        # Actual computation happens here
-        x = c - mean_field + e
-        if param in FORCED_POSITIVE:
-            warnings.warn(f"Clipping {param} to be positive")
-            x = np.maximum(x, 0)
-        #################################
-
-        assert x.shape == e.shape, (x.shape, e.shape)
-
-        check_data_values(x, name=param)
-        out.write(x, template=field)
+        for j in range(n_numbers):
+            ensemble_field = ensembles[i * n_numbers + j]
+            check_compatible(center_field, ensemble_field)
+            ensembles_np[j] = ensemble_field.to_numpy()
+
+        mean_np = ensembles_np.mean(axis=0)
+
+        for j in range(n_numbers):
+            template = ensembles[i * n_numbers + j]
+            e = ensembles_np[j]
+            m = mean_np
+            c = center_np
+
+            assert e.shape == c.shape == m.shape, (e.shape, c.shape, m.shape)
+
+            FORCED_POSITIVE = [
+                "q",
+                "cp",
+                "lsp",
+                "tp",
+            ]  # add "swl4", "swl3", "swl2", "swl1", "swl0", and more ?
+
+            x = c - m + e
+
+            if param in FORCED_POSITIVE:
+                warnings.warn(f"Clipping {param} to be positive")
+                x = np.maximum(x, 0)
+
+            assert x.shape == e.shape, (x.shape, e.shape)
+
+            check_data_values(x, name=param)
+            out.write(x, template=template)
+            template = None
 
     out.close()
 
@@ -166,51 +148,3 @@ def ensemble_perturbations(context, dates, ensembles, center, mean, remapping={}
 
 
 execute = ensemble_perturbations
-
-if __name__ == "__main__":
-    import yaml
-
-    config = yaml.safe_load(
-        """
-
-    common: &common
-        name: mars
-        # marser is the MARS containing ERA5 reanalysis dataset, avoid hitting the FDB server for nothing
-        database: marser
-        class: ea
-        # date: $datetime_format($dates,%Y%m%d)
-        # time: $datetime_format($dates,%H%M)
-        date: 20221230/to/20230103
-        time: '0000/1200'
-        expver: '0001'
-        grid: 20.0/20.0
-        levtype: sfc
-        param: [2t]
-        # levtype: pl
-        # param: [10u, 10v, 2d, 2t, lsm, msl, sdor, skt, slor, sp, tcw, z]
-
-    config:
-        ensembles: # the ensemble data has one additional dimension
-          <<: *common
-          stream: enda
-          type: an
-          number: [0, 1]
-          # number: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-        center: # the new center of the data
-          <<: *common
-          stream: oper
-          type: an
-
-        mean: # the previous center of the data
-          <<: *common
-          stream: enda
-          type: em
-
-    """
-    )["config"]
-    for k, v in config.items():
-        print(k, v)
-
-    for f in ensemble_perturbations(**config):
-        print(f, f.to_numpy().mean())
diff --git a/tests/create/perturbations.yaml b/tests/create/perturbations.yaml
index 29c177a..30cd3d2 100644
--- a/tests/create/perturbations.yaml
+++ b/tests/create/perturbations.yaml
@@ -21,14 +21,11 @@ common:
   ensembles: &ensembles
     stream: enda
     type: an
-    number: [0, 2, 4]
-    # number: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+    number: [1, 2, 4]
+    # number: [1, 2, 3, 4, 5, 6, 7, 8, 9]
   center: &center
     stream: oper
     type: an
-  mean: &mean
-    stream: enda
-    type: em
 
 dates:
   start: 2020-12-30 00:00:00
@@ -47,14 +44,6 @@ data_sources:
       - accumulations:
           <<: *ensembles
           <<: *common_acc
-  mean:
-    join:
-      - mars:
-          <<: *mean
-          <<: *common
-      - accumulations:
-          <<: *mean
-          <<: *common_acc
   center:
     join:
       - mars:
@@ -68,8 +57,6 @@ input:
   ensemble_perturbations:
     # the ensemble data which has one additional dimension
     ensembles: ${data_sources.ensembles}
-    # the previous center of the data
-    mean: ${data_sources.mean}
     # the new center of the data
     center: ${data_sources.center}
 

From 6300d0082a652ed8de8ac571830e5a2267182a86 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 10:46:30 +0000
Subject: [PATCH 21/25] simplify config

---
 ecml_tools/create/config.py | 33 ++++++++++++++++++---------------
 tests/create/join.yaml      |  8 +-------
 2 files changed, 19 insertions(+), 22 deletions(-)

diff --git a/ecml_tools/create/config.py b/ecml_tools/create/config.py
index fe0f61d..2cc2946 100644
--- a/ecml_tools/create/config.py
+++ b/ecml_tools/create/config.py
@@ -145,16 +145,17 @@ def __init__(self, config, *args, **kwargs):
 
         # TODO: should use a json schema to validate the config
 
+        if "dataset_status" not in self:
+            self.dataset_status = "experimental"
+
         if "description" not in self:
-            raise ValueError("Must provide a description in the config.")
+            self.description = "No description provided."
 
         if "config_format_version" not in self:
-            # Should be changed to 2
-            self.config_format_version = 1
-            print(f"Setting config_format_version={self.config_format_version} because it was not provided.")
+            self.config_format_version = 3
 
-        if self.config_format_version != 2:
-            raise ValueError("Config format has changed. Must provide config with format version >= 2.")
+        if self.config_format_version != 3:
+            raise ValueError("Config format has changed. Must provide config with format version == 3.")
 
         if "dates" in self.output:
             raise ValueError("Obsolete: Dates should not be provided in output config.")
@@ -169,14 +170,14 @@ def __init__(self, config, *args, **kwargs):
         if "loop" in self:
             raise ValueError(f"Do not use 'loop'. Use dates instead. {list(self.keys())}")
 
-        self.options = self.get("options", {})
-
         if "licence" not in self:
             self.licence = "unknown"
-            print(f"❗ Setting licence={self.licence} because it was not provided.")
         if "copyright" not in self:
             self.copyright = "unknown"
-            print(f"❗ Setting copyright={self.copyright} because it was not provided.")
+
+        self.build = self.get("build", {})
+        if "group_by" not in self.build:
+            self.build.group_by = "monthly"
 
         check_dict_value_and_set(self.output, "flatten_grid", True)
         check_dict_value_and_set(self.output, "ensemble_dimension", 2)
@@ -192,17 +193,19 @@ def __init__(self, config, *args, **kwargs):
         if "order_by" in self.output:
             self.output.order_by = normalize_order_by(self.output.order_by)
 
-        self.output.chunking = self.output.get("chunking", {})
-        self.output.dtype = self.output.get("dtype", "float32")
+        if "chunking" not in self.output:
+            self.output.chunking = dict(dates=1, ensembles=1)
+        if "dtype" not in self.output:
+            self.output.dtype = "float32"
+
+        if "group_by" in self.build:
+            self.dates["group_by"] = self.build.group_by
 
         self.reading_chunks = self.get("reading_chunks")
         assert "flatten_values" not in self.output
         assert "flatten_grid" in self.output, self.output
         assert "statistics" in self.output
 
-        if "group_by" in self.options:
-            self.dates["group_by"] = self.options.group_by
-
     def get_serialisable_dict(self):
         return _prepare_serialisation(self)
 
diff --git a/tests/create/join.yaml b/tests/create/join.yaml
index 782e41f..d0a35b1 100644
--- a/tests/create/join.yaml
+++ b/tests/create/join.yaml
@@ -1,8 +1,4 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-purpose: aifs
 name: test-join
-config_format_version: 2
 
 common:
   mars_request: &mars_request
@@ -16,7 +12,7 @@ dates:
   frequency: 12h
 
 
-options:
+build:
   group_by: monthly
 
 input:
@@ -48,8 +44,6 @@ input:
           - cos_latitude
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"

From 39ff096b7e6f50365972daf6d7c5fab0b5431353 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 11:30:02 +0000
Subject: [PATCH 22/25] simplify config

---
 ecml_tools/create/config.py  |  6 +++++-
 ecml_tools/create/loaders.py | 40 ++++++++++++++++++++++++++++++------
 ecml_tools/data.py           |  1 -
 tests/create/join.yaml       |  6 ------
 4 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/ecml_tools/create/config.py b/ecml_tools/create/config.py
index 2cc2946..13abef8 100644
--- a/ecml_tools/create/config.py
+++ b/ecml_tools/create/config.py
@@ -141,6 +141,9 @@ def statistics(self):
 class LoadersConfig(Config):
 
     def __init__(self, config, *args, **kwargs):
+        if "build" not in config:
+            config["build"] = {}
+
         super().__init__(config, *args, **kwargs)
 
         # TODO: should use a json schema to validate the config
@@ -175,7 +178,6 @@ def __init__(self, config, *args, **kwargs):
         if "copyright" not in self:
             self.copyright = "unknown"
 
-        self.build = self.get("build", {})
         if "group_by" not in self.build:
             self.build.group_by = "monthly"
 
@@ -201,6 +203,8 @@ def __init__(self, config, *args, **kwargs):
         if "group_by" in self.build:
             self.dates["group_by"] = self.build.group_by
 
+        ###########
+
         self.reading_chunks = self.get("reading_chunks")
         assert "flatten_values" not in self.output
         assert "flatten_grid" in self.output, self.output
diff --git a/ecml_tools/create/loaders.py b/ecml_tools/create/loaders.py
index ad45b2b..f75a942 100644
--- a/ecml_tools/create/loaders.py
+++ b/ecml_tools/create/loaders.py
@@ -39,6 +39,28 @@
 VERSION = "0.20"
 
 
+def default_statistics_dates(dates):
+    first = dates[0]
+    last = dates[-1]
+    n_years = (last - first).days // 365
+
+    if n_years >= 20:
+        end = datetime.datetime(last.year - 2, last.month, last.day, last.hour, last.minute, last.second)
+        print(f"Number of years {n_years} >= 20, leaving out 2 years. {end=}")
+        return dates[0], end
+
+    if n_years >= 10:  # leave out 1 year
+        end = datetime.datetime(last.year - 1, last.month, last.day, last.hour, last.minute, last.second)
+        print(f"Number of years {n_years} >= 10, leaving out 1 years. {end=}")
+        return dates[0], end
+
+    # leave out 20% of the data
+    k = int(len(dates) * 0.8)
+    end = dates[k]
+    print(f"Number of years {n_years} < 10, leaving out 20%. {end=}")
+    return dates[0], end
+
+
 class Loader:
     def __init__(self, *, path, print=print, **kwargs):
         # Catch all floating point errors, including overflow, sqrt(<0), etc
@@ -89,12 +111,18 @@ def build_input(self):
 
     def build_statistics_dates(self, start, end):
         ds = open_dataset(self.path)
-        subset = ds.dates_interval_to_indices(start, end)
-        start, end = ds.dates[subset[0]], ds.dates[subset[-1]]
-        return (
-            start.astype(datetime.datetime).isoformat(),
-            end.astype(datetime.datetime).isoformat(),
-        )
+        dates = ds.dates
+
+        if end is None and start is None:
+            start, end = default_statistics_dates(dates)
+        else:
+            subset = ds.dates_interval_to_indices(start, end)
+            start = dates[subset[0]]
+            end = dates[subset[-1]]
+
+        start = start.astype(datetime.datetime)
+        end = end.astype(datetime.datetime)
+        return (start.isoformat(), end.isoformat())
 
     def read_dataset_metadata(self):
         ds = open_dataset(self.path)
diff --git a/ecml_tools/data.py b/ecml_tools/data.py
index 2928f22..7e17e32 100644
--- a/ecml_tools/data.py
+++ b/ecml_tools/data.py
@@ -367,7 +367,6 @@ def __contains__(self, key):
 
 
 def open_zarr(path):
-    print(path)
     try:
         store = path
 
diff --git a/tests/create/join.yaml b/tests/create/join.yaml
index d0a35b1..ab09d4c 100644
--- a/tests/create/join.yaml
+++ b/tests/create/join.yaml
@@ -1,5 +1,3 @@
-name: test-join
-
 common:
   mars_request: &mars_request
     expver: "0001"
@@ -11,10 +9,6 @@ dates:
   end: 2021-01-03 12:00:00
   frequency: 12h
 
-
-build:
-  group_by: monthly
-
 input:
   join:
     - mars:

From 691eb834974dd26de8b7e52f0dc6b5e9d59087fa Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 12:46:41 +0000
Subject: [PATCH 23/25] fix

---
 ecml_tools/create/loaders.py    | 20 ++++++++++++--------
 tests/create/concat.yaml        | 18 ++----------------
 tests/create/data_sources.yaml  |  9 +--------
 tests/create/missing.yaml       |  8 --------
 tests/create/nan.yaml           | 11 -----------
 tests/create/perturbations.yaml | 13 +------------
 tests/create/pipe.yaml          | 19 +++----------------
 7 files changed, 19 insertions(+), 79 deletions(-)

diff --git a/ecml_tools/create/loaders.py b/ecml_tools/create/loaders.py
index f75a942..ffb3f70 100644
--- a/ecml_tools/create/loaders.py
+++ b/ecml_tools/create/loaders.py
@@ -42,23 +42,27 @@
 def default_statistics_dates(dates):
     first = dates[0]
     last = dates[-1]
+    first = first.tolist()
+    last = last.tolist()
+    assert isinstance(first, datetime.datetime), first
+    assert isinstance(last, datetime.datetime), last
+
     n_years = (last - first).days // 365
 
     if n_years >= 20:
         end = datetime.datetime(last.year - 2, last.month, last.day, last.hour, last.minute, last.second)
         print(f"Number of years {n_years} >= 20, leaving out 2 years. {end=}")
-        return dates[0], end
 
-    if n_years >= 10:  # leave out 1 year
+    elif n_years >= 10:  # leave out 1 year
         end = datetime.datetime(last.year - 1, last.month, last.day, last.hour, last.minute, last.second)
         print(f"Number of years {n_years} >= 10, leaving out 1 years. {end=}")
-        return dates[0], end
+    else:
+        # leave out 20% of the data
+        k = int(len(dates) * 0.8)
+        end = dates[k]
+        print(f"Number of years {n_years} < 10, leaving out 20%. {end=}")
 
-    # leave out 20% of the data
-    k = int(len(dates) * 0.8)
-    end = dates[k]
-    print(f"Number of years {n_years} < 10, leaving out 20%. {end=}")
-    return dates[0], end
+    return dates[0], np.datetime64(end)
 
 
 class Loader:
diff --git a/tests/create/concat.yaml b/tests/create/concat.yaml
index 416e9c6..070386d 100644
--- a/tests/create/concat.yaml
+++ b/tests/create/concat.yaml
@@ -1,16 +1,8 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-name: test-concat
-config_format_version: 2
-
 dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
 
-options:
-  group_by: monthly
-
 common:
   mars_request: &mars_request
     expver: "0001"
@@ -37,13 +29,7 @@ input:
         <<: *mars_request
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
-  order_by:
-    - valid_datetime
-    - param_level
-    - number
-  statistics: param_level
-  statistics_end: 2021
+  order_by: [valid_datetime, param_level, number]
   remapping: 
     param_level: "{param}_{levelist}"
+  statistics: param_level
\ No newline at end of file
diff --git a/tests/create/data_sources.yaml b/tests/create/data_sources.yaml
index ad83c08..4a051f2 100644
--- a/tests/create/data_sources.yaml
+++ b/tests/create/data_sources.yaml
@@ -1,8 +1,3 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-name: test-data-sources
-config_format_version: 2
-
 common:
   mars_request: &mars_request
     expver: "0001"
@@ -31,10 +26,8 @@ input:
       - cos_latitude
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
   statistics: param_level
-  statistics_end: 2021
+  statistics_end: 2021
\ No newline at end of file
diff --git a/tests/create/missing.yaml b/tests/create/missing.yaml
index f9a8993..ba9e659 100644
--- a/tests/create/missing.yaml
+++ b/tests/create/missing.yaml
@@ -1,9 +1,3 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-purpose: aifs
-name: test-small
-config_format_version: 2
-
 common:
   mars_request: &mars_request
     expver: "0001"
@@ -35,8 +29,6 @@ input:
       #- sin_latitude
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
diff --git a/tests/create/nan.yaml b/tests/create/nan.yaml
index 002b90f..ac0fa39 100644
--- a/tests/create/nan.yaml
+++ b/tests/create/nan.yaml
@@ -1,16 +1,8 @@
-description: "testing for nans using sst"
-dataset_status: testing
-name: test-nan
-config_format_version: 2
-
 dates:
   start: 2020-12-30 00:00:00
   end: 2021-01-03 12:00:00
   frequency: 12h
 
-options:
-  group_by: monthly
-
 input:
   mars:
     expver: "0001"
@@ -25,10 +17,7 @@ input:
 has_nans: [sst]
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
   order_by: [valid_datetime, param_level, number]
   remapping:
     param_level: "{param}_{levelist}"
   statistics: param_level
-  statistics_end: 2020
diff --git a/tests/create/perturbations.yaml b/tests/create/perturbations.yaml
index 30cd3d2..476da9e 100644
--- a/tests/create/perturbations.yaml
+++ b/tests/create/perturbations.yaml
@@ -1,9 +1,3 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-purpose: aifs
-name: create-pertubations
-config_format_version: 2
-
 common:
   mars_request: &common
     class: ea
@@ -61,12 +55,7 @@ input:
     center: ${data_sources.center}
 
 output:
-  chunking: { dates: 1 }
-  dtype: float32
-  order_by:
-    - valid_datetime
-    - param_level
-    - number
+  order_by: [valid_datetime, param_level, number]
   statistics: param_level
   statistics_end: 2021
   remapping:
diff --git a/tests/create/pipe.yaml b/tests/create/pipe.yaml
index 484aebf..3b0c49a 100644
--- a/tests/create/pipe.yaml
+++ b/tests/create/pipe.yaml
@@ -1,8 +1,3 @@
-description: "develop version of the dataset for a few days and a few variables, once data on mars is cached it should take a few seconds to generate the dataset"
-dataset_status: testing
-name: test-small
-config_format_version: 2
-
 common:
   mars_request: &mars_request
     expver: "0001"
@@ -14,9 +9,6 @@ dates: &dates_anchor
   end: 2021-01-03 12:00:00
   frequency: 12h
 
-options:
-  group_by: monthly
-
 input:
   join:
     - mars:
@@ -47,13 +39,8 @@ input:
           - cos_latitude
 
 output:
-  chunking: { dates: 1, ensembles: 1 }
-  dtype: float32
-  order_by:
-    - valid_datetime
-    - param_level
-    - number
+  order_by: [valid_datetime, param_level, number]
+  remapping:
+    param_level: "{param}_{levelist}"
   statistics: param_level
   statistics_end: 2021
-  remapping: &remapping
-    param_level: "{param}_{levelist}"

From 47e3deccda423c6bd07791035a70b8405cff821d Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 13:57:28 +0000
Subject: [PATCH 24/25] dataset creation: using mockup source

---
 ecml_tools/create/functions/__init__.py       | 110 +++++++++++-------
 .../create/functions/actions/accumulations.py |   4 +-
 ecml_tools/create/functions/actions/mars.py   |   4 +-
 3 files changed, 74 insertions(+), 44 deletions(-)

diff --git a/ecml_tools/create/functions/__init__.py b/ecml_tools/create/functions/__init__.py
index 9dd5da9..b7499f6 100644
--- a/ecml_tools/create/functions/__init__.py
+++ b/ecml_tools/create/functions/__init__.py
@@ -6,50 +6,78 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 #
-from collections import defaultdict
+import os
+
+from climetlab import load_source
+
+
+class Mockup:
+    write_directory = None
+    read_directory = None
+
+    def get_filename(self, r):
+        import hashlib
+
+        h = hashlib.md5(str(r).encode("utf8")).hexdigest()
+        return h + ".copy"
+
+    def write(self, ds, *args, **kwargs):
+        if self.write_directory is None:
+            return
+        if not hasattr(ds, "path"):
+            return
+
+        path = os.path.join(self.write_directory, self.get_filename([args, kwargs]))
+        print(f"Saving to {path} for {args}, {kwargs}")
+        import shutil
+
+        shutil.copy(ds.path, path)
+
+    def load_source(self, *args, **kwargs):
+        if self.read_directory is None:
+            return None
+        path = os.path.join(self.read_directory, self.get_filename([args, kwargs]))
+        if os.path.exists(path):
+            print(f"Loading to {path} for {args}, {kwargs}")
+            return load_source("file", path)
+        return None
+
+
+MOCKUP = Mockup()
+
+
+def enable_save_mars(d):
+    MOCKUP.write_directory = d
+
+
+def disable_save_mars():
+    MOCKUP.write_directory = None
+
+
+def enable_read_mars(d):
+    MOCKUP.read_directory = d
+
+
+def disable_read_mars():
+    MOCKUP.read_directory = None
+
+
+if os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"):
+    enable_save_mars(os.environ.get("MOCKUP_MARS_SAVE_REQUESTS"))
+
+if os.environ.get("MOCKUP_MARS_READ_REQUESTS"):
+    enable_read_mars(os.environ.get("MOCKUP_MARS_READ_REQUESTS"))
+
+
+def _load_source(*args, **kwargs):
+    ds = MOCKUP.load_source(*args, **kwargs)
+    if ds is None:
+        ds = load_source(*args, **kwargs)
+    MOCKUP.write(ds, *args, **kwargs)
+    return ds
 
 
 def assert_is_fieldset(obj):
     from climetlab.readers.grib.index import FieldSet
 
     assert isinstance(obj, FieldSet), type(obj)
-
-
-def wrapped_mars_source(name, param, **kwargs):
-    from climetlab import load_source
-
-    assert name == "mars", name  # untested with other sources
-
-    for_accumlated = dict(
-        ea="era5-accumulations",
-        oper="oper-accumulations",
-        ei="oper-accumulations",
-    )[kwargs["class"]]
-
-    param_to_source = defaultdict(lambda: "mars")
-    param_to_source.update(
-        dict(
-            tp=for_accumlated,
-            cp=for_accumlated,
-            lsp=for_accumlated,
-        )
-    )
-
-    source_names = defaultdict(list)
-    for p in param:
-        source_names[param_to_source[p]].append(p)
-
-    sources = []
-    for n, params in source_names.items():
-        sources.append(load_source(n, param=params, **patch_time_to_hours(kwargs)))
-    return load_source("multi", sources)
-
-
-def patch_time_to_hours(dic):
-    # era5-accumulations requires time in hours
-    if "time" not in dic:
-        return dic
-    time = dic["time"]
-    assert isinstance(time, (tuple, list)), time
-    time = [f"{int(t[:2]):02d}" for t in time]
-    return {**dic, "time": time}
diff --git a/ecml_tools/create/functions/actions/accumulations.py b/ecml_tools/create/functions/actions/accumulations.py
index 5c6250f..615f18d 100644
--- a/ecml_tools/create/functions/actions/accumulations.py
+++ b/ecml_tools/create/functions/actions/accumulations.py
@@ -8,11 +8,11 @@
 #
 from copy import deepcopy
 
-from climetlab import load_source
-
 from ecml_tools.create.functions.actions.mars import factorise_requests
 from ecml_tools.create.utils import to_datetime_list
 
+from .. import _load_source as load_source
+
 DEBUG = True
 
 
diff --git a/ecml_tools/create/functions/actions/mars.py b/ecml_tools/create/functions/actions/mars.py
index 4d7cefc..f0e5c52 100644
--- a/ecml_tools/create/functions/actions/mars.py
+++ b/ecml_tools/create/functions/actions/mars.py
@@ -9,11 +9,12 @@
 import datetime
 from copy import deepcopy
 
-from climetlab import load_source
 from climetlab.utils.availability import Availability
 
 from ecml_tools.create.utils import to_datetime_list
 
+from .. import _load_source as load_source
+
 DEBUG = False
 
 
@@ -92,6 +93,7 @@ def mars(context, dates, *requests, **kwargs):
         r = {k: v for k, v in r.items() if v != ("-",)}
         if DEBUG:
             context.trace("✅", f"load_source(mars, {r}")
+
         ds = ds + load_source("mars", **r)
     return ds
 

From 648f6d526cb6b9626b0605ed0a95f9efcd09a062 Mon Sep 17 00:00:00 2001
From: Florian Pinault <Florian.Pinault@ecmwf.int>
Date: Tue, 12 Mar 2024 16:08:57 +0000
Subject: [PATCH 25/25] tests: test data on s3

---
 ecml_tools/create/functions/__init__.py | 10 +++++++++-
 tests/create/test_create.py             |  2 ++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/ecml_tools/create/functions/__init__.py b/ecml_tools/create/functions/__init__.py
index b7499f6..d4fd61c 100644
--- a/ecml_tools/create/functions/__init__.py
+++ b/ecml_tools/create/functions/__init__.py
@@ -38,8 +38,16 @@ def load_source(self, *args, **kwargs):
             return None
         path = os.path.join(self.read_directory, self.get_filename([args, kwargs]))
         if os.path.exists(path):
-            print(f"Loading to {path} for {args}, {kwargs}")
+            print(f"Loading path {path} for {args}, {kwargs}")
             return load_source("file", path)
+        elif path.startswith("http"):
+            import requests
+
+            print(f"Loading url {path} for {args}, {kwargs}")
+            try:
+                return load_source("url", path)
+            except requests.exceptions.HTTPError:
+                pass
         return None
 
 
diff --git a/tests/create/test_create.py b/tests/create/test_create.py
index 6d0a17b..3db5a8e 100755
--- a/tests/create/test_create.py
+++ b/tests/create/test_create.py
@@ -12,6 +12,7 @@
 import pytest
 
 from ecml_tools.create import Creator
+from ecml_tools.create.functions import enable_read_mars
 from ecml_tools.data import open_dataset
 from ecml_tools.data import open_zarr
 
@@ -29,6 +30,7 @@
 assert NAMES, "No yaml files found in " + HERE
 
 TEST_DATA_ROOT = "https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create/"
+enable_read_mars("https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/anemoi-datasets/create")
 
 
 def compare_dot_zattrs(a, b):