Skip to content

Commit

Permalink
Merge branch 'release/0.20.2'
Browse files Browse the repository at this point in the history
  • Loading branch information
floriankrb committed Feb 8, 2024
2 parents 23cb69d + ee27e91 commit 6863d2a
Show file tree
Hide file tree
Showing 20 changed files with 285 additions and 197 deletions.
4 changes: 2 additions & 2 deletions climetlab-and-ecmwflibs-install.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ print(eccodes.__file__)

``` python
import climetlab as cml
url = "https://github.com/ecmwf/climetlab/blob/develop/docs/examples/test.grib"
url = "https://github.com/ecmwf/climetlab/raw/develop/docs/examples/test.grib"
ds=cml.load_source("url", url)
print(ds[0])
```


# Support

Issues related to this workaround should be raised at https://github.com/ecmwf/ecmwflibs/issues.
Issues related to this workaround should be raised at https://github.com/ecmwf/ecmwflibs/issues.
2 changes: 1 addition & 1 deletion climetlab/core/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def new_mask_index(self, *args, **kwargs):
def __len__(self):
self._not_implemented()

def _normalize_kwargs_names(**kwargs):
def _normalize_kwargs_names(self, **kwargs):
return kwargs

def sel(self, *args, remapping=None, **kwargs):
Expand Down
19 changes: 1 addition & 18 deletions climetlab/indexing/cube.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,22 +67,8 @@ def __init__(

self._field_shape = None

# Sort the source according to their
# internal_args = reduce(operator.add, [Separator.split(a) for a in args], [])
# for i in ds:
# print(i)
# print("before")
# print(ds[0])
# print(ds[1])
# print(ds[2])
# print(ds[3])
self.source = ds.order_by(*args, remapping=remapping, patches=patches)
del ds
# print("after")
# print(self.source[0])
# print(self.source[1])
# print(self.source[2])
# print(self.source[3])

# Get a mapping of user names to unique values
# With possible reduce dimentionality if the user use 'level+param'
Expand Down Expand Up @@ -252,17 +238,14 @@ def __init__(self, cube, coords, coords_names=None):
self.owner = cube
assert all(isinstance(_, int) for _ in coords), coords
self.coords = coords
self.extended_icoords = self.coords
self.flatten_values = cube.flatten_values

def __repr__(self):
return (
f"{self.__class__.__name__}({self.coords},index_names={self._coords_names})"
)

@property
def extended_icoords(self):
return self.coords

def to_numpy(self, **kwargs):
return self.owner[self.coords].to_numpy(
reshape=not self.flatten_values, **kwargs
Expand Down
36 changes: 36 additions & 0 deletions climetlab/indexing/fieldset.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# (C) Copyright 2023 ECMWF.
#
# This software is licensed under the terms of the Apache Licence Version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.
#

from climetlab.core import Base
from climetlab.core.index import Index, MaskIndex, MultiIndex

class Field(Base):
pass

class FieldSet(Index):


@classmethod
def new_mask_index(self, *args, **kwargs):
return MaskFieldSet(*args, **kwargs)

@classmethod
def merge(cls, sources):
assert all(isinstance(_, FieldSet) for _ in sources)
return MultiFieldSet(sources)


class MaskFieldSet(FieldSet, MaskIndex):
def __init__(self, *args, **kwargs):
MaskIndex.__init__(self, *args, **kwargs)


class MultiFieldSet(FieldSet, MultiIndex):
def __init__(self, *args, **kwargs):
MultiIndex.__init__(self, *args, **kwargs)
3 changes: 2 additions & 1 deletion climetlab/readers/grib/codes.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from climetlab.core import Base
from climetlab.core.constants import DATETIME
from climetlab.indexing.fieldset import Field
from climetlab.profiling import call_counter
from climetlab.utils.bbox import BoundingBox

Expand Down Expand Up @@ -321,7 +322,7 @@ def at_offset(self, offset):
# count = defaultdict(int)


class GribField(Base):
class GribField(Field):
def __init__(self, path, offset, length, handle_cache=None):
self.path = path
self._offset = offset
Expand Down
28 changes: 4 additions & 24 deletions climetlab/readers/grib/index/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

from lru import LRU

from climetlab.core.index import Index, MaskIndex, MultiIndex
from climetlab.decorators import normalize_grib_key_values, normalize_grib_keys
from climetlab.indexing.database import (
FILEPARTS_KEY_NAMES,
Expand All @@ -26,11 +25,12 @@
from climetlab.readers.grib.fieldset import FieldSetMixin
from climetlab.utils import progress_bar
from climetlab.utils.availability import Availability
from climetlab.indexing.fieldset import FieldSet

LOG = logging.getLogger(__name__)


class FieldSet(FieldSetMixin, Index):
class GribFieldSet(FieldSetMixin, FieldSet):
_availability = None

def __init__(self, *args, **kwargs):
Expand All @@ -39,21 +39,12 @@ def __init__(self, *args, **kwargs):
):
self._availability = Availability(self.availability_path)

Index.__init__(self, *args, **kwargs)

@classmethod
def new_mask_index(self, *args, **kwargs):
return MaskFieldSet(*args, **kwargs)
FieldSet.__init__(self, *args, **kwargs)

@property
def availability_path(self):
return None

@classmethod
def merge(cls, sources):
assert all(isinstance(_, FieldSet) for _ in sources)
return MultiFieldSet(sources)

def available(self, request, as_list_of_dicts=False):
from climetlab.utils.availability import Availability

Expand Down Expand Up @@ -150,18 +141,7 @@ def is_full_hypercube(self):
def _normalize_kwargs_names(self, **kwargs):
return kwargs


class MaskFieldSet(FieldSet, MaskIndex):
def __init__(self, *args, **kwargs):
MaskIndex.__init__(self, *args, **kwargs)


class MultiFieldSet(FieldSet, MultiIndex):
def __init__(self, *args, **kwargs):
MultiIndex.__init__(self, *args, **kwargs)


class FieldSetInFiles(FieldSet):
class FieldSetInFiles(GribFieldSet):
# Remote Fieldsets (with urls) are also here,
# as the actual fieldset is accessed on a file in cache.
# This class changes the interface (_getitem__ and __len__)
Expand Down
8 changes: 5 additions & 3 deletions climetlab/readers/grib/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def write(
self.update_metadata(handle, metadata, compulsary)
# print("<-", metadata)

if check_nans:
if check_nans and values is not None:
import numpy as np

if np.isnan(values).any():
Expand All @@ -140,7 +140,8 @@ def write(
for k, v in metadata.items():
handle.set(k, v)

handle.set_values(values)
if values is not None:
handle.set_values(values)

# Set values will set generatingProcessIdentifier to 255
if "generatingProcessIdentifier" in metadata:
Expand All @@ -167,7 +168,7 @@ def update_metadata(self, handle, metadata, compulsary):
# TODO: revisit that logic
combined = Combined(handle, metadata)

if "step" in metadata:
if "step" in metadata or "endStep" in metadata:
if combined["type"] == "an":
metadata["type"] = "fc"

Expand Down Expand Up @@ -338,6 +339,7 @@ def _gg_field(self, values, metadata):
1373624: (512, False),
2140702: (640, False),
5447118: (1024, False),
6599680: (1280, True),
8505906: (1280, False),
20696844: (2000, False),
}
Expand Down
2 changes: 1 addition & 1 deletion climetlab/readers/grib/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import logging

from climetlab.readers import Reader
from climetlab.readers.grib.index import MultiFieldSet
from climetlab.indexing.fieldset import MultiFieldSet
from climetlab.readers.grib.index.file import FieldSetInOneFile

LOG = logging.getLogger(__name__)
Expand Down
Loading

0 comments on commit 6863d2a

Please sign in to comment.