Skip to content

Commit

Permalink
qa
Browse files Browse the repository at this point in the history
  • Loading branch information
EddyCMWF committed Aug 29, 2023
1 parent e33ca61 commit 05178e0
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 1,289 deletions.
27 changes: 13 additions & 14 deletions .github/workflows/on-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,19 @@ jobs:
runs-on: ubuntu-latest
needs: [combine-environments]
steps:
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Build distributions
run: |
$CONDA/bin/python -m pip install build
$CONDA/bin/python -m build
- name: Publish a Python distribution to PyPI
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
- uses: actions/checkout@v3
with:
ref: ${{ github.event.pull_request.head.sha || github.ref }}
- name: Build distributions
run: |
$CONDA/bin/python -m pip install build
$CONDA/bin/python -m build
- name: Publish a Python distribution to PyPI
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_API_TOKEN }}
# BOPEN METHOD:
# steps:
# - uses: actions/checkout@v3
Expand All @@ -99,4 +99,3 @@ jobs:
# with:
# user: __token__
# password: ${{ secrets.PYPI_API_TOKEN }}

1 change: 0 additions & 1 deletion ci/combined-environment-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,3 @@ dependencies:
- xarray
- pip:
- git+https://github.com/ecmwf/multiurl

56 changes: 26 additions & 30 deletions earthkit/climate/shapes.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import logging
import typing as T
from copy import deepcopy
import logging

import geopandas as gpd
import numpy as np
Expand All @@ -15,6 +15,7 @@

logger = logging.getLogger(__name__)


def _transform_from_latlon(lat, lon):
"""
Return an Affine transformation of input 1D arrays of lat and lon.
Expand Down Expand Up @@ -286,7 +287,6 @@ def reduce(
return out
else:
return _reduce_dataarray(dataarray, geodataframe, **kwargs)



def _reduce_dataarray(
Expand Down Expand Up @@ -338,15 +338,16 @@ def _reduce_dataarray(
Each slice of layer corresponds to a feature in layer.
"""

extra_out_attrs = {}
# If how is string, fetch function from dictionary:
if isinstance(how, str):
how_label = deepcopy(how)
how = get_how(how)
assert isinstance(how, T.Callable), f"how must be a callable: {how}"

new_long_name = f"{how_label.title()} {dataarray.attrs.get('long_name', dataarray.name)}"
new_long_name = (
f"{how_label.title()} {dataarray.attrs.get('long_name', dataarray.name)}"
)
new_short_name = f"{dataarray.name}_{how_label or how.__name__}"

if isinstance(extra_reduce_dims, str):
Expand All @@ -373,10 +374,8 @@ def _reduce_dataarray(
# If weighted, use xarray weighted arrays which correctly handle missing values etc.
if weights is not None:
dataarray.weighted(weights)

reduced = this.reduce(
how, dim=reduce_dims, **red_kwargs
).compute()

reduced = this.reduce(how, dim=reduce_dims, **red_kwargs).compute()
reduced = reduced.assign_attrs(dataarray.attrs)
reduced_list.append(reduced)
# context.debug(f"Shapes.average reduced ({i}): {reduced} \n{i}")
Expand Down Expand Up @@ -407,16 +406,18 @@ def _reduce_dataarray(
}
)
out_xr = out_xr.assign_attrs()
if return_as in ["pandas"]: # Return as a fully expanded pandas.DataFrame
logger.warn("Returning reduced data in pandas format is considered experimental and the format")
if return_as in ["pandas"]: # Return as a fully expanded pandas.DataFrame
logger.warn(
"Returning reduced data in pandas format is considered experimental and the format"
)
# Convert to DataFrame
out = geodataframe.set_index(mask_dim)
out = out.join(out_xr.to_dataframe())
out.attrs.update({
**extra_out_attrs
})
out.attrs.update({**extra_out_attrs})
elif return_as in ["pandas_compact"]:
logger.warn("Returning reduced data in pandas format is considered experimental and the format")
logger.warn(
"Returning reduced data in pandas format is considered experimental and the format"
)
# Out dims for attributes:
out_dims = {
dim: dataarray.coords.get(dim).values if dim in dataarray.coords else None
Expand All @@ -427,25 +428,20 @@ def _reduce_dataarray(
reduced_list = [red.values for red in reduced_list]
# reduced_list = [red.to_dataframe() for red in reduced_list]
reduce_attrs = geodataframe.attrs.get("reduce_attrs", {})
reduce_attrs.update({
f"{dataarray.name}": dataarray.attrs,
f"{new_short_name}": {
"dims": out_dims,
"long_name": new_long_name,
"units": dataarray.attrs.get('units'),
**extra_out_attrs
},
})
out = geodataframe.assign(**{new_short_name: reduced_list})
out.attrs.update(
reduce_attrs.update(
{
"reduce_attrs": reduce_attrs
f"{dataarray.name}": dataarray.attrs,
f"{new_short_name}": {
"dims": out_dims,
"long_name": new_long_name,
"units": dataarray.attrs.get("units"),
**extra_out_attrs,
},
}
)
out = geodataframe.assign(**{new_short_name: reduced_list})
out.attrs.update({"reduce_attrs": reduce_attrs})
else:
out = out_xr.assign_attrs({
**geodataframe.attrs,
**extra_out_attrs
})
out = out_xr.assign_attrs({**geodataframe.attrs, **extra_out_attrs})

return out
2 changes: 1 addition & 1 deletion notebooks/requirements-for-notebooks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ conda install -c conda-forge jupyter
conda install -c conda-forge geopandas

pip install rasterio
pip install matplotlib
pip install matplotlib
Loading

0 comments on commit 05178e0

Please sign in to comment.