Merge pull request #10 from observingClouds/level2_to_merge

Unit aware Level2 reader update plus many small changes

Author: observingClouds

.dockerignore

@@ -27,15 +27,3 @@ jobs:
       - run: pipenv run flake8
       - run: pipenv run mypy
       - run: pipenv run pytest --cov --cov-fail-under=50
-
-  docker-image:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build docker image
-        run: docker build . -t pysonde:test
-
-      - name: Smoke test docker image
-        run: |
-          docker run --rm pysonde:test 10

.github/workflows/test.yml

@@ -44,6 +44,6 @@ repos:
     name: pytest
     stages: [push]
     language: system
-    entry: pipenv run pytest --cov --cov-fail-under=100
+    entry: pipenv run pytest --cov --cov-fail-under=50
     types: [python]
     pass_filenames: false

.pre-commit-config.yaml

@@ -1,68 +1,33 @@
 [[source]]
-name = "pypi"
-url = "https://pypi.python.org/simple"
+url = "https://pypi.org/simple"
 verify_ssl = true
-
-[requires]
-python_version = "3.6"
+name = "pypi"
 
 [packages]
-numpy = "==1.19.5"
-xarray = "==0.16.2"
-omegaconf = "*"
-netCDF4 = "==1.5.5.1"
-cftime = "==1.3.1"
-tqdm = "==4.56.0"
-dask = "==2021.1.1"
-toolz = "==0.11.1"
-appdirs = "==1.4.4"
-attrs = "==20.3.0"
-black = "==20.8b1"
-cfgv = "==3.2.0"
-click = "==7.1.2"
+black = "*"
 coverage = "==5.3.1"
-dataclasses = "==0.8"
-distlib = "==0.3.1"
-filelock = "==3.0.12"
-flake8 = "==3.8.4"
-identify = "==1.5.13"
-importlib-metadata = "==3.4.0"
-importlib-resources = "==5.1.0"
-iniconfig = "==1.1.1"
-isort = "==5.7.0"
-mccabe = "==0.6.1"
-mypy = "==0.800"
-mypy-extensions = "==0.4.3"
-nodeenv = "==1.5.0"
-pandas = "==1.1.5"
-pathspec = "==0.8.1"
-pint-xarray = "==0.1"
-pluggy = "==0.13.1"
-pre-commit = "==2.9.3"
-py = "==1.10.0"
-pycodestyle = "==2.6.0"
-pyflakes = "==2.2.0"
-pyparsing = "==2.4.7"
-pytest = "==6.2.1"
-pytest-cov = "==2.11.1"
-python-dateutil = "==2.8.1"
-pytz = "==2020.5"
-regex = "==2020.11.13"
-six = "==1.15.0"
-toml = "==0.10.2"
-typed-ast = "==1.4.2"
-typing-extensions = "==3.7.4.3"
-virtualenv = "==20.4.0"
-zipp = "==3.4.0"
-Pint = "==0.16.1"
-Pint-Pandas = "==0.1"
-PyYAML = "==5.4.1"
-
-[dev-packages]
-black = "==20.8b1"
+dask = "*"
 flake8 = "*"
 isort = "*"
 mypy = "*"
+numpy = "*"
+omegaconf = "*"
+pandas = "*"
+pint-xarray = "*"
 pre-commit = "*"
 pytest = "*"
 pytest-cov = "*"
+regex = "*"
+tqdm = "*"
+xarray = "*"
+Bottleneck = "*"
+MetPy = "*"
+netCDF4 = "*"
+Pint = "*"
+Pint-Pandas = "*"
+PyYAML = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

Pipfile

@@ -2,12 +2,14 @@ global_attrs:
     title: ${main.campaign} level 1 sounding data
     description: ???
     location: ${main.location}
+    location_coord: "{location_coordinates}"
     campaign_id: ${main.campaign_id}
     platform: ${main.platform}
     instrument: "Radiosonde RS41-SGP by Vaisala"
     instrument_id: "Vaisala-RS"
     time_of_launch_HHmmss: "{time_launch}"
     date_YYYYMMDD: "{day_launch}"
+    date_YYYYMMDDTHHMM: "{date_launch}"
     resolution: "{resolution}"
     number_of_probe: ${meta_level0.number_of_probe}
     sonde_type: ${meta_level0.sonde_type}
@@ -141,7 +143,7 @@ variables:
         attrs:
             standard_name: "wind_speed"
             long_name: "wind speed"
-            units: "m/s"
+            units: "knots"
             coordinates: "launch_time flight_time lon lat p"
         encodings:
             dtype: "float32"
@@ -196,3 +198,13 @@ variables:
         coordinates:
             - sounding
         internal_varname : "sounding"
+    level:
+        attrs:
+            cf_role: "level"
+            long_name: "level"
+            description: "Level of measurement"
+        encodings:
+            dtype: "int16"
+        coordinates:
+            - level
+        internal_varname : "level"

Pipfile.lock

@@ -5,17 +5,18 @@ setup:
     max_gap_fill: 50
 global_attrs:
     title: ${main.campaign} level 2 sounding data
-    description: ???
-    campaign_id: ${main.campaign_id}
-    platform: ${main.platform}
-    instrument: "Radiosonde RS41-SGP by Vaisala"
-    doi: ???
-    version: ???
+#    source: "{input_file}"
+    version: "{version}"
     history: "created with {package} ({version}) on {date}"
-    featureType: "trajectory"
-    acknowledgement: ???
-    references: ???
-    Conventions: "CF-1.7"
+#    description: ???
+#    campaign_id: ${main.campaign_id}
+#    platform: ${main.platform}
+#    instrument: "Radiosonde RS41-SGP by Vaisala"
+#    doi: ???
+#    featureType: "trajectory"
+#    acknowledgement: ???
+#    references: ???
+#    Conventions: "CF-1.7"
 
 coordinates:
     sounding:
@@ -49,6 +50,7 @@ variables:
         coordinates:
             - alt
             - nv
+        internal_varname: "alt_bnds"
     launch_time:
         attrs:
             standard_name: "time"
@@ -65,7 +67,7 @@ variables:
             long_name: "platform identifier"
             coordinates: "launch_time"
             flag_meanings: "BCO Meteor"
-            flag_values: [1, 2]
+            flag_values: "[1, 2]"
             units: "1"
         encodings:
             dtype: "int8"
@@ -76,8 +78,8 @@ variables:
         attrs:
             long_name: "indicator of vertical flight direction"
             flag_meanings: "ascending descending"
-            flag_values: [1,0]
-            valid_range: [ 0,1 ]
+            flag_values: "[1,0]"
+            valid_range: "[ 0,1 ]"
             coordinates: "launch_time"
         encodings:
             dtype: "int8"
@@ -192,7 +194,7 @@ variables:
         coordinates:
             - sounding
             - alt
-        internal_varname: "dew_point"
+        internal_varname: "dewpoint"
     wspd:
         attrs:
             standard_name: "wind_speed"
@@ -219,7 +221,7 @@ variables:
         coordinates:
             - sounding
             - alt
-        invernal_varname: "wind_u"
+        internal_varname: "wind_u"
     v:
         attrs:
             standard_name: "northward_wind"
@@ -233,7 +235,7 @@ variables:
         coordinates:
             - sounding
             - alt
-        invernal_varname: "wind_v"
+        internal_varname: "wind_v"
     mr:
         attrs:
             standard_name: "humidity_mixing_ratio"
@@ -301,7 +303,7 @@ variables:
         coordinates:
             - sounding
             - alt
-        internal_varname : "humidity"
+        internal_varname : "relative_humidity"
     N_ptu:
         attrs:
             standard_name: "number_of_observations"
@@ -331,7 +333,7 @@ variables:
             long_name: "bin method"
             description: "method used to derive level 2 PTU-data average"
             flag_meanings: "no_data interpolation averaging"
-            flag_values: [ 0, 1, 2 ]
+            flag_values: "[ 0, 1, 2 ]"
             coordinates: "lon lat flight_time launch_time"
         encodings:
             dtype: "int8"
@@ -344,7 +346,7 @@ variables:
             long_name: "bin method"
             description: "method used to derive level 2 GPS-data average"
             flag_meanings: "no_data interpolation averaging"
-            flag_values: [ 0, 1, 2 ]
+            flag_values: "[ 0, 1, 2 ]"
             coordinates: "lon lat flight_time launch_time"
         encodings:
             dtype: "int8"

config/level1.yaml

@@ -47,7 +47,9 @@ def create_dataset(cfg):
     cfg : OmegaConf
         Config containing dataset template
     """
+
     ds = xr.Dataset()
+    # ds = set_global_attrs(cfg, ds)
     ds = set_coords(cfg, ds)
     ds = set_variables(cfg, ds)
     return ds
@@ -79,10 +81,14 @@ def set_variables(cfg, ds):
     logging.debug("Add variables to dataset")
     if "variables" in cfg.keys():
         for var, params in cfg.variables.items():
-            coord_dict = {coord: ds[coord] for coord in params.coordinates}
-            ds[var] = xr.DataArray(None, coords=coord_dict, dims=params.coordinates)
-            if "attrs" in params.keys():
-                ds[var].attrs = params["attrs"]
-            if "encodings" in params.keys():
-                ds[var].encoding = params["encodings"]
+            if var == "level" or var == "sounding":
+                if "encodings" in params.keys():
+                    ds[var].encoding = params["encodings"]
+            else:
+                coord_dict = {coord: ds[coord] for coord in params.coordinates}
+                ds[var] = xr.DataArray(None, coords=coord_dict, dims=params.coordinates)
+                if "attrs" in params.keys():
+                    ds[var].attrs = params["attrs"]
+                if "encodings" in params.keys():
+                    ds[var].encoding = params["encodings"]
     return ds