Skip to content

Commit

Permalink
Mapping update fix final (hopefully)
Browse files Browse the repository at this point in the history
  • Loading branch information
@RoryPTB
RoryPTB committed Feb 16, 2024
1 parent e45092a commit 75c8e53
Show file tree
Hide file tree
Showing 7 changed files with 27 additions and 59 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export BUFR_ORIGINATING_SUBCENTRE=<subcentre_value>
To run synop2bufr from a Docker container:

```console
docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash
docker run -it -v /$(pwd):/local wmoim/dim_eccodes_baseimage:2.34.0 bash
apt-get update && apt-get install -y git
cd /local
python3 setup.py install
Expand Down
2 changes: 1 addition & 1 deletion docs/source/installation.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ You can then run synop2bufr from an ecCodes base image as follows:

.. code-block:: bash
docker run -it -v ${pwd}:/local wmoim/dim_eccodes_baseimage:2.34.0 bash
docker run -it -v /$(pwd):/local wmoim/dim_eccodes_baseimage:2.34.0 bash
apt-get update && apt-get install -y git
cd /local
python3 setup.py install
Expand Down
4 changes: 4 additions & 0 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import os
import re
from setuptools import Command, find_packages, setup
import subprocess


class PyTest(Command):
Expand Down Expand Up @@ -73,6 +74,9 @@ def get_package_version():
if (os.path.exists('MANIFEST')):
os.unlink('MANIFEST')

# Install dependencies not on PyPI
subprocess.check_call("pip install 'pymetdecoder @ git+https://github.com/wmo-im/pymetdecoder.git@v0.1.11'", shell=True)
subprocess.check_call("pip install 'csv2bufr @ git+https://github.com/wmo-im/csv2bufr.git@v0.8.1'", shell=True)

setup(
name='synop2bufr',
Expand Down
40 changes: 5 additions & 35 deletions synop2bufr/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1571,18 +1571,6 @@ def update_data_mapping(mapping: list, update: dict):
mapping.append(update)
return mapping

# Add delayed descriptor replication factor (0 31 001)
# to represent the number of section 3 cloud groups
if num_s3_clouds > 0:
s3_delayed_replication = {
"eccodes_key":
"#1#delayedDescriptorReplicationFactor",
"value": f"const:{num_s3_clouds}"
}
mapping['data'] = update_data_mapping(
mapping=mapping['data'],
update=s3_delayed_replication)

# Now add the rest of the mappings for section 3 clouds
for idx in range(num_s3_clouds):
# Build the dictionary of mappings for section 3
Expand All @@ -1597,9 +1585,9 @@ def update_data_mapping(mapping: list, update: dict):
# low-high cloud amount, low-high cloud drift)
s3_mappings = [
{"eccodes_key":
f"#{idx+8}#verticalSignificanceSurfaceObservations", # noqa
f"#{idx+6}#verticalSignificanceSurfaceObservations", # noqa
"value": f"data:vs_s3_{idx+1}"},
{"eccodes_key": f"#{idx+3}#cloudAmount",
{"eccodes_key": f"#{idx+2}#cloudAmount",
"value": f"data:cloud_amount_s3_{idx+1}",
"valid_min": "const:0",
"valid_max": "const:8"},
Expand All @@ -1612,18 +1600,6 @@ def update_data_mapping(mapping: list, update: dict):
mapping['data'] = update_data_mapping(
mapping=mapping['data'], update=m)

# Add delayed descriptor replication factor (0 31 001)
# to represent the number of section 4 cloud groups
if num_s4_clouds > 0:
s4_delayed_replication = {
"eccodes_key":
"#2#delayedDescriptorReplicationFactor",
"value": f"const:{num_s4_clouds}"
}
mapping['data'] = update_data_mapping(
mapping=mapping['data'],
update=s4_delayed_replication)

# Now add the rest of the mappings for section 4 clouds
for idx in range(num_s4_clouds):
# Based upon the station height metadata, the
Expand All @@ -1645,10 +1621,10 @@ def update_data_mapping(mapping: list, update: dict):
# the above, so we must add 'num_s3_clouds'
s4_mappings = [
{"eccodes_key":
f"#{idx+num_s3_clouds+8}#verticalSignificanceSurfaceObservations", # noqa
f"#{idx+num_s3_clouds+6}#verticalSignificanceSurfaceObservations", # noqa
"value": f"const:{vs_s4}"},
{"eccodes_key":
f"#{idx+num_s3_clouds+3}#cloudAmount",
f"#{idx+num_s3_clouds+2}#cloudAmount",
"value": f"data:cloud_amount_s4_{idx+1}",
"valid_min": "const:0",
"valid_max": "const:8"},
Expand All @@ -1671,11 +1647,6 @@ def update_data_mapping(mapping: list, update: dict):
error_msgs.append(
f"Missing station height for station {tsi}")
conversion_success[tsi] = False
# Now section 3 and 4 cloud groups have been
# added to the mapping file, write the file
# for debugging purposes
with open('updated_mappings.json', 'w') as f:
json.dump(mapping, f, indent=2)

if conversion_success[tsi]:
# At this point we have a dictionary for the data, a
Expand All @@ -1684,8 +1655,7 @@ def update_data_mapping(mapping: list, update: dict):
unexpanded_descriptors = [301150, bufr_template]
short_delayed_replications = []
# update replications
delayed_replications = [max(1, num_s3_clouds),
max(1, num_s4_clouds)]
delayed_replications = [num_s3_clouds, num_s4_clouds]
extended_delayed_replications = []
table_version = 37
try:
Expand Down
13 changes: 5 additions & 8 deletions synop2bufr/resources/synop-mappings-307080.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"id": "b3abe03b-e502-48c5-a225-133b189207ee"
},
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [1,1],
"inputDelayedDescriptorReplicationFactor": [0,0],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"names_on_row": 1,
"column_names_row": 1,
"wigos_station_identifier": "data:Station_ID",
"header":[
{"eccodes_key": "edition", "value": "const:4"},
Expand Down Expand Up @@ -72,15 +72,12 @@
{"eccodes_key": "#1#cloudType", "value": "data:low_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#2#cloudType", "value": "data:middle_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#cloudType", "value": "data:high_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:low_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
{"eccodes_key": "#2#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:middle_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
{"eccodes_key": "#3#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:high_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
{"eccodes_key": "#5#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#6#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#7#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#1#stateOfGround", "value": "data:ground_state", "valid_min": "const:0", "valid_max": "const:31"},
{"eccodes_key": "#1#totalSnowDepth", "value": "data:snow_depth", "valid_min": "const:0.0", "valid_max": "const:25"},
{"eccodes_key": "#1#groundMinimumTemperaturePast12Hours", "value": "data:ground_temperature", "valid_min": "const:0.0", "valid_max": "const:655.35"},
Expand Down
13 changes: 5 additions & 8 deletions synop2bufr/resources/synop-mappings-307096.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"id": "f7b5e865-fd82-4e93-bdc3-40b705d73860"
},
"inputShortDelayedDescriptorReplicationFactor": [],
"inputDelayedDescriptorReplicationFactor": [1,1],
"inputDelayedDescriptorReplicationFactor": [0,0],
"inputExtendedDelayedDescriptorReplicationFactor": [],
"number_header_rows": 1,
"names_on_row": 1,
"column_names_row": 1,
"wigos_station_identifier": "data:Station_ID",
"header":[
{"eccodes_key": "edition", "value": "const:4"},
Expand Down Expand Up @@ -71,15 +71,12 @@
{"eccodes_key": "#1#verticalSignificanceSurfaceObservations", "value": "data:cloud_vs_s1", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#1#cloudAmount", "value": "data:cloud_amount_s1", "valid_min": "const:0", "valid_max": "const:8"},
{"eccodes_key": "#1#heightOfBaseOfCloud", "value": "data:lowest_cloud_base", "valid_min": "const:0", "valid_max": "const:20070"},
{"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#1#cloudType", "value": "data:low_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#2#cloudType", "value": "data:middle_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#cloudType", "value": "data:high_cloud_type", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#5#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#6#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#7#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#2#verticalSignificanceSurfaceObservations", "value": "const:7", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#3#verticalSignificanceSurfaceObservations", "value": "const:8", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#4#verticalSignificanceSurfaceObservations", "value": "const:9", "valid_min": "const:0", "valid_max": "const:63"},
{"eccodes_key": "#1#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:low_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
{"eccodes_key": "#2#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:middle_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
{"eccodes_key": "#3#trueDirectionFromWhichAPhenomenonOrCloudsAreMovingOrInWhichTheyAreObserved", "value": "data:high_cloud_drift_direction", "valid_min": "const:0.0", "valid_max": "const:360"},
Expand Down
12 changes: 6 additions & 6 deletions tests/test_synop2bufr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,9 @@ def test_bufr_307080(multiple_reports_307080, metadata_string):
for item in result:
msgs[item['_meta']['id']] = item
# Test the md5 keys
assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'db1d075059a70e978647eb1cb0a3f4d2' # noqa
assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '82035d667ce986cb528d0b11c2c8bd77' # noqa
assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '598aaead55964cc01ba5a58e53a59e9f' # noqa
assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'deb294033aee19f090aabc63660f273c' # noqa
assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'ef62c7b58ddc99724a585d6cd9b16628' # noqa
assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '7fc119bab009baf45bbbb49e0b3dd5fd' # noqa

# Test the bufr template used for all the reports
# (they should be the same for every report)
Expand All @@ -168,9 +168,9 @@ def test_bufr_307096(multiple_reports_307096, metadata_string):
for item in result:
msgs[item['_meta']['id']] = item
# Test the md5 keys
assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '30200eed11629d03562aafb899bb7729' # noqa
assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '7a368736fb403aa75408633fa17366e3' # noqa
assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '91fef5babfcba6af9358b7b7e38f6960' # noqa
assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '9e84ebe4283f114353c454e37b052076' # noqa
assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '8d6fb0d39ffee750f3da3228bf673862' # noqa
assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '8d70449d5f9f5021ecedd445a9a0b90a' # noqa

# Test the bufr template used for all the reports
# (they should be the same for every report)
Expand Down

0 comments on commit 75c8e53

Please sign in to comment.