Merge branch main into rory-s4-fix

.github/workflows/tests.yml

@@ -8,7 +8,9 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7, 3.8, 3.9]
-
+    env:
+      BUFR_ORIGINATING_CENTRE: 123
+      BUFR_ORIGINATING_SUBCENTRE: 123
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.gitignore

@@ -42,6 +42,8 @@ nosetests.xml
 logs
 .vscode/
 .vscode/settings.json
+# Ignore decoded CSV files
+decoded_*.csv
 
 # pycharm
 .idea

Dockerfile

@@ -14,9 +14,13 @@ RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";"
     && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \
     && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip
 
+# Environment variables
+
 ENV LOG_LEVEL=INFO
+# The following need to changed to the correct values for your centre!
+ENV BUFR_ORIGINATING_CENTRE=65535
+ENV BUFR_ORIGINATING_SUBCENTRE=65535
 
-#WORKDIR /build
 # copy the app
 COPY . /build
 

README.md

@@ -13,6 +13,24 @@ The synop2bufr Python module contains both a command line interface and API to c
 
 Dependencies are listed in [requirements.txt](https://github.com/wmo-im/synop2bufr/blob/main/requirements.txt). Dependencies are automatically installed during synop2bufr installation.
 
+### Setting Environment Variables
+
+Before using synop2bufr, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).**
+
+It is recommended that you set these environment variables in the Dockerfile, by editing the following lines with your originating centre and subcentre values:
+
+```bash
+ENV BUFR_ORIGINATING_CENTRE=<centre_value>
+ENV BUFR_ORIGINATING_SUBCENTRE=<subcentre_value>
+```
+
+Alternatively, you can set these environment variables in your shell if you want to run synop2bufr on your local machine. Here's how you can do it in a Bash shell:
+
+```bash
+export BUFR_ORIGINATING_CENTRE=<centre_value>
+export BUFR_ORIGINATING_SUBCENTRE=<subcentre_value>
+```
+
 ## Running
 
 To run synop2bufr from a Docker container:
@@ -33,9 +51,9 @@ synop2bufr data transform --metadata data/station_list.csv --year 2023 --month 0
 
 To run synop2bufr inside a Lambda function on Amazon Web Services, please refer to [aws-lambda/README.md](aws-lambda/README.md) and use this [Dockerfile](aws-lambda/Dockerfile) to build the container image for the Lambda function.
 
-## Usage Guide
+## API Usage Guide
 
-Here we detail how synop2bufr can be used.
+Here we detail how the synop2bufr API can be used in Python.
 
 To begin, suppose we have some SYNOP data.
 

docs/source/quickstart.rst

@@ -31,6 +31,27 @@ Command line interface (CLI)
        --month <month-of-observation> \
        <input-fm12.txt>
 
+Setting Environment Variables
+-----------------------------
+
+Before running the `synop2bufr data transform` command, we highly encourage you to set the `BUFR_ORIGINATING_CENTRE` and `BUFR_ORIGINATING_SUBCENTRE` environment variables. These variables are used to specify the originating centre and subcentre of the SYNOP messages. **Without these set, they will default to missing (255).**
+
+It is recommended that you set these environment variables in the Dockerfile, by editing the following lines with your originating centre and subcentre values:
+
+.. code-block:: shell
+
+  ENV BUFR_ORIGINATING_CENTRE=<centre_value>
+  ENV BUFR_ORIGINATING_SUBCENTRE=<subcentre_value>
+
+Alternatively, you can set these environment variables in your shell if you want to run synop2bufr on your local machine. Here's how you can do it in a Bash shell:
+
+.. code-block:: shell
+
+   export BUFR_ORIGINATING_CENTRE=<centre_value>
+   export BUFR_ORIGINATING_SUBCENTRE=<subcentre_value>
+
+Now, you can run the `synop2bufr data transform` command as described in the previous section.
+
 Input FM-12 file (input-fm12.txt)
 ---------------------------------
 The FM-12 input data format is described in the `WMO Manual on Codes, Volume I.1 <https://library.wmo.int/doc_num.php?explnum_id=10235>`__.

synop2bufr/__init__.py

@@ -1278,10 +1278,19 @@ def transform(data: str, metadata: str, year: int,
 
     :returns: iterator
     """
+    # =============================================
     # Make warning and error messages array global
+    # =============================================
     global warning_msgs
     global error_msgs
 
+    # Boolean to ensure environment variable warning is only displayed once
+    # Note: The resetting of the warning_msgs array for
+    # each report necessitates this approach, because
+    # we want to ensure the warning is only appended
+    # to the first conversion
+    can_var_warning_be_displayed = True
+
     # ===================
     # First parse metadata file
     # ===================
@@ -1310,7 +1319,6 @@ def transform(data: str, metadata: str, year: int,
                 error_msgs.append(str(e))
 
         fh.close()
-        # metadata = metadata_dict[wsi]
     else:
         LOGGER.error("Invalid metadata")
         raise ValueError("Invalid metadata")
@@ -1504,12 +1512,47 @@ def transform(data: str, metadata: str, year: int,
                 else:
                     # If station has not been found in the station
                     # list, don't repeat warning unnecessarily
-                    if not (f"Station {tsi} not found in station file"
-                            in warning_msgs):
+                    if f"Station {tsi} not found in station file" not in warning_msgs:  # noqa
                         LOGGER.warning(f"Invalid metadata for station {tsi} found in station file, unable to parse")  # noqa
                         warning_msgs.append(f"Invalid metadata for station {tsi} found in station file, unable to parse")  # noqa
 
+            # Add information to the mappings
             if conversion_success[tsi]:
+                # First check if the BUFR header centre
+                # and subcentre codes are present
+                missing_env_vars = []
+
+                if os.environ.get("BUFR_ORIGINATING_CENTRE") is None:
+                    missing_env_vars.append("BUFR_ORIGINATING_CENTRE")
+                else:
+                    # Add the BUFR header centre and subcentre to mappings
+                    mapping["header"].append({
+                        "eccodes_key": "bufrHeaderCentre",
+                        "value": f"const:{os.environ.get('BUFR_ORIGINATING_CENTRE')}"  # noqa
+                    })
+
+                if os.environ.get("BUFR_ORIGINATING_SUBCENTRE") is None:
+                    missing_env_vars.append("BUFR_ORIGINATING_SUBCENTRE")
+                else:
+                    mapping["header"].append({
+                        "eccodes_key": "bufrHeaderSubCentre",
+                        "value": f"const:{os.environ.get('BUFR_ORIGINATING_SUBCENTRE')}"  # noqa
+                    })
+
+                # If either of these environment variables are not set,
+                # we will default to missing and warn the user once
+                if missing_env_vars and can_var_warning_be_displayed:
+                    # Display ewarning messages
+                    for var in missing_env_vars:
+                        var_warning = f"The {var} environment variable is not set, will default to missing!"  # noqa
+                        LOGGER.warning(var_warning)
+                        warning_msgs.append(var_warning)
+                        can_var_warning_be_displayed = False
+                    # Stop duplicated warnings
+                    can_var_warning_be_displayed = False
+
+                # Now we need to add the mappings for the cloud groups
+                # of section 3 and 4
                 try:
                     # Define a new method which handles the updating of
                     # the mapping file with section 3 and 4 cloud data

synop2bufr/resources/synop-mappings-307080.json

@@ -1,4 +1,15 @@
 {
+    "conformsTo": "csv2bufr-template-v2.json",
+    "metadata": {
+        "label": "Surface land stations (manual)",
+        "description": "csv2bufr mapping template for use with synop2bufr and converting FM-12 reports to BUFR, using BUFR sequence 301150, 307080. This mapping is used when the wind variables are estimations.",
+        "version": "2",
+        "author": "Rory Burke",
+        "editor": "",
+        "dateCreated": "2023-09-01",
+        "dateModified": "2024-02-05",
+        "id": "b3abe03b-e502-48c5-a225-133b189207ee"
+    },
     "inputShortDelayedDescriptorReplicationFactor": [],
     "inputDelayedDescriptorReplicationFactor": [1,1],
     "inputExtendedDelayedDescriptorReplicationFactor": [],
@@ -8,8 +19,8 @@
     "header":[
         {"eccodes_key": "edition", "value": "const:4"},
         {"eccodes_key": "masterTableNumber", "value": "const:0"},
-        {"eccodes_key": "bufrHeaderCentre", "value": "const:0"},
-        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"},
+        {"eccodes_key": "bufrHeaderCentre", "value": "const:65535"},
+        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:65535"},
         {"eccodes_key": "updateSequenceNumber", "value": "const:0"},
         {"eccodes_key": "dataCategory", "value": "const:0"},
         {"eccodes_key": "internationalDataSubCategory", "value": "const:2"},

synop2bufr/resources/synop-mappings-307096.json

@@ -1,4 +1,15 @@
 {
+    "conformsTo": "csv2bufr-template-v2.json",
+    "metadata": {
+        "label": "Surface land stations (manual)",
+        "description": "csv2bufr mapping template for use with synop2bufr and converting FM-12 reports to BUFR, using BUFR sequence 301150, 307096. This mapping is used when the wind variables are measured with an anemometer.",
+        "version": "2",
+        "author": "Rory Burke",
+        "editor": "",
+        "dateCreated": "2023-09-01",
+        "dateModified": "2024-02-05",
+        "id": "f7b5e865-fd82-4e93-bdc3-40b705d73860"
+    },
     "inputShortDelayedDescriptorReplicationFactor": [],
     "inputDelayedDescriptorReplicationFactor": [1,1],
     "inputExtendedDelayedDescriptorReplicationFactor": [],
@@ -8,8 +19,8 @@
     "header":[
         {"eccodes_key": "edition", "value": "const:4"},
         {"eccodes_key": "masterTableNumber", "value": "const:0"},
-        {"eccodes_key": "bufrHeaderCentre", "value": "const:0"},
-        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:0"},
+        {"eccodes_key": "bufrHeaderCentre", "value": "const:65535"},
+        {"eccodes_key": "bufrHeaderSubCentre", "value": "const:65535"},
         {"eccodes_key": "updateSequenceNumber", "value": "const:0"},
         {"eccodes_key": "dataCategory", "value": "const:0"},
         {"eccodes_key": "internationalDataSubCategory", "value": "const:2"},

tests/test_synop2bufr.py

@@ -149,9 +149,9 @@ def test_bufr_307080(multiple_reports_307080, metadata_string):
     for item in result:
         msgs[item['_meta']['id']] = item
     # Test the md5 keys
-    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == 'f5874a7e7da9f6132018690f73017a10'  # noqa
-    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '230d74d29fba410e38cbb6fae1d6fb3f'  # noqa
-    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '15cefdcd27eded01db05cd5acf7abe02'  # noqa
+    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '1e564e1ec2d679bbc120141ba031ab7a'  # noqa
+    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'db62277233118df3f1cf7b6a073f1cbe'  # noqa
+    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '538db43645fb4b2459edfcb467048b7a'  # noqa
 
     # Test the bufr template used for all the reports
     # (they should be the same for every report)
@@ -168,9 +168,9 @@ def test_bufr_307096(multiple_reports_307096, metadata_string):
     for item in result:
         msgs[item['_meta']['id']] = item
     # Test the md5 keys
-    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '3611e3fd99a441f6b80bfa5bea607fee'  # noqa
-    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == '1948789be15abd74b9312d0be1e179d6'  # noqa
-    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '19fa379ecf284c6b17f51baa36cfc473'  # noqa
+    assert msgs['WIGOS_0-20000-0-15015_20220321T120000']['_meta']['properties']['md5'] == '5f1744ec26875630efca0e1583cddca9'  # noqa
+    assert msgs['WIGOS_0-20000-0-15020_20220321T120000']['_meta']['properties']['md5'] == 'e2dc1199d4e38fae25d26ded815597da'  # noqa
+    assert msgs['WIGOS_0-20000-0-15090_20220321T120000']['_meta']['properties']['md5'] == '7c352acb43530946f2445a95eb349e68'  # noqa
 
     # Test the bufr template used for all the reports
     # (they should be the same for every report)