From 3c00a1d581399258e1bdb466f94f62dd2725743c Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:58:32 +0200 Subject: [PATCH 1/3] Restructure of return objects to always include _meta --- synop2bufr/__init__.py | 72 ++++++++++++++++++++++++++++++++-------- synop2bufr/cli.py | 58 ++++++++++++++++++-------------- tests/test_synop2bufr.py | 2 +- 3 files changed, 94 insertions(+), 38 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 7b8cb9d..f1b5df9 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -33,7 +33,7 @@ from pymetdecoder import synop from csv2bufr import BUFRMessage -__version__ = '0.5.dev2' +__version__ = '0.6.dev1' LOGGER = logging.getLogger(__name__) @@ -1329,7 +1329,33 @@ def transform(data: str, metadata: str, year: int, LOGGER.error(e) error_msgs.append(str(e)) messages = [] # Fallback to an empty list if no reports extracted - yield {'warnings': warning_msgs, 'errors': error_msgs} + yield { + "_meta": { + "id": None, + "geometry": { + "type": "Point", + "coordinates": None + }, + "properties": { + "md5": None, + "wigos_station_identifier": None, + "datetime": None, + "originating_centre": None, + "data_category": None + }, + "result": { + "code": FAILED, + "message": "Error encoding, BUFR set to None", + "warnings": warning_msgs, + "errors": error_msgs + }, + "template": None, + "csv": None + } + } + # Reset warning and error messages array for next iteration + warning_msgs = [] + error_msgs = [] # Count how many conversions were successful using a dictionary conversion_success = {} @@ -1347,9 +1373,9 @@ def transform(data: str, metadata: str, year: int, match = re.match(nil_pattern, message) if match: LOGGER.warning( - f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa + f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa warning_msgs.append( - f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa + f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa continue # create dictionary to store / return result in @@ -1366,8 +1392,27 @@ def transform(data: str, metadata: str, year: int, LOGGER.error( f"Error parsing SYNOP report: {message}. {str(e)}!") error_msgs.append(f"Error parsing SYNOP report: {message}. {str(e)}!") # noqa - yield {'warnings': warning_msgs, 'errors': error_msgs} - continue + yield { + "_meta": { + "id": None, + "geometry": None, + "properties": { + "md5": None, + "wigos_station_identifier": None, + "datetime": None, + "originating_centre": None, + "data_category": None + }, + "result": { + "code": FAILED, + "message": "Error encoding, BUFR set to None", + "warnings": warning_msgs, + "errors": error_msgs + }, + "template": None, + "csv": None + } + } # Now determine and load the appropriate mappings # file depending on the value of the wind indicator. @@ -1540,8 +1585,8 @@ def transform(data: str, metadata: str, year: int, # Only convert to BUFR if there's no errors so far if conversion_success[tsi]: - # now identifier based on WSI and observation - # date as identifier + + # Use WSI and observation date as identifier isodate = message.get_datetime().strftime('%Y%m%dT%H%M%S') # Write message to CSV object in memory @@ -1564,7 +1609,9 @@ def transform(data: str, metadata: str, year: int, try: result["bufr4"] = message.as_bufr() # encode to BUFR - status = {"code": PASSED} + status = {"code": PASSED, + "warnings": warning_msgs, + "errors": error_msgs} except Exception as e: LOGGER.error("Error encoding BUFR, null returned") @@ -1574,7 +1621,9 @@ def transform(data: str, metadata: str, year: int, result["bufr4"] = None status = { "code": FAILED, - "message": "Error encoding, BUFR set to None" + "message": "Error encoding, BUFR set to None", + "warnings": warning_msgs, + "errors": error_msgs } conversion_success[tsi] = False @@ -1603,9 +1652,6 @@ def transform(data: str, metadata: str, year: int, "csv": csv_string } - result["warnings"] = warning_msgs - result["errors"] = error_msgs - # now yield result back to caller yield result diff --git a/synop2bufr/cli.py b/synop2bufr/cli.py index 71cc37d..efb6934 100644 --- a/synop2bufr/cli.py +++ b/synop2bufr/cli.py @@ -108,39 +108,49 @@ def transform(ctx, synop_file, metadata, output_dir, year, month, verbosity): try: result = transform_synop( content, metadata.read(), year, month - ) + ) except Exception as e: raise click.ClickException(e) for item in result: - # Write the CSV file of decoded data - csv_string = item["_meta"]["csv"] - timestamp = item["_meta"]["properties"]["datetime"].strftime( - '%Y%m%dT%H%M%S' - ) - filename = f"decoded_{timestamp}.csv" + # Return object may not have a datetime if there is an error + # parsing a report + if item["_meta"]["properties"].get("datetime") is not None: + timestamp = item["_meta"]["properties"]["datetime"].strftime( + '%Y%m%dT%H%M%S' + ) + filename = f"decoded_{timestamp}.csv" - if header_written: - mode = "a" # Append to file if headers - else: - mode = "w" # Write to file if no headers + # Write the CSV file of decoded data + csv_string = item["_meta"]["csv"] - with open(filename, mode) as f: - if not header_written: - # Write the whole string including headers - f.write(csv_string) - header_written = True + if header_written: + mode = "a" # Append to file if headers else: - # Skip the header row of the string - f.write(csv_string.split("\n")[1]) - - # Write the BUFR file - key = item['_meta']["id"] - bufr_filename = f"{output_dir}{os.sep}{key}.bufr4" - with open(bufr_filename, "wb") as fh: - fh.write(item["bufr4"]) + mode = "w" # Write to file if no headers + + with open(filename, mode) as f: + # Check there was no problem writing the report to CSV + if csv_string is not None: + if not header_written: + # Write the whole string including headers + f.write(csv_string) + header_written = True + else: + # Skip the header row of the string + f.write(csv_string.split("\n")[1]) + + # Check there was no problem encoding the BUFR message + # before writing to a file + if item.get("bufr4") is not None: + # Write the BUFR file + key = item['_meta']["id"] + bufr_filename = f"{output_dir}{os.sep}{key}.bufr4" + + with open(bufr_filename, "wb") as fh: + fh.write(item["bufr4"]) except Exception as e: raise click.ClickException(e) diff --git a/tests/test_synop2bufr.py b/tests/test_synop2bufr.py index b92d705..64cf34f 100644 --- a/tests/test_synop2bufr.py +++ b/tests/test_synop2bufr.py @@ -263,7 +263,7 @@ def test_range_qc(metadata_string): result = transform(out_of_range, metadata_string, 2000, 1) for item in result: - warning_msgs = item["warnings"] + warning_msgs = item["_meta"]["result"]["warnings"] assert "#1#nonCoordinatePressure: Value (47650.0) out of valid range (50000 - 108000).; Element set to missing" in warning_msgs # noqa assert "#1#airTemperature: Value (334.15) out of valid range (193.15 - 333.15).; Element set to missing" in warning_msgs # noqa From 2607932f2e2e363d441ee865877d5a5c3b5dcee2 Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:13:04 +0200 Subject: [PATCH 2/3] Added extra warning/error reset --- synop2bufr/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index f1b5df9..32c679b 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1413,6 +1413,10 @@ def transform(data: str, metadata: str, year: int, "csv": None } } + # Reset warning and error messages array for next iteration + warning_msgs = [] + error_msgs = [] + continue # Now determine and load the appropriate mappings # file depending on the value of the wind indicator. From c5a6b1f2f35bda3c818d5d1ecc67503010b1e75c Mon Sep 17 00:00:00 2001 From: RoryPTB <47696929+RoryPTB@users.noreply.github.com> Date: Fri, 15 Sep 2023 14:06:20 +0200 Subject: [PATCH 3/3] Set geometry to None when there is an error --- synop2bufr/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 32c679b..b9861c5 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -33,7 +33,7 @@ from pymetdecoder import synop from csv2bufr import BUFRMessage -__version__ = '0.6.dev1' +__version__ = '0.6.0' LOGGER = logging.getLogger(__name__) @@ -1332,10 +1332,7 @@ def transform(data: str, metadata: str, year: int, yield { "_meta": { "id": None, - "geometry": { - "type": "Point", - "coordinates": None - }, + "geometry": None, "properties": { "md5": None, "wigos_station_identifier": None,