Skip to content

Commit

Permalink
Restructure of return objects to always include _meta
Browse files Browse the repository at this point in the history
  • Loading branch information
@RoryPTB
RoryPTB committed Sep 15, 2023
1 parent 68292bd commit 3c00a1d
Show file tree
Hide file tree
Showing 3 changed files with 94 additions and 38 deletions.
72 changes: 59 additions & 13 deletions synop2bufr/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
from pymetdecoder import synop
from csv2bufr import BUFRMessage

__version__ = '0.5.dev2'
__version__ = '0.6.dev1'

LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -1329,7 +1329,33 @@ def transform(data: str, metadata: str, year: int,
LOGGER.error(e)
error_msgs.append(str(e))
messages = [] # Fallback to an empty list if no reports extracted
yield {'warnings': warning_msgs, 'errors': error_msgs}
yield {
"_meta": {
"id": None,
"geometry": {
"type": "Point",
"coordinates": None
},
"properties": {
"md5": None,
"wigos_station_identifier": None,
"datetime": None,
"originating_centre": None,
"data_category": None
},
"result": {
"code": FAILED,
"message": "Error encoding, BUFR set to None",
"warnings": warning_msgs,
"errors": error_msgs
},
"template": None,
"csv": None
}
}
# Reset warning and error messages array for next iteration
warning_msgs = []
error_msgs = []

# Count how many conversions were successful using a dictionary
conversion_success = {}
Expand All @@ -1347,9 +1373,9 @@ def transform(data: str, metadata: str, year: int,
match = re.match(nil_pattern, message)
if match:
LOGGER.warning(
f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa
f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa
warning_msgs.append(
f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa
f"NIL report detected for station {match.group(1)}, no BUFR file created.") # noqa
continue

# create dictionary to store / return result in
Expand All @@ -1366,8 +1392,27 @@ def transform(data: str, metadata: str, year: int,
LOGGER.error(
f"Error parsing SYNOP report: {message}. {str(e)}!")
error_msgs.append(f"Error parsing SYNOP report: {message}. {str(e)}!") # noqa
yield {'warnings': warning_msgs, 'errors': error_msgs}
continue
yield {
"_meta": {
"id": None,
"geometry": None,
"properties": {
"md5": None,
"wigos_station_identifier": None,
"datetime": None,
"originating_centre": None,
"data_category": None
},
"result": {
"code": FAILED,
"message": "Error encoding, BUFR set to None",
"warnings": warning_msgs,
"errors": error_msgs
},
"template": None,
"csv": None
}
}

# Now determine and load the appropriate mappings
# file depending on the value of the wind indicator.
Expand Down Expand Up @@ -1540,8 +1585,8 @@ def transform(data: str, metadata: str, year: int,

# Only convert to BUFR if there's no errors so far
if conversion_success[tsi]:
# now identifier based on WSI and observation
# date as identifier

# Use WSI and observation date as identifier
isodate = message.get_datetime().strftime('%Y%m%dT%H%M%S')

# Write message to CSV object in memory
Expand All @@ -1564,7 +1609,9 @@ def transform(data: str, metadata: str, year: int,

try:
result["bufr4"] = message.as_bufr() # encode to BUFR
status = {"code": PASSED}
status = {"code": PASSED,
"warnings": warning_msgs,
"errors": error_msgs}

except Exception as e:
LOGGER.error("Error encoding BUFR, null returned")
Expand All @@ -1574,7 +1621,9 @@ def transform(data: str, metadata: str, year: int,
result["bufr4"] = None
status = {
"code": FAILED,
"message": "Error encoding, BUFR set to None"
"message": "Error encoding, BUFR set to None",
"warnings": warning_msgs,
"errors": error_msgs
}
conversion_success[tsi] = False

Expand Down Expand Up @@ -1603,9 +1652,6 @@ def transform(data: str, metadata: str, year: int,
"csv": csv_string
}

result["warnings"] = warning_msgs
result["errors"] = error_msgs

# now yield result back to caller
yield result

Expand Down
58 changes: 34 additions & 24 deletions synop2bufr/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,39 +108,49 @@ def transform(ctx, synop_file, metadata, output_dir, year, month, verbosity):
try:
result = transform_synop(
content, metadata.read(), year, month
)
)

except Exception as e:
raise click.ClickException(e)

for item in result:
# Write the CSV file of decoded data
csv_string = item["_meta"]["csv"]
timestamp = item["_meta"]["properties"]["datetime"].strftime(
'%Y%m%dT%H%M%S'
)

filename = f"decoded_{timestamp}.csv"
# Return object may not have a datetime if there is an error
# parsing a report
if item["_meta"]["properties"].get("datetime") is not None:
timestamp = item["_meta"]["properties"]["datetime"].strftime(
'%Y%m%dT%H%M%S'
)
filename = f"decoded_{timestamp}.csv"

if header_written:
mode = "a" # Append to file if headers
else:
mode = "w" # Write to file if no headers
# Write the CSV file of decoded data
csv_string = item["_meta"]["csv"]

with open(filename, mode) as f:
if not header_written:
# Write the whole string including headers
f.write(csv_string)
header_written = True
if header_written:
mode = "a" # Append to file if headers
else:
# Skip the header row of the string
f.write(csv_string.split("\n")[1])

# Write the BUFR file
key = item['_meta']["id"]
bufr_filename = f"{output_dir}{os.sep}{key}.bufr4"
with open(bufr_filename, "wb") as fh:
fh.write(item["bufr4"])
mode = "w" # Write to file if no headers

with open(filename, mode) as f:
# Check there was no problem writing the report to CSV
if csv_string is not None:
if not header_written:
# Write the whole string including headers
f.write(csv_string)
header_written = True
else:
# Skip the header row of the string
f.write(csv_string.split("\n")[1])

# Check there was no problem encoding the BUFR message
# before writing to a file
if item.get("bufr4") is not None:
# Write the BUFR file
key = item['_meta']["id"]
bufr_filename = f"{output_dir}{os.sep}{key}.bufr4"

with open(bufr_filename, "wb") as fh:
fh.write(item["bufr4"])

except Exception as e:
raise click.ClickException(e)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_synop2bufr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ def test_range_qc(metadata_string):
result = transform(out_of_range, metadata_string, 2000, 1)

for item in result:
warning_msgs = item["warnings"]
warning_msgs = item["_meta"]["result"]["warnings"]

assert "#1#nonCoordinatePressure: Value (47650.0) out of valid range (50000 - 108000).; Element set to missing" in warning_msgs # noqa
assert "#1#airTemperature: Value (334.15) out of valid range (193.15 - 333.15).; Element set to missing" in warning_msgs # noqa
Expand Down

0 comments on commit 3c00a1d

Please sign in to comment.