From bd743adb4f43d66e2caa36a7ab1d6f44465a3a6d Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 09:24:38 +0200 Subject: [PATCH 01/14] pygeoapi plugin added --- requirements.txt | 2 + synop2bufr/pygeoapi_plugin.py | 149 ++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 synop2bufr/pygeoapi_plugin.py diff --git a/requirements.txt b/requirements.txt index b743142..b2992b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ +attrs==22.2.0 click==8.1.3 +numpy==1.21.6 csv2bufr diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py new file mode 100644 index 0000000..6b44db4 --- /dev/null +++ b/synop2bufr/pygeoapi_plugin.py @@ -0,0 +1,149 @@ +############################################################################### +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +############################################################################### +import logging +from pygeoapi.process.base import BaseProcessor + +from synop2bufr import transform + +LOGGER = logging.getLogger(__name__) + +PROCESS_METADATA = { + "version": "0.1.0", + "id": "x-wmo:synop2bufr", + "title": {"en": "synop2bufr"}, + "description": {"en": "Process to convert FM 12-SYNOP bulletin to BUFR"}, + "keywords": ["SYNOP", "BUFR", "FM 12"], + "links": [{ + "type": "text/html", + "rel": "about", + "title": "homepage", + "href": "https://github.com/wmo-im/synop2bufr", + "hreflang": "en-US", + }], + "inputs": { + "data": { + "title": "FM 12-SYNOP bulletin string", + "description": "Input FM 12-SYNOP bulletin to convert to BUFR.", + "minOccurs": 1, + "maxOccurs": 1, + "metadata": None, + "keywords": [], + }, + "metadata":{ + "title": "Station metadata", + "description": "CSV formatted data containing list of stations required by synop2bufr.", # noqa + "schema": {"type": "string"}, + "minOccurs": 1, + "maxOccurs": 1, + "metadata": None, + "keywords": [], + }, + "year": { + "title": "Year", + "description": "Year (UTC) corresponding to FM 12-SYNOP bulletin", + "schema": {"type": "integer"}, + "minOccurs": 1, + "maxOccurs": 1, + "metadata": None, + "keywords": [] + }, + "month":{ + "title": "Month", + "description": "Month (UTC) corresponding to FM 12-SYNOP bulletin", + "schema": {"type": "integer"}, + "minOccurs": 1, + "maxOccurs": 1, + "metadata": None, + "keywords": [] + } + }, + "outputs": { + "messages": { + "title": "BUFR encoded data in base64", # noqa + "schema": {"type": "array"}, + "description": "One result per line in input data" # noqa + } + }, + "example": { + "inputs": { + "data":r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa + "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", + "year": 2022, + "month": 02 + }, + "output": {"messages": []} + }, +} + + +class processor(BaseProcessor): + def __init__(self, processor_def): + """ + Initialize object + :param processor_def: provider definition + :returns: pygeoapi.process.csv2bufr.csv2bufr + """ + + super().__init__(processor_def, PROCESS_METADATA) + + def execute(self, data): + """ + This method is invoked by pygeoapi when this class is set as a + `process` type resource in pygeoapi config file. + + :param data: It is the value of `inputs` passed in payload. e.g. + { + "inputs": { + "data": "csv data to encode", + "mappings": "csv2bufr mapping json file" + } + } + + :return: media_type, json + """ + + mimetype = "application/json" + try: + fm12 = data['data'] + metadata = data['metadata'] + year = data['year'] + month = data['month'] + bufr_generator = transform(data = fm12, + metadata = metadata, + year = year, + month = month) + # transform returns a generator, we need to iterate over + # and add to single output object + bufr = [] + for result in bufr_generator: + bufr.append(result) + + output = {"messages": bufr} + + LOGGER.error(output) + except Exception as e: + LOGGER.exception(e) + output = {"messages": None, "errors": e} + + return mimetype, output + + def __repr__(self): + return " {}".format(self.name) From 348989d78ca24f6d61a688e5d57a7fee8e8548f8 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 09:29:13 +0200 Subject: [PATCH 02/14] pygeoapi plugin added --- synop2bufr/pygeoapi_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 6b44db4..2e18ea0 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -87,7 +87,7 @@ "data":r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", "year": 2022, - "month": 02 + "month": 2 }, "output": {"messages": []} }, From 46e6038cba9bf355d1207f8d4676041a6d4b7eac Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 09:31:32 +0200 Subject: [PATCH 03/14] pygeoapi plugin added --- synop2bufr/pygeoapi_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 2e18ea0..28fcfdb 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -42,6 +42,7 @@ "data": { "title": "FM 12-SYNOP bulletin string", "description": "Input FM 12-SYNOP bulletin to convert to BUFR.", + "schema": {"type": "string"}, "minOccurs": 1, "maxOccurs": 1, "metadata": None, From 6d99acf6050091f84db1524b0450d47d11317d93 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 09:38:56 +0200 Subject: [PATCH 04/14] debugging, no messages produced in test data --- synop2bufr/pygeoapi_plugin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 28fcfdb..091ca9a 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -134,11 +134,15 @@ def execute(self, data): # transform returns a generator, we need to iterate over # and add to single output object bufr = [] + count = 0 for result in bufr_generator: bufr.append(result) - + count += 1 + if count != 1: + LOGGER.error(fm12) + LOGGER.error(metadata) output = {"messages": bufr} - + LOGGER.error(f"Number of messages processed = {count}") LOGGER.error(output) except Exception as e: LOGGER.exception(e) From b3e82c71b5adc2c7dec17eafad5fe6e1bbe68e79 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 10:01:34 +0200 Subject: [PATCH 05/14] debugging, no messages produced in test data --- synop2bufr/pygeoapi_plugin.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 091ca9a..71075b8 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -86,7 +86,7 @@ "example": { "inputs": { "data":r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa - "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", + "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", "year": 2022, "month": 2 }, @@ -122,6 +122,7 @@ def execute(self, data): """ mimetype = "application/json" + errors = [] try: fm12 = data['data'] metadata = data['metadata'] @@ -134,19 +135,13 @@ def execute(self, data): # transform returns a generator, we need to iterate over # and add to single output object bufr = [] - count = 0 for result in bufr_generator: bufr.append(result) - count += 1 - if count != 1: - LOGGER.error(fm12) - LOGGER.error(metadata) - output = {"messages": bufr} - LOGGER.error(f"Number of messages processed = {count}") - LOGGER.error(output) + output = {"messages": bufr, "errors": errors} except Exception as e: - LOGGER.exception(e) - output = {"messages": None, "errors": e} + LOGGER.error(e) + errors.append(f"{e}") + output = {"messages": None, "errors": errors} return mimetype, output From 435f8ca6ccaa246332d0c0b9c38469c974ebfd0b Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 10:05:04 +0200 Subject: [PATCH 06/14] bugfix: wsi referenced before assignmnet, now set to none when TSI not found. --- synop2bufr/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synop2bufr/__init__.py b/synop2bufr/__init__.py index 713ed12..4b7d79e 100644 --- a/synop2bufr/__init__.py +++ b/synop2bufr/__init__.py @@ -1202,6 +1202,7 @@ def transform(data: str, metadata: str, year: int, except Exception: conversion_success[tsi] = False LOGGER.warning(f"Station {tsi} not found in station file") + wsi = None # parse WSI to get sections try: From ddc14d17fb52bfea926c9e4c904d6e102fafe972 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 10:21:09 +0200 Subject: [PATCH 07/14] Issue with returning binary, now b64 returned. --- synop2bufr/pygeoapi_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 71075b8..b60104d 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -136,6 +136,7 @@ def execute(self, data): # and add to single output object bufr = [] for result in bufr_generator: + result['bufr4'] = base64.b64encode( result['bufr4'] ).decode("utf-8") # noqa bufr.append(result) output = {"messages": bufr, "errors": errors} except Exception as e: From 33c38a19f52dc96ffff644dc43832d821611ca33 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 10:22:22 +0200 Subject: [PATCH 08/14] Import bas64 --- synop2bufr/pygeoapi_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index b60104d..384669f 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -18,6 +18,7 @@ # under the License. # ############################################################################### +import base64 import logging from pygeoapi.process.base import BaseProcessor From a61d6a767220fb893499d5c3dbc92f65194aa210 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 11:19:42 +0200 Subject: [PATCH 09/14] debugging result. --- synop2bufr/pygeoapi_plugin.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 384669f..0f97565 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -111,19 +111,12 @@ def execute(self, data): This method is invoked by pygeoapi when this class is set as a `process` type resource in pygeoapi config file. - :param data: It is the value of `inputs` passed in payload. e.g. - { - "inputs": { - "data": "csv data to encode", - "mappings": "csv2bufr mapping json file" - } - } - :return: media_type, json """ mimetype = "application/json" errors = [] + bufr = [] try: fm12 = data['data'] metadata = data['metadata'] @@ -135,7 +128,7 @@ def execute(self, data): month = month) # transform returns a generator, we need to iterate over # and add to single output object - bufr = [] + for result in bufr_generator: result['bufr4'] = base64.b64encode( result['bufr4'] ).decode("utf-8") # noqa bufr.append(result) @@ -143,8 +136,9 @@ def execute(self, data): except Exception as e: LOGGER.error(e) errors.append(f"{e}") - output = {"messages": None, "errors": errors} + output = {"messages": bufr, "errors": errors} + LOGGER.error("returning") return mimetype, output def __repr__(self): From 5c21dec7717ba4038d6f5bf81f2ac95abac5a2cb Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 11:54:47 +0200 Subject: [PATCH 10/14] debugging result. --- synop2bufr/pygeoapi_plugin.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 0f97565..0064ac5 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -78,20 +78,23 @@ } }, "outputs": { - "messages": { + "result": { "title": "BUFR encoded data in base64", # noqa - "schema": {"type": "array"}, - "description": "One result per line in input data" # noqa + "schema": {"contentMediaType": "application/json"} + }, + "errors": { + "title": "Errors", + "schema": {"contentMediaType": "application/json"} } }, "example": { "inputs": { "data":r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa - "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", + "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", #noqa "year": 2022, "month": 2 }, - "output": {"messages": []} + "output": {} }, } @@ -132,11 +135,13 @@ def execute(self, data): for result in bufr_generator: result['bufr4'] = base64.b64encode( result['bufr4'] ).decode("utf-8") # noqa bufr.append(result) - output = {"messages": bufr, "errors": errors} + except Exception as e: LOGGER.error(e) errors.append(f"{e}") - output = {"messages": bufr, "errors": errors} + + output = {"result": json.dumps(result), "errors": json.dumps(errors)} + LOGGER.error("returning") return mimetype, output From 6a2a0ca3889340d46fac2f02c9fe652484622a01 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 11:55:30 +0200 Subject: [PATCH 11/14] debugging result. --- synop2bufr/pygeoapi_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 0064ac5..bc05a83 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -140,7 +140,7 @@ def execute(self, data): LOGGER.error(e) errors.append(f"{e}") - output = {"result": json.dumps(result), "errors": json.dumps(errors)} + output = {"result": json.dumps(bufr), "errors": json.dumps(errors)} LOGGER.error("returning") From 9595bfae52f17f95449db20249382bf315f56c86 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 12:10:15 +0200 Subject: [PATCH 12/14] debugging result. --- synop2bufr/pygeoapi_plugin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index bc05a83..73170eb 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -19,6 +19,7 @@ # ############################################################################### import base64 +import json import logging from pygeoapi.process.base import BaseProcessor @@ -89,7 +90,7 @@ }, "example": { "inputs": { - "data":r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa + "data": r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", #noqa "year": 2022, "month": 2 @@ -133,7 +134,11 @@ def execute(self, data): # and add to single output object for result in bufr_generator: + # need to convert BUFR binary to base64 result['bufr4'] = base64.b64encode( result['bufr4'] ).decode("utf-8") # noqa + # convert datetime to string + result['_meta']['properties']['datetime'] = \ + result['_meta']['properties']['datetime'].isoformat() bufr.append(result) except Exception as e: @@ -142,8 +147,6 @@ def execute(self, data): output = {"result": json.dumps(bufr), "errors": json.dumps(errors)} - - LOGGER.error("returning") return mimetype, output def __repr__(self): From 7cbadd81e26a0915d3bd3755341dc607570a2da5 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 12:11:41 +0200 Subject: [PATCH 13/14] flake8 --- synop2bufr/pygeoapi_plugin.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 73170eb..9b204ec 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -126,16 +126,16 @@ def execute(self, data): metadata = data['metadata'] year = data['year'] month = data['month'] - bufr_generator = transform(data = fm12, - metadata = metadata, - year = year, - month = month) + bufr_generator = transform(data=fm12, + metadata=metadata, + year=year, + month=month) + # transform returns a generator, we need to iterate over # and add to single output object - for result in bufr_generator: # need to convert BUFR binary to base64 - result['bufr4'] = base64.b64encode( result['bufr4'] ).decode("utf-8") # noqa + result['bufr4'] = base64.b64encode(result['bufr4']).decode("utf-8") # noqa # convert datetime to string result['_meta']['properties']['datetime'] = \ result['_meta']['properties']['datetime'].isoformat() From 11f1de8120f4187f4019e5370ea1f5f948bb4019 Mon Sep 17 00:00:00 2001 From: david-i-berry Date: Fri, 7 Jul 2023 12:15:06 +0200 Subject: [PATCH 14/14] flake8 --- synop2bufr/pygeoapi_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synop2bufr/pygeoapi_plugin.py b/synop2bufr/pygeoapi_plugin.py index 9b204ec..385e493 100644 --- a/synop2bufr/pygeoapi_plugin.py +++ b/synop2bufr/pygeoapi_plugin.py @@ -50,7 +50,7 @@ "metadata": None, "keywords": [], }, - "metadata":{ + "metadata": { "title": "Station metadata", "description": "CSV formatted data containing list of stations required by synop2bufr.", # noqa "schema": {"type": "string"}, @@ -68,7 +68,7 @@ "metadata": None, "keywords": [] }, - "month":{ + "month": { "title": "Month", "description": "Month (UTC) corresponding to FM 12-SYNOP bulletin", "schema": {"type": "integer"}, @@ -91,7 +91,7 @@ "example": { "inputs": { "data": r"AAXX 21121 15015 02999 02501 10103 21090 39765 42952 57020 60001 333 4/000 55310 0//// 22591 3//// 60007 91003 91104=", # noqa - "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", #noqa + "metadata": r"station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,territory_name,wmo_region\\nOCNA SUGATAG,0-20000-0-15015,15015,Land (fixed),47.77706163,23.94046026,503,Romania,6", # noqa "year": 2022, "month": 2 },