Merge branch 'main' of https://github.com/wmo-im/synop2bufr into rory…

…-s4-fix

Dockerfile

@@ -11,7 +11,7 @@ RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";"
     && apt-get update -y \
     && apt-get install -y ${DEBIAN_PACKAGES} \
     && apt-get install -y python3 python3-pip libeccodes-tools \
-    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.1.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \
     && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip
 
 ENV LOG_LEVEL=INFO

README.md

@@ -13,13 +13,15 @@ The synop2bufr Python module contains both a command line interface and API to c
 
 Dependencies are listed in [requirements.txt](https://github.com/wmo-im/synop2bufr/blob/main/requirements.txt). Dependencies are automatically installed during synop2bufr installation.
 
+## Running
+
+To run synop2bufr from a Docker container:
+
 ```console
 docker build -t synop2bufr:local .
 docker run -it -v ${pwd}:/local synop2bufr
 ```
 
-## Running
-
 Example data can be found in `data` directory, with the corresponding reference BUFR4 in `data/bufr`.
 
 To transform SYNOP data file into BUFR:
@@ -29,6 +31,8 @@ mkdir output-data
 synop2bufr data transform --metadata data/station_list.csv --year 2023 --month 03 --output-dir output-data data/A_SMRO01YRBK211200_C_EDZW_20220321120500_12524785.txt
 ```
 
+To run synop2bufr inside a Lambda function on Amazon Web Services, please refer to [aws-lambda/README.md](aws-lambda/README.md) and use this [Dockerfile](aws-lambda/Dockerfile) to build the container image for the Lambda function.
+
 ## Usage Guide
 
 Here we detail how synop2bufr can be used.

aws-lambda/Dockerfile

@@ -0,0 +1,43 @@
+# Define custom function directory
+ARG FUNCTION_DIR="/function"
+
+FROM ghcr.io/wmo-im/dim_eccodes_baseimage:2.31.0
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+
+# Install awslambdaric
+RUN pip install \
+    --target ${FUNCTION_DIR} \
+        awslambdaric
+
+ENV TZ="Etc/UTC" \
+    DEBIAN_FRONTEND="noninteractive" \
+    DEBIAN_PACKAGES="gnupg2 cron bash vim git libffi-dev libeccodes0 python3-eccodes python3-cryptography libssl-dev libudunits2-0 python3-paho-mqtt python3-dateparser python3-tz python3-setuptools" \
+    ECCODES_DIR=/opt/eccodes \
+    PATH="$PATH;/opt/eccodes/bin"
+
+RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";" | cat > /etc/apt/apt.conf.d/10no--check-valid-until \
+    && apt-get update -y \
+    && apt-get install -y ${DEBIAN_PACKAGES} \
+    && apt-get install -y python3 python3-pip libeccodes-tools \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.6.2.zip 
+
+COPY requirements.txt .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Copy function code
+RUN mkdir -p ${FUNCTION_DIR}
+COPY . ${FUNCTION_DIR}
+
+ENV LOG_LEVEL=INFO
+
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+
+# Set runtime interface client as default command for the container runtime
+ENTRYPOINT [ "/usr/bin/python3", "-m", "awslambdaric" ]
+# Pass the name of the function handler as an argument to the runtime
+CMD [ "lambda_function.handler" ]

aws-lambda/README.md

@@ -0,0 +1,30 @@
+# Using synop2bufr on AWS Lambda 
+
+## Overview
+
+AWS Lambda is a service from Amazon that enables publishing code which is executed as on demand functions. 
+
+This directory contains a `Dockerfile` and example AWS Lambda function code that will run the synop2bufr transformation on files received in S3.
+
+## AWS Lambda container
+
+The Dockerfile in this directory will build the container image that can be used to run synop2bufr on AWS Lambda.
+
+# build and deploy
+```bash
+docker build -t synop2bufr-lambda .
+```
+
+Once built, you then need to deploy to ECR. 
+
+Depending on environment permissions, you may need to create a ECR repo with appropriate policies first.
+
+```bash
+aws ecr get-login-password --region <region> | docker login --username AWS --password-stdin <aws-account-id>.dkr.ecr.us-east-1.amazonaws.com
+docker tag synop2bufr-lambda:latest <ECR repo url>:latest
+docker push <ECR repo url>:latest
+```
+
+In the AWS console, you can then create an AWS Lambda function using the URI for this container image. Setup your AWS Lambda function to be triggered by the S3 bucket where your synop files are stored.
+
+The example AWS Lambda function will run the synop2bufr transformation on the file stored in S3 and write the output to the `wis2box-public` bucket.

aws-lambda/lambda_function.py

@@ -0,0 +1,57 @@
+import urllib.parse
+import boto3
+
+from datetime import datetime
+
+from synop2bufr import transform as transform_synop
+
+print('Loading function')
+s3 = boto3.client('s3')
+
+
+def handler(event, context):
+
+    # Get the object from the event
+    bucket = event['Records'][0]['s3']['bucket']['name']
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8') # noqa
+    size = event['Records'][0]['s3']['object']['size']
+    print("object="+key+" received with size="+str(size))
+    if size == 0:
+        print("object="+key+" size=0, don't process !")
+        return 0
+
+    filename = key.split('/')[-1]
+    foldername = key.replace(filename, '')
+
+    response = s3.get_object(Bucket=bucket, Key=key)
+    body = response["Body"].read().decode("utf-8")
+
+    # TODO: extract year and month from the file name
+    year_utc = datetime.utcnow().year
+    month_utc = datetime.utcnow().month
+
+    # TODO: read the metadata file from S3
+    metadata_file = open('/function/station_list.csv', 'r')
+
+    nbufr_created = 0
+    bufr_generator = transform_synop(
+        body,
+        metadata_file.read(),
+        year_utc,
+        month_utc
+    )
+    for item in bufr_generator:
+        if 'bufr4' in item and item['bufr4'] is not None:
+            identifier = item['_meta']['id']
+            print('identifier='+identifier)
+            s3.put_object(
+                Bucket='wis2box-public',
+                Key=foldername+identifier+'.bufr4',
+                Body=item['bufr4']
+            )
+            nbufr_created += 1
+        else:
+            print('No BUFR message created for '+item['_meta']['id'])
+    print('Created '+str(nbufr_created)+' BUFR messages')
+
+    return 0

aws-lambda/requirements.txt

@@ -0,0 +1 @@
+boto3

aws-lambda/station_list.csv

@@ -0,0 +1,2 @@
+station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
+SINGAPORE/CHANGI AIRPORT,0-20000-0-48698,48698,Land (fixed),1.3679,103.9824,14.0,15.1,Singapore,V