synop2bufr on aws-lambda example

Dockerfile

@@ -11,7 +11,7 @@ RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";"
     && apt-get update -y \
     && apt-get install -y ${DEBIAN_PACKAGES} \
     && apt-get install -y python3 python3-pip libeccodes-tools \
-    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.1.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \
     && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip
 
 ENV LOG_LEVEL=INFO

aws-lambda/Dockerfile

@@ -0,0 +1,43 @@
+# Define custom function directory
+ARG FUNCTION_DIR="/function"
+
+FROM wmoim/dim_eccodes_baseimage:2.31.0
+
+# Include global arg in this stage of the build
+ARG FUNCTION_DIR
+
+# Install awslambdaric
+RUN pip install \
+    --target ${FUNCTION_DIR} \
+        awslambdaric
+
+ENV TZ="Etc/UTC" \
+    DEBIAN_FRONTEND="noninteractive" \
+    DEBIAN_PACKAGES="gnupg2 cron bash vim git libffi-dev libeccodes0 python3-eccodes python3-cryptography libssl-dev libudunits2-0 python3-paho-mqtt python3-dateparser python3-tz python3-setuptools" \
+    ECCODES_DIR=/opt/eccodes \
+    PATH="$PATH;/opt/eccodes/bin"
+
+RUN echo "Acquire::Check-Valid-Until \"false\";\nAcquire::Check-Date \"false\";" | cat > /etc/apt/apt.conf.d/10no--check-valid-until \
+    && apt-get update -y \
+    && apt-get install -y ${DEBIAN_PACKAGES} \
+    && apt-get install -y python3 python3-pip libeccodes-tools \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/csv2bufr/archive/refs/tags/v0.7.4.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/pymetdecoder/archive/refs/tags/v0.1.10.zip \
+    && pip3 install --no-cache-dir https://github.com/wmo-im/synop2bufr/archive/refs/tags/v0.6.2.zip 
+
+COPY requirements.txt .
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Copy function code
+RUN mkdir -p ${FUNCTION_DIR}
+COPY . ${FUNCTION_DIR}
+
+ENV LOG_LEVEL=INFO
+
+# Set working directory to function root directory
+WORKDIR ${FUNCTION_DIR}
+
+# Set runtime interface client as default command for the container runtime
+ENTRYPOINT [ "/usr/bin/python3", "-m", "awslambdaric" ]
+# Pass the name of the function handler as an argument to the runtime
+CMD [ "lambda_function.handler" ]

aws-lambda/README.md

@@ -0,0 +1,30 @@
+# Using synop2bufr on AWS Lambda 
+
+## Overview
+
+AWS Lambda is a service from Amazon that enables publishing
+code which is executed as on demand functions. 
+
+This directory contains the Dockerfile and example lambda function code that will run the synop2bufr-transformation on files received in S3.
+
+## lambda container
+
+To run synop2bufr on Lambda requires a custom container image.  The Dockerfile in this directory.
+
+# to build docker container
+```bash
+docker build -t synop2bufr-lambda .
+```
+
+Once built, you need to deploy to ECR. 
+Depending on environment permissions, you may need to create a ECR repo with appropriate policies first.
+
+```bash
+aws ecr get-login-password --region <region> | docker login --username AWS --password-stdin <aws-account-id>.dkr.ecr.us-east-1.amazonaws.com
+docker tag synop2bufr-lambda:latest <ECR repo url>:latest
+docker push <ECR repo url>:latest
+```
+
+In the AWS console you can then create an AWS Lambda function using the URI for this container image. Setup your lambda function to be triggered by the S3 bucket where your synop files are stored.
+
+The example lambda-function will run the synop2bufr-transformation on the file stored in S3 and write the output to the 'wis2box-public' bucket.

aws-lambda/lambda_function.py

@@ -0,0 +1,47 @@
+import urllib.parse
+import boto3
+
+from datetime import datetime
+
+from synop2bufr import transform as transform_synop
+
+print('Loading function')
+s3 = boto3.client('s3')
+
+def handler(event, context):
+
+    # Get the object from the event 
+    bucket = event['Records'][0]['s3']['bucket']['name']
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')
+    size = event['Records'][0]['s3']['object']['size']
+    print("object="+key+" received with size="+str(size))
+    if size == 0:
+        print("object="+key+" size=0, don't process !")
+        return 0
+
+    filename = key.split('/')[-1]
+    foldername = key.replace(filename, '')
+
+    response = s3.get_object(Bucket=bucket, Key=key)
+    body = response["Body"].read().decode("utf-8")
+
+    # TODO: extract year and month from the file name
+    year_utc = datetime.utcnow().year
+    month_utc = datetime.utcnow().month
+
+    # TODO: read the metadata file from S3
+    metadata_file = open('/function/station_list.csv', 'r')
+
+    nbufr_created = 0
+    bufr_generator = transform_synop(body, metadata_file.read(), year_utc, month_utc)
+    for item in bufr_generator:
+        if 'bufr4' in item and item['bufr4'] is not None:
+            identifier = item['_meta']['id']
+            print('identifier='+identifier)
+            s3.put_object(Bucket='wis2box-public', Key=foldername+identifier+'.bufr4', Body=item['bufr4'])
+            nbufr_created += 1
+        else:
+            print('No BUFR message created for '+item['_meta']['id'])
+    print('Created '+str(nbufr_created)+' BUFR messages')
+
+    return 0

aws-lambda/requirements.txt

@@ -0,0 +1 @@
+boto3

aws-lambda/station_list.csv

@@ -0,0 +1,2 @@
+station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
+SINGAPORE/CHANGI AIRPORT,0-20000-0-48698,48698,Land (fixed),1.3679,103.9824,14.0,15.1,Singapore,V