Merge pull request #18 from oislen/dev

14-polars-translation

.dockerignore

@@ -0,0 +1,6 @@
+data/Met_Eireann/arch
+scripts/aws_rootkey.py
+*__pycache__
+*.ipynb_checkpoints
+*.xlsx
+.creds

Dockerfile

@@ -1,5 +1,5 @@
 # get base image
-FROM ubuntu:20.04
+FROM ubuntu:latest
 
 # set environment variables
 ENV user=ubuntu
@@ -10,21 +10,20 @@ ENV GIT_BRANCH=${GIT_BRANCH}
 
 # install required software and programmes for development environment
 RUN apt-get update 
-RUN apt-get install -y apt-utils vim curl wget unzip git python3 python3-pip tree htop
+RUN apt-get install -y apt-utils vim curl wget unzip git tree htop
 
 # set up home environment
-RUN useradd ${user}
 RUN mkdir -p /home/${user} && chown -R ${user}: /home/${user}
 
 # clone git repo
 RUN git clone https://github.com/oislen/IrishClimateDashboard.git --branch ${GIT_BRANCH} /home/ubuntu/IrishClimateDashboard
+#COPY . /home/ubuntu/IrishClimateDashboard
 
 # install required python packages
-COPY requirements.txt /tmp/
 RUN apt-get install -y python3 python3-venv python3-pip
 RUN python3 -m venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
-RUN /opt/venv/bin/python3 -m pip install -r /tmp/requirements.txt
+RUN /opt/venv/bin/python3 -m pip install -v -r /home/ubuntu/IrishClimateDashboard/requirements.txt
 
 WORKDIR /home/${user}/IrishClimateDashboard
 CMD ["bokeh", "serve","scripts/bokeh_dash_app.py"]

exeDocker.bat

@@ -15,7 +15,7 @@ call docker build --no-cache -t %DOCKER_IMAGE% .  --build-arg GIT_BRANCH=%GIT_BR
 
 :: run docker container
 SET UBUNTU_DIR=/home/ubuntu
-call docker run --name %DOCKER_CONTAINER_NAME% --shm-size=512m -p 5006:5006 -it %DOCKER_IMAGE%
+call docker run --name %DOCKER_CONTAINER_NAME% --memory 6GB --shm-size=512m -p 5006:5006 --rm %DOCKER_IMAGE%
 
 :: useful docker commands
 :: docker images

scripts/utilities/gen_boto3_excel.py

@@ -1,18 +1,20 @@
 import io
 import boto3
-import pandas as pd
-from aws_rootkey import aws_rootkey_fpath
+import json
 from beartype import beartype
 
 @beartype
 def gen_boto3_excel(
+    sessionToken:str,
     bucket:str="irishclimateapp", 
     prefix:str="data/Met_Eireann"
     ) -> list:
     """Retrieves the raw Met Eireann data from AWS s3
 
     Parameters
     ----------
+    sessionToken : str
+        The file path to an active aws session token
     bucket : str
         The s3 bucket containing the Met Eireann data files
     prefix : str
@@ -23,14 +25,18 @@ def gen_boto3_excel(
     list
         The raw Met Eireann data
     """
-    # load aws root key
-    rootkey = pd.read_csv(aws_rootkey_fpath, sep="=", header=None, index_col=0)[1]
-    # generate boto3 s3 connection
-    client = boto3.client(
-        "s3",
-        aws_access_key_id=rootkey["AWSAccessKeyId"],
-        aws_secret_access_key=rootkey["AWSSecretKey"],
+    # load aws config
+    with open(sessionToken, "r") as j:
+        aws_config = json.loads(j.read())
+    # connect to aws boto3
+    session = boto3.Session(
+        aws_access_key_id=aws_config['Credentials']["AccessKeyId"],
+        aws_secret_access_key=aws_config['Credentials']["SecretAccessKey"],
+        aws_session_token=aws_config['Credentials']["SessionToken"],
+        region_name="eu-west-1"
     )
+    # generate boto3 s3 connection
+    client = session.client("s3")
     # create a paginator to list all objects
     paginator = client.get_paginator("list_objects_v2")
     # apply the paginator to list all files in the irishclimateapp bucket with key data/Met_Eireann