8 ec2 hosting

aws/cons.py

@@ -0,0 +1,23 @@
+import platform
+import os
+import sys
+import json
+
+root_dir = 'E:\\GitHub\\IrishClimateDashboard' if platform.system() == 'Windows' else '/home/ubuntu/IrishClimateDashboard'
+sys.path.append(root_dir)
+# set directories
+data_dir = os.path.join(root_dir, 'data')
+creds_data = os.path.join(root_dir, '.creds')
+ec2_ref_data_dir = os.path.join(data_dir, "ec2", "ref")
+session_token_fpath = os.path.join(creds_data, "sessionToken.json")
+launch_template_config_fpath = os.path.join(ec2_ref_data_dir, "launch_template_config.json")
+create_fleet_config_fpath = os.path.join(ec2_ref_data_dir, "create_fleet_config.json")
+run_instances_config_fpath = os.path.join(ec2_ref_data_dir, "run_instances_config.json")
+
+# load aws ec2 references
+with open(launch_template_config_fpath) as json_file: 
+    launch_template_config = json.load(json_file)
+with open(create_fleet_config_fpath) as json_file: 
+    create_fleet_config = json.load(json_file)
+with open(run_instances_config_fpath) as json_file: 
+    run_instances_config = json.load(json_file)

aws/ec2_connect.bat

@@ -12,8 +12,7 @@ FOR /F "tokens=* USEBACKQ" %%F IN (`type %EC2_DNS_FPATH%`) DO (
     SET EC2_DNS!=%%F
 )
 :: scp docker file and linuc setup shell script to EC2 home
-::call scp -i %EC2_PEM_FPATH% -r %EC2_CREDS_FDIR% %EC2_USER%@%EC2_DNS%:~/.
-call scp -i %EC2_PEM_FPATH% %EC2_SETUP_FPATH% %EC2_USER%@%EC2_DNS%:~/linux_docker_setup.sh
+:: call scp -i %EC2_PEM_FPATH% %EC2_SETUP_FPATH% %EC2_USER%@%EC2_DNS%:~/linux_docker_setup.sh
 :: ssh to EC2
 call ssh -i %EC2_PEM_FPATH% %EC2_USER%@%EC2_DNS%
 ENDLOCAL

aws/linux_docker_setup.sh

@@ -1,68 +1,19 @@
-# NOTE:
-# 1. make sure to add ip address to security groups inbound rules
-# 2. make sure to increase volume in /dev/nvme0n1 (/dev/xvda) e.g. 100gb
-
-# linux file formatting
-# sudo yum install -y dos2unix 
-# dos2unix ~/linux_docker_setup.sh 
-# bash ~/linux_docker_setup.sh 
-
-#-- EC2 Spot Instance Checks --#
-
-# check available memory and cpu capacity
-free -h
-df -h
-lscpu
-# calculate percentage of used memory
-free -m | awk 'FNR == 2 {print $3/($3+$4)*100}'
-
-#-- Configure Permissions and Overcommit Settings --#
-
 # reset premission for the /opt /dev /run and /sys directories
 ls -larth /.
 sudo chmod -R 777 /opt /dev /run /sys/fs/cgroup
 sudo chmod 775 /var/run/screen
 ls -larth /.
-# mask permissoin for home docker file
-#sudo chmod 700 ~/.creds
-#sudo chmod 600 ~/.creds/*
 # update overcommit memory setting
 cat /proc/sys/vm/overcommit_memory
 echo 1 | sudo tee /proc/sys/vm/overcommit_memory
 
-#-- Increase EBS Volume --#
-
-# verify that the root partition mounted under "/" is full (100%)
-df -h
-# gather details about your attached block devices
-lsblk
-lsblk -f
-# mount the temporary file system tmpfs to the /tmp mount point
-sudo mount -o size=10M,rw,nodev,nosuid -t tmpfs tmpfs /tmp
-# Run the growpart command to grow the size of the root partition or partition 1
-sudo growpart /dev/nvme0n1 1 
-# Run the lsblk command to verify that partition 1 is expanded
-lsblk
-# Expand the file system
-sudo xfs_growfs -d /
-# file system on partition 1 is expanded
-sudo resize2fs /dev/nvme0n1p1
-# use the df -h command to verify that the OS can see the additional space
-df -h
-# Run the unmount command to unmount the tmpfs file system.
-sudo umount /tmp
-
-#-- Download Required Programmes --#
-
 # update os
 sudo yum update -y
 # install required base software
 sudo yum install -y htop vim tmux dos2unix docker git
 # remove unneed dependencies
 sudo yum autoremove
 
-#-- Pull and Run Git Repo --#
-
 # pull git repo
 sudo mkdir /home/ubuntu
 sudo git clone https://github.com/oislen/IrishClimateDashboard.git --branch v0.0.0 /home/ubuntu/IrishClimateDashboard
@@ -73,19 +24,3 @@ python3 -m pip install -v -r /home/ubuntu/IrishClimateDashboard/requirements.txt
 # run bokeh app
 bokeh serve /home/ubuntu/IrishClimateDashboard/dashboard/bokeh_dash_app.py --allow-websocket-origin=*.*.*.*:5006
 # http://34.243.42.137:5006/bokeh_dash_app
-
-#-- Pull and Run Docker Contianer --#
-
-# login to docker
-sudo gpasswd -a $USER 
-sudo systemctl start docker
-sudo chmod 666 /var/run/docker.sock
-cat ~/.creds/docker | docker login --username oislen --password-stdin
-# set docker constants
-export DOCKER_IMAGE=oislen/irishclimatedashboard:latest
-export DOCKER_CONTAINER_NAME=icd
-# pull docker container
-docker pull $DOCKER_IMAGE
-docker logout
-# run pulled docker container
-docker run -it oislen/irishclimatedashboard:latest

aws/utilities/EC2Client.py

@@ -0,0 +1,101 @@
+import io
+import boto3
+import json
+import logging
+import pandas as pd
+from typing import Union
+from beartype import beartype
+import cons
+
+class EC2Client():
+
+    @beartype
+    def __init__(self, sessionToken:str):
+        # load aws config
+        with open(sessionToken, "r") as j:
+            aws_config = json.loads(j.read())
+        # connect to aws boto3
+        self.session = boto3.Session(
+            aws_access_key_id=aws_config["Credentials"]["AccessKeyId"],
+            aws_secret_access_key=aws_config["Credentials"]["SecretAccessKey"],
+            aws_session_token=aws_config["Credentials"]["SessionToken"],
+            region_name="eu-west-1"
+        )
+        # generate boto3 s3 connection
+        self.client = self.session.client("ec2")
+
+    def create_launch_template(self, launch_template_config):
+        """
+        """
+        # create ec2 launch template
+        response = self.client.create_launch_template(**launch_template_config)
+        return response
+
+    def delete_launch_template(self, launch_template_config):
+        """
+        """
+        # delete ec2 launch template
+        response = self.client.delete_launch_template(DryRun=launch_template_config["DryRun"], LaunchTemplateName=launch_template_config["LaunchTemplateName"])
+        return response
+
+    def create_fleet(self, create_fleet_config):
+        """
+        """
+        response = self.client.create_fleet(**create_fleet_config)
+        return response
+
+    def describe_fleets(self):
+        """
+        """
+        response = self.client.describe_fleets()
+        return response
+
+    def delete_fleets(self, FleetIds=[], TerminateInstances=False):
+        """
+        """
+        response = self.client.delete_fleets(FleetIds=FleetIds, TerminateInstances=TerminateInstances)
+        return response
+
+    def run_instances(self, run_instances_config):
+        """
+        """
+        response = self.client.run_instances(**run_instances_config)
+        return response
+
+    def stop_instances(self, InstanceIds=[]):
+        """
+        """
+        response = self.client.stop_instances(InstanceIds=InstanceIds)
+        return response
+
+    def terminate_instances(self, InstanceIds=[]):
+        """
+        """
+        response = self.client.terminate_instances(InstanceIds=InstanceIds)
+        return response
+
+    def describe_instances(self, InstanceIds=[], Filters=[], MaxResults=20):
+        """
+        """
+        response = self.client.describe_instances(InstanceIds=InstanceIds, Filters=Filters, MaxResults=MaxResults)
+        return response
+
+
+ec2_client = EC2Client(sessionToken=cons.session_token_fpath)
+ec2_client.delete_launch_template(cons.launch_template_config)
+ec2_client.create_launch_template(cons.launch_template_config)
+#ec2_client.create_fleet(cons.create_fleet_config)
+ec2_client.run_instances(cons.run_instances_config)
+InstanceIds=["i-0dfe7150a875c4d45"]
+ec2_client.stop_instances(InstanceIds=InstanceIds)
+ec2_client.terminate_instances(InstanceIds=InstanceIds)
+ec2_client.describe_instances()
+
+# Issues with Fleets
+#ec2_client.client.describe_capacity_reservation_fleets()
+#ec2_client.client.cancel_capacity_reservation_fleets(CapacityReservationFleetIds=['9f3e14aa-26f2-4e36-b635-938d752a2bc5'])
+ec2_client.client.describe_fleets()
+#ec2_client.client.describe_fleet_instances()
+#ec2_client.client.delete_fleets(FleetIds=["fleet-4605729e-1523-4f9e-add0-7d6229915c25"], TerminateInstances=True)
+#ec2_client.client.delete_fleets(FleetIds=["fleet-b9d79eb1-dce2-4649-b321-f2158039454a"], TerminateInstances=True)
+#ec2_client.client.delete_fleets(FleetIds=["fleet-24040711-8eb0-42dd-b80f-5e6b54b200b0"], TerminateInstances=True)

data/ec2/ref/create_fleet_config.json

@@ -0,0 +1,25 @@
+{
+    "DryRun": false,
+    "TargetCapacitySpecification": {
+        "TotalTargetCapacity": 1,
+        "OnDemandTargetCapacity": 1,
+        "SpotTargetCapacity": 0,
+        "DefaultTargetCapacityType": "on-demand"
+    },
+    "LaunchTemplateConfigs": [
+        {
+            "LaunchTemplateSpecification": {
+                "LaunchTemplateName": "irishclimatedashboard",
+                "Version": "$Latest"
+            },
+            "Overrides": [
+                {
+                    "InstanceType": "t2.micro"
+                }
+            ]
+        }
+    ],
+    "SpotOptions": {
+        "AllocationStrategy": "diversified"
+    }
+}

data/ec2/ref/launch_template_config.json

@@ -0,0 +1,30 @@
+{
+    "DryRun": false,
+    "LaunchTemplateName": "irishclimatedashboard",
+    "VersionDescription": "Initial version",
+    "LaunchTemplateData": {
+        "BlockDeviceMappings": [
+            {
+                "DeviceName": "/dev/xvda",
+                "Ebs": {
+                    "VolumeSize": 20,
+                    "VolumeType": "gp3"
+                }
+            }
+        ],
+        "NetworkInterfaces":[
+            {
+                "DeviceIndex":0,
+                "AssociatePublicIpAddress": true,
+                "SubnetId":"subnet-0b4107fd011ea3b8a",
+                "Groups":["sg-03864b806cd78ded3"]
+            }
+        ],
+        "ImageId": "ami-00385a401487aefa4",
+        "InstanceType": "t2.micro",
+        "KeyName": "kaggle",
+        "Placement": {
+            "AvailabilityZone": "eu-west-1a"
+        }
+    }
+}

data/ec2/ref/run_instances_config.json

@@ -0,0 +1,9 @@
+{
+    "DryRun": false,
+    "LaunchTemplate": {
+        "LaunchTemplateName": "irishclimatedashboard",
+        "Version": "$Latest"
+    },
+    "MaxCount":1,
+    "MinCount":1
+}