Merge pull request #108 from airavata-courses/project_3

Project 3 integration

Author: pranavacharya

.DS_Store

@@ -0,0 +1,33 @@
+name: Garuda_CD
+on:
+  push:
+    branches: [main, release]
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: executing remote ssh commands using key file
+      uses: appleboy/ssh-action@master
+      with:
+        # host for jetstream server
+        host: 149.165.154.67
+        # username for jetstream server
+        username: exouser
+        # jetstream userkey, added to server and github secrets
+        key: ${{ secrets.JETSTREAM_PVT_KEY }}
+        port: 22
+        script: |
+          cd $HOME
+          rm -rf garuda/
+          git clone https://github.com/airavata-courses/garuda.git
+          sh env_replacer.sh
+          echo "Building docker images"
+          cd garuda/
+          docker-compose build
+          cd kubernetes/
+          echo "Pushing created images into DockerHub"
+          sh docker_push.sh
+          echo "Deploy resources into kubernetes cluster"
+          sh kubernetes_init.sh

.github/workflows/garuda__github_actions_CD.yml

@@ -6,9 +6,9 @@ name: Garuda_CI
 # events but only for the development branch and master branch
 on:
   push:
-    branches: [data_extractor, queue_worker, main]
+    branches: [main, release]
   pull_request:
-    branches: [data_extractor, queue_worker, main]
+    branches: [main, release]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

.github/workflows/garuda__github_actions_CI.yml

@@ -14,6 +14,11 @@ Softwares/prerequisites needed to run garuda: [Docker](https://docs.docker.com/e
 
 #### Start Application
 
+Export ENV variables
+```sh
+export $PROJECT=PROJECT3
+```
+
 Pull the latest images from DockerHub
 
 ```sh
@@ -25,9 +30,14 @@ Start application services
 ```sh
 docker-compose up
 ```
-
 Run the above command on your terminal from the root of project folder to create all the resources to run the project.
 
+
+### Adding hostnames in /etc/hosts
+```sh
+sudo sh scipts/host.sh
+```
+
 > Note: The above command creates 6 containers for the running the application.
 
 > Note: The services run in non-detached mode. On exiting the process from terminal all the containers stop.
@@ -36,7 +46,7 @@ Run the above command on your terminal from the root of project folder to create
 
 #### Access Web-Application
 
-URL for the web-application: http://localhost:3000
+URL for the web-application: http://garuda.org:3000
 
 #### Stop Application
 
@@ -59,6 +69,12 @@ Build resource again if needed
 ```sh
 docker-compose build
 ```
+> Note: Before building make sure you have these env variables exported in terminal
+>  1. NASA_USERNAME - NASA MERRA2 dashboard username
+>  2. NASA_PASSWORD - NASA MERRA2 dashboard password
+>  3. AWS_ACCESS_KEY_ID - JetStream Object Store access key ID
+>  4. AWS_SECRET_ACCESS_KEY - JetStream Object Store access secret
+>  5. PROJECT - Version of project you want to build
 
 ## Run on Windows based systems
 
@@ -70,6 +86,11 @@ Softwares/prerequisites needed to run garuda: [Docker](https://docs.docker.com/d
 
 #### Start Application
 
+Export ENV variables
+```sh
+export $PROJECT=PROJECT3
+```
+
 Pull the latest images from DockerHub
 
 ```sh
@@ -82,25 +103,23 @@ Start application services
 docker-compose up
 ```
 
+Run the above command on your cmd from the root of project folder to create all the resources to run the project.
+
 ### Adding hostnames in /etc/hosts
 ```sh
 sudo sh scipts/host.sh
 ```
 
-Run the above command on your cmd from the root of project folder to create all the resources to run the project.
-
 > Note: The above command creates 6 containers for the running the application.
 
 > Note: The services run in non-detached mode. On exiting the process from terminal all the containers stop.
 
 > Note: This command might take some time to run. It's spinning up all the containers required to run the project. After all the resources are done loading, logs won't be printing on the terminal. You can use the application now !
 
 
-
-
 #### Access Web-Application
 
-URL for the web-application: http://localhost:3000
+URL for the web-application: http://garuda.org:3000
 
 #### Stop Application
 
@@ -131,6 +150,18 @@ docker compose build
 >  4. AWS_SECRET_ACCESS_KEY - JetStream Object Store access secret
 >  5. PROJECT - Version of project you want to build
 
+## Access JetStream production deployment
+
+Add changes in etc/hosts/ for production url
+```sh
+sudo sh scipts/prod_host.sh
+```
+
+Install CORS plugin in browser to enable cors headers since application is using jetstream object strore
+[sample cors plugin](https://chrome.google.com/webstore/detail/allow-cors-access-control/lhobafahddgcelffkeicbaginigeejlf?hl=en)
+
+Access application at http://garuda.org
+
 ## Modules
 
 1. [Data Extractor](./data_extractor/README.md) : Apache Maven project to build a utility JAR file which extracts requested NEXRAD data from S3.

README.md

@@ -0,0 +1,18 @@
+# Run the following lines to install ansible
+python3 -m pip install --user ansible
+
+# Create hosts file and paste the following lines in it
+[control_plane]
+control1 ansible_host=149.165.153.132 ansible_user=garuda 
+
+[workers]
+worker1 ansible_host=149.165.153.213 ansible_user=garuda
+worker2 ansible_host=149.165.155.35 ansible_user=garuda
+
+[all:vars]
+ansible_python_interpreter=/usr/bin/python3
+
+
+
+# move the hosts file to the ansible directory
+sudo mv hosts /etc/ansible/hosts

ansible/steps.txt

@@ -0,0 +1 @@
+__pycache__/*

apigateway/.gitignore

@@ -29,7 +29,7 @@ def getConstants():
         constants["RABBITMQ_HOST"] = os.environ.get('RABBITMQ_HOST')
 
     constants["RABBITMQ_PORT"] = '5672'
-    if os.environ.get('RABBITMQ_HOST') is not None:
+    if os.environ.get('RABBITMQ_PORT') is not None:
         constants["RABBITMQ_PORT"] = os.environ.get('RABBITMQ_PORT')
 
     return constants

apigateway/constants.py

@@ -25,16 +25,16 @@ def log_error(err_code):
     elif err_code == -3:
         print("Status value not valid")
 
-def push_to_rabbitmq(data):
+def push_to_rabbitmq(data, queue_name):
     creds = pika.PlainCredentials('guest', 'guest')
     connection = pika.BlockingConnection(
         pika.ConnectionParameters(host = constants['RABBITMQ_HOST'], port = constants['RABBITMQ_PORT'], virtual_host = "/", credentials = creds)
     )
     channel = connection.channel()
-    channel.queue_declare(queue='offload_request')
+    channel.queue_declare(queue=queue_name)
     message = json.dumps(data)
 
-    channel.basic_publish(exchange='', routing_key='offload_request', body=message)
+    channel.basic_publish(exchange='', routing_key=queue_name, body=message)
     connection.close()
     '''
     Queue Request example: 
@@ -121,23 +121,48 @@ def generate_new_request():
         }
         return jsonify(response)
     try:
-        station_key = request_data['station_name']
-        db_date_absolute = str(request_data['date'])     
-        db_date = db_date_absolute.replace(" ", "").replace("-", "")
-        time_start, time_end = str(request_data['time']).replace(" ","").replace(":", "").split('-')
-        property = request_data['property']
-        user_email = request_data['user_email']
+        # request type - nexrad/nasa
+        data_type = 'nexrad'
+        if 'type' in request_data:
+            data_type = request_data['type']            
+
+        # nexrad data
+        if data_type == 'nexrad':
+            station_key = request_data['station_name']
+            db_date_absolute = str(request_data['date'])     
+            db_date = db_date_absolute.replace(" ", "").replace("-", "")
+            time_start, time_end = str(request_data['time']).replace(" ","").replace(":", "").split('-')
+            property = request_data['property']
+            user_email = request_data['user_email']
+        # nasa data
+        else:
+            minlon = int(request_data["minlon"])
+            maxlon = int(request_data["maxlon"])
+            minlat = int(request_data["minlat"])
+            maxlat = int(request_data["maxlat"])
+            begTime = request_data["begTime"]
+            endTime = request_data["endTime"]
+            begHour = request_data["begHour"]
+            endHour = request_data["endHour"]
+            property = request_data["property"]
+            user_email = request_data['user_email']
     except:
         response = {
             "response_code" : "3",
             "response_message" : "Incorrect keys passed in the json request"
         }
         return jsonify(response)
-    db_key = f"{station_key}_{db_date}_{time_start}_{time_end}_{property}"
+
+    if data_type == 'nexrad':
+        db_key = f"{station_key}_{db_date}_{time_start}_{time_end}_{property}_{data_type}"
+    else:
+        # nasa data request ID
+        db_key = f"{property}_{minlon}_{maxlon}_{minlat}_{maxlat}_{begTime}_{endTime}_{begHour}_{endHour}_{data_type}"
 
     # Make call to check db for this key
     URL = "http://" + constants["DB_MIDDLEWARE_READER_HOST"] + ":" + constants["DB_MIDDLEWARE_READER_PORT"] + "/" + "postCheckRequest"
     METHOD = 'POST'
+    # TODO: remove property
     PAYLOAD = json.dumps({
         "request_id" : db_key,
         "property" : str(property)
@@ -182,28 +207,51 @@ def generate_new_request():
             response['data_dump'] = ""
         if db_response['data_status'] == "false":
             # Make a call to rabbitmq
-            '''
-            Queue Request example: 
-            {"requestID":"1234","stationID":"KABR","year":"2007","month":"01","date":"01","start_time":"000000","end_time":"003000","property":"Reflectivity"}
-            '''
-            rmq_month, rmq_date, rmq_year = db_date_absolute.split('-')
-            rabbitmq_data = {
-                "requestID" : str(db_key),
-                "stationID" : str(station_key),
-                "year" : rmq_year,
-                "month" : rmq_month,
-                "date" : rmq_date,
-                "start_time" : time_start,
-                "end_time" : time_end,
-                "property" : str(property)
-            }
+            rabbitmq_data  = {}
+            if data_type == 'nexrad':
+                '''
+                Queue Request example: 
+                {"requestID":"1234","stationID":"KABR","year":"2007","month":"01","date":"01","start_time":"000000","end_time":"003000","property":"Reflectivity"}
+                '''
+                rmq_month, rmq_date, rmq_year = db_date_absolute.split('-')
+                rabbitmq_data = {
+                    "requestID" : str(db_key),
+                    "stationID" : str(station_key),
+                    "year" : rmq_year,
+                    "month" : rmq_month,
+                    "date" : rmq_date,
+                    "start_time" : time_start,
+                    "end_time" : time_end,
+                    "property" : str(property)
+                }
+            else:
+                '''
+                Queue Request example: 
+                {"minlon":-180,"maxlon":180,"minlat":-90,"maxlat":-45,"begTime":"2021-01-01","endTime":"2021-01-02","begHour":"00:00","endHour":"00:00","requestID":"1234","property":"T"}
+                '''
+                rabbitmq_data = {
+                    "minlon":minlon,
+                    "maxlon":maxlon,
+                    "minlat":minlat,
+                    "maxlat":maxlat,
+                    "begTime":begTime,
+                    "endTime":endTime,
+                    "begHour":begHour,
+                    "endHour":endHour,
+                    "requestID":str(db_key),
+                    "property":str(property)
+                }
             try:
-                push_to_rabbitmq(data=rabbitmq_data)
+                queue_name = ''
+                if data_type == 'nexrad':
+                    queue_name = 'offload_request'
+                else:
+                    queue_name = 'offload_queue_nasa'
+                push_to_rabbitmq(data=rabbitmq_data, queue_name=queue_name)
             except:
                 response['response_code'] = "3"
                 response['response_message'] = "Failed to add new job to rabbitmq"
                 response['data_dump'] = ""
-
     else:
         response['response_code'] = "1"
         response['response_message'] = "Fail"

apigateway/server.py

@@ -1,6 +1,9 @@
 FROM node:14-alpine 
 
+
 WORKDIR /db_middleware
+
+
 COPY package.json .
 RUN npm install
 COPY . .