Merge branch 'main' of github.com:airavata-courses/garuda into release

.github/workflows/garuda__github_actions_CD.yml

@@ -0,0 +1,33 @@
+name: Garuda_CD
+on:
+  push:
+    branches: [main, release]
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: executing remote ssh commands using key file
+      uses: appleboy/ssh-action@master
+      with:
+        # host for jetstream server
+        host: 149.165.154.67
+        # username for jetstream server
+        username: exouser
+        # jetstream userkey, added to server and github secrets
+        key: ${{ secrets.JETSTREAM_PVT_KEY }}
+        port: 22
+        script: |
+          cd $HOME
+          rm -rf garuda/
+          git clone https://github.com/airavata-courses/garuda.git
+          sh env_replacer.sh
+          echo "Building docker images"
+          cd garuda/
+          docker-compose build
+          cd kubernetes/
+          echo "Pushing created images into DockerHub"
+          sh docker_push.sh
+          echo "Deploy resources into kubernetes cluster"
+          sh kubernetes_init.sh

.github/workflows/garuda__github_actions_CI.yml

@@ -6,9 +6,9 @@ name: Garuda_CI
 # events but only for the development branch and master branch
 on:
   push:
-    branches: [data_extractor, queue_worker, main]
+    branches: [main, release]
   pull_request:
-    branches: [data_extractor, queue_worker, main]
+    branches: [main, release]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:

README.md

@@ -14,6 +14,11 @@ Softwares/prerequisites needed to run garuda: [Docker](https://docs.docker.com/e
 
 #### Start Application
 
+Export ENV variables
+```sh
+export $PROJECT=PROJECT3
+```
+
 Pull the latest images from DockerHub
 
 ```sh
@@ -25,9 +30,14 @@ Start application services
 ```sh
 docker-compose up
 ```
-
 Run the above command on your terminal from the root of project folder to create all the resources to run the project.
 
+
+### Adding hostnames in /etc/hosts
+```sh
+sudo sh scipts/host.sh
+```
+
 > Note: The above command creates 6 containers for the running the application.
 
 > Note: The services run in non-detached mode. On exiting the process from terminal all the containers stop.
@@ -36,7 +46,7 @@ Run the above command on your terminal from the root of project folder to create
 
 #### Access Web-Application
 
-URL for the web-application: http://localhost:3000
+URL for the web-application: http://garuda.org:3000
 
 #### Stop Application
 
@@ -59,6 +69,12 @@ Build resource again if needed
 ```sh
 docker-compose build
 ```
+> Note: Before building make sure you have these env variables exported in terminal
+>  1. NASA_USERNAME - NASA MERRA2 dashboard username
+>  2. NASA_PASSWORD - NASA MERRA2 dashboard password
+>  3. AWS_ACCESS_KEY_ID - JetStream Object Store access key ID
+>  4. AWS_SECRET_ACCESS_KEY - JetStream Object Store access secret
+>  5. PROJECT - Version of project you want to build
 
 ## Run on Windows based systems
 
@@ -70,6 +86,11 @@ Softwares/prerequisites needed to run garuda: [Docker](https://docs.docker.com/d
 
 #### Start Application
 
+Export ENV variables
+```sh
+export $PROJECT=PROJECT3
+```
+
 Pull the latest images from DockerHub
 
 ```sh
@@ -84,15 +105,21 @@ docker-compose up
 
 Run the above command on your cmd from the root of project folder to create all the resources to run the project.
 
+### Adding hostnames in /etc/hosts
+```sh
+sudo sh scipts/host.sh
+```
+
 > Note: The above command creates 6 containers for the running the application.
 
 > Note: The services run in non-detached mode. On exiting the process from terminal all the containers stop.
 
 > Note: This command might take some time to run. It's spinning up all the containers required to run the project. After all the resources are done loading, logs won't be printing on the terminal. You can use the application now !
 
+
 #### Access Web-Application
 
-URL for the web-application: http://localhost:3000
+URL for the web-application: http://garuda.org:3000
 
 #### Stop Application
 
@@ -116,18 +143,46 @@ Build resource again if needed
 docker compose build
 ```
 
+> Note: Before building make sure you have these env variables exported in terminal
+>  1. NASA_USERNAME - NASA MERRA2 dashboard username
+>  2. NASA_PASSWORD - NASA MERRA2 dashboard password
+>  3. AWS_ACCESS_KEY_ID - JetStream Object Store access key ID
+>  4. AWS_SECRET_ACCESS_KEY - JetStream Object Store access secret
+>  5. PROJECT - Version of project you want to build
+
+## Access JetStream production deployment
+
+Add changes in etc/hosts/ for production url
+```sh
+sudo sh scipts/prod_host.sh
+```
+
+Install CORS plugin in browser to enable cors headers since application is using jetstream object strore
+[sample cors plugin](https://chrome.google.com/webstore/detail/allow-cors-access-control/lhobafahddgcelffkeicbaginigeejlf?hl=en)
+
+Access application at http://garuda.org
+
 ## Modules
 
 1. [Data Extractor](./data_extractor/README.md) : Apache Maven project to build a utility JAR file which extracts requested NEXRAD data from S3.
 
 2. [Queue Worker](./queue_worker/README.md) : Apache Maven project to build a JAR file which runs a consumer on a rabbitmq queue. It processes the request using data_extractor utitlity JAR and published the data to a API endpoint.
 
 3. [DB_Middleware](./db_middleware/README.md): Microservice to interact with database. This microservices provides APIs to perform read and writes to database. Reads are performed by API_Gateway module and Writes are performed by Queue_Worker module and API_Gateway module.
+It also dumps the dataset of the request to the object store (AWS S3 bucket) and saves the object url in the database
 
 4. [API_Gateway](./apigateway/README.md): API_Gateway module provides a middle-ware layer for all the back-end services. Front-end application communicate with API_Gateway module to interact with all other micro-services.
 
 5. [Web_App](./web_app/README.md): Web Application module is the application with which the end users interacts. It communicates with API_Gateway module to maintain user data and fetch NEXRAD data.
 
+6. [Queue Worker Nasa](./queue_worker_nasa/README.md) : Python application which runs a consumer on a rabbitmq queue. It processes the request using extractor utitlity and published the data in a conerted formatted to a API endpoint.
+
+## Optimization
+In the project 3 milestone after brainstorming we found scope for improvement in our system through which we reduced load from the backend significantly. The improvement was to store request dataset to the object-store(AWS S3 bucket) and then web app retrieves the data from the object store whenever user requests to plot the map. To check the systems performance with and without object store. We benchmarked the system with JMeter by making 100 concurrent request.
+The average response time in without object store was 14194ms and in with object store was 399ms.
+The average response time was reduced by <b> 135% <b>.
+The details of the reports are present [here](https://courses.airavata.org/garuda/Architecture_analysis_case_study/Optimization%20using%20object-store.pdf)
+
 ## Architecture
 
 ![Garuda Architecture Diagram](./docs/diagram/Architecture_diagram.jpg)
@@ -148,6 +203,10 @@ docker compose build
 
   3.  CD is triggered on each push to master branch.
 
+  4.  CD logs into JetStream2 remote server, builds all the docker images, pushes the docker images to DockerHub, replaces old deployments with latest deployments on remote kubernetes cluster.
+
+![Garuda CI-CD Workflow](./docs/diagram/garuda-ci-cd.jpg)
+
 ## Packages / Distribution builds
 
 [Garuda's Data Extractor Maven Package](https://github.com/airavata-courses/garuda/packages/1236747)

ansible/steps.txt

@@ -0,0 +1,18 @@
+# Run the following lines to install ansible
+python3 -m pip install --user ansible
+
+# Create hosts file and paste the following lines in it
+[control_plane]
+control1 ansible_host=149.165.153.132 ansible_user=garuda 
+
+[workers]
+worker1 ansible_host=149.165.153.213 ansible_user=garuda
+worker2 ansible_host=149.165.155.35 ansible_user=garuda
+
+[all:vars]
+ansible_python_interpreter=/usr/bin/python3
+
+
+
+# move the hosts file to the ansible directory
+sudo mv hosts /etc/ansible/hosts

apigateway/.gitignore

@@ -0,0 +1 @@
+__pycache__/*

apigateway/constants.py

@@ -29,7 +29,7 @@ def getConstants():
         constants["RABBITMQ_HOST"] = os.environ.get('RABBITMQ_HOST')
 
     constants["RABBITMQ_PORT"] = '5672'
-    if os.environ.get('RABBITMQ_HOST') is not None:
+    if os.environ.get('RABBITMQ_PORT') is not None:
         constants["RABBITMQ_PORT"] = os.environ.get('RABBITMQ_PORT')
 
     return constants

apigateway/server.py

@@ -25,16 +25,16 @@ def log_error(err_code):
     elif err_code == -3:
         print("Status value not valid")
 
-def push_to_rabbitmq(data):
+def push_to_rabbitmq(data, queue_name):
     creds = pika.PlainCredentials('guest', 'guest')
     connection = pika.BlockingConnection(
         pika.ConnectionParameters(host = constants['RABBITMQ_HOST'], port = constants['RABBITMQ_PORT'], virtual_host = "/", credentials = creds)
     )
     channel = connection.channel()
-    channel.queue_declare(queue='offload_request')
+    channel.queue_declare(queue=queue_name)
     message = json.dumps(data)
 
-    channel.basic_publish(exchange='', routing_key='offload_request', body=message)
+    channel.basic_publish(exchange='', routing_key=queue_name, body=message)
     connection.close()
     '''
     Queue Request example: 
@@ -121,23 +121,48 @@ def generate_new_request():
         }
         return jsonify(response)
     try:
-        station_key = request_data['station_name']
-        db_date_absolute = str(request_data['date'])     
-        db_date = db_date_absolute.replace(" ", "").replace("-", "")
-        time_start, time_end = str(request_data['time']).replace(" ","").replace(":", "").split('-')
-        property = request_data['property']
-        user_email = request_data['user_email']
+        # request type - nexrad/nasa
+        data_type = 'nexrad'
+        if 'type' in request_data:
+            data_type = request_data['type']            
+
+        # nexrad data
+        if data_type == 'nexrad':
+            station_key = request_data['station_name']
+            db_date_absolute = str(request_data['date'])     
+            db_date = db_date_absolute.replace(" ", "").replace("-", "")
+            time_start, time_end = str(request_data['time']).replace(" ","").replace(":", "").split('-')
+            property = request_data['property']
+            user_email = request_data['user_email']
+        # nasa data
+        else:
+            minlon = int(request_data["minlon"])
+            maxlon = int(request_data["maxlon"])
+            minlat = int(request_data["minlat"])
+            maxlat = int(request_data["maxlat"])
+            begTime = request_data["begTime"]
+            endTime = request_data["endTime"]
+            begHour = request_data["begHour"]
+            endHour = request_data["endHour"]
+            property = request_data["property"]
+            user_email = request_data['user_email']
     except:
         response = {
             "response_code" : "3",
             "response_message" : "Incorrect keys passed in the json request"
         }
         return jsonify(response)
-    db_key = f"{station_key}_{db_date}_{time_start}_{time_end}_{property}"
+
+    if data_type == 'nexrad':
+        db_key = f"{station_key}_{db_date}_{time_start}_{time_end}_{property}_{data_type}"
+    else:
+        # nasa data request ID
+        db_key = f"{property}_{minlon}_{maxlon}_{minlat}_{maxlat}_{begTime}_{endTime}_{begHour}_{endHour}_{data_type}"
 
     # Make call to check db for this key
     URL = "http://" + constants["DB_MIDDLEWARE_READER_HOST"] + ":" + constants["DB_MIDDLEWARE_READER_PORT"] + "/" + "postCheckRequest"
     METHOD = 'POST'
+    # TODO: remove property
     PAYLOAD = json.dumps({
         "request_id" : db_key,
         "property" : str(property)
@@ -182,28 +207,51 @@ def generate_new_request():
             response['data_dump'] = ""
         if db_response['data_status'] == "false":
             # Make a call to rabbitmq
-            '''
-            Queue Request example: 
-            {"requestID":"1234","stationID":"KABR","year":"2007","month":"01","date":"01","start_time":"000000","end_time":"003000","property":"Reflectivity"}
-            '''
-            rmq_month, rmq_date, rmq_year = db_date_absolute.split('-')
-            rabbitmq_data = {
-                "requestID" : str(db_key),
-                "stationID" : str(station_key),
-                "year" : rmq_year,
-                "month" : rmq_month,
-                "date" : rmq_date,
-                "start_time" : time_start,
-                "end_time" : time_end,
-                "property" : str(property)
-            }
+            rabbitmq_data  = {}
+            if data_type == 'nexrad':
+                '''
+                Queue Request example: 
+                {"requestID":"1234","stationID":"KABR","year":"2007","month":"01","date":"01","start_time":"000000","end_time":"003000","property":"Reflectivity"}
+                '''
+                rmq_month, rmq_date, rmq_year = db_date_absolute.split('-')
+                rabbitmq_data = {
+                    "requestID" : str(db_key),
+                    "stationID" : str(station_key),
+                    "year" : rmq_year,
+                    "month" : rmq_month,
+                    "date" : rmq_date,
+                    "start_time" : time_start,
+                    "end_time" : time_end,
+                    "property" : str(property)
+                }
+            else:
+                '''
+                Queue Request example: 
+                {"minlon":-180,"maxlon":180,"minlat":-90,"maxlat":-45,"begTime":"2021-01-01","endTime":"2021-01-02","begHour":"00:00","endHour":"00:00","requestID":"1234","property":"T"}
+                '''
+                rabbitmq_data = {
+                    "minlon":minlon,
+                    "maxlon":maxlon,
+                    "minlat":minlat,
+                    "maxlat":maxlat,
+                    "begTime":begTime,
+                    "endTime":endTime,
+                    "begHour":begHour,
+                    "endHour":endHour,
+                    "requestID":str(db_key),
+                    "property":str(property)
+                }
             try:
-                push_to_rabbitmq(data=rabbitmq_data)
+                queue_name = ''
+                if data_type == 'nexrad':
+                    queue_name = 'offload_request'
+                else:
+                    queue_name = 'offload_queue_nasa'
+                push_to_rabbitmq(data=rabbitmq_data, queue_name=queue_name)
             except:
                 response['response_code'] = "3"
                 response['response_message'] = "Failed to add new job to rabbitmq"
                 response['data_dump'] = ""
-
     else:
         response['response_code'] = "1"
         response['response_message'] = "Fail"

db_middleware/Dockerfile

@@ -1,6 +1,9 @@
 FROM node:14-alpine 
 
+
 WORKDIR /db_middleware
+
+
 COPY package.json .
 RUN npm install
 COPY . .