Merge pull request #138 from johnbedeir/dev

Adjust-Kasten-EKS

Cluster-Backup/Kasten-EKS/README.md

@@ -1,13 +1,71 @@
-# Cluster Backup on EKS using Kasten
+### `For Prerequisites check` [here](https://github.com/johnbedeir/Devops-Tools-Documentation/blob/main/Cluster-Backup/Kasten-Minikube/prerequisites.sh)
 
-## Step 1: [Install helm](https://helm.sh/docs/intro/install/)
+---
 
-## Step 2: Create Jenkins Job
+# Backup EKS Cluster using Kasten (K10)
 
-Create Jenkins Job to run the Jenkinsfile pipeline
+## Step 1: Build Infrastructure
 
-`NOTE: make sure you have port 4000 open on your EC2 before deploying Kasten`
+This step will build S3 Bucket where all the backup will be stored and IAM Role that will be used by Kasten to Access your S3 Bucket.
 
-## Step 3: Deploy Kasten
+Build the infrastructure by Terraform using the following commands
 
-Run the Jenkins job you just created
+```
+cd Terraform
+
+terraform init
+
+terraform plan
+
+terraform apply -auto-approve
+```
+
+## Step 2: Create Cluster and Deploy Application
+
+```
+ansible-playbook aws-eks-app-deploy.yaml
+```
+
+## To delete cluster and application deployment
+
+```
+ansible-playbook aws-eks-app-remove.yaml
+```
+
+## Step 3: Deploy Kasten AWS
+
+Deploy Kasten using the automated script **kasten-deployment.sh**
+
+```
+cd Terrafrom
+
+chmod +x kasten-deployment.sh
+
+./kasten-deployment.sh
+```
+
+which will do the following:
+
+1. Run pre-check before deployment
+2. Update kubeconfig
+3. Show the available OIDC then associate it with the cluster
+4. Add helm repo for Kasten
+5. Create namespace for Kasten
+6. Install Kasten using helm
+7. Sleep for 1 minute until the pods are started
+8. Update the helm repo and upgrade Kasten
+9. Set external gateway to access Kasten via LoadBalancer
+10. Reveal Kasten URL
+11. Reveal Kubernetes token to access Kasten
+
+## Delete Kasten Deployment
+
+For deleting kasten deployment use the script **delete-kasten-deployment.sh**
+
+```
+cd Terraform
+
+chmod +x delete-kasten-deployment.sh
+
+./delete-kasten-deployment.sh
+```

Cluster-Backup/Kasten-EKS/Terraform/delete-kasten-deployment.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+NAMESPACE=kasten-io
+EKS_CLUSTER_NAME=simple-project-cluster
+oidc_id=$(aws eks describe-cluster --name ${EKS_CLUSTER_NAME} --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
+oidc_arn=$(aws iam list-open-id-connect-providers | grep $oidc_id | awk '{print $2}' | tr -d '"')
+
+#Delete OIDC
+echo "--------------------Delete Associated OIDC--------------------"
+aws iam delete-open-id-connect-provider --open-id-connect-provider-arn $oidc_arn
+
+#Remove kasten helm repo 
+echo "--------------------Remove K10 Helm Repo--------------------"
+helm repo remove kasten 
+
+#Delete Kasten using Helm 
+echo "--------------------Delete K10 Helm Deployment--------------------"
+helm uninstall k10 --namespace=kasten-io
+
+#Delete deployments
+echo "--------------------Delete Deployment--------------------"
+kubectl delete deploy --all -n $NAMESPACE || true
+
+#Delete services
+echo "--------------------Delete Services--------------------"
+kubectl delete service --all -n $NAMESPACE || true
+
+#Delete configmap
+echo "--------------------Delete Configmap--------------------"
+kubectl delete configmap --all -n $NAMESPACE || true
+
+#Delete namespace
+echo "--------------------Delete Namespace--------------------"
+timeout 5s kubectl delete namespace $NAMESPACE || true
+
+#Get namespace json file 
+echo "--------------------Get NS Json--------------------"
+kubectl get namespace $NAMESPACE -o json > ns.json
+
+#Remove kubernetes from namespace metadata to be able to delete namespace
+echo "--------------------Edit NS Json--------------------"
+sed -i '/"kubernetes"/d' ./ns.json
+
+# #replace the new json with the old one 
+echo "--------------------Replace NS Json--------------------"
+kubectl replace --raw "/api/v1/namespaces/$NAMESPACE/finalize" -f ./ns.json
+
+#Wait to completely remove K10
+echo "--------------------Wait to completely remove k10--------------------"
+sleep 30s

Cluster-Backup/Kasten-EKS/Terraform/kasten-deployment.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+NAMESPACE=kasten-io
+EKS_CLUSTER_NAME=simple-project-cluster
+KASTEN_AWS_ACCESS_KEY=$(terraform output access_key_id)
+KASTEN_AWS_SECRET_KEY=$(terraform output secret_access_key)
+KASTEN_IAM_ARN=$(terraform output user_arn)
+oidc_id=$(aws eks describe-cluster --name ${EKS_CLUSTER_NAME} --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
+sa_secret=$(kubectl get serviceaccount k10-k10 -o jsonpath="{.secrets[0].name}" --namespace kasten-io)
+k10_url=$(kubectl get service gateway-ext -n kasten-io | awk '{print $4}')
+token=$(kubectl get secret $sa_secret --namespace kasten-io -ojsonpath="{.data.token}{'\n'}" | base64 --decode > token.txt)
+
+echo "--------------------Deploy the the pre-check tool--------------------"
+curl https://docs.kasten.io/tools/k10_primer.sh | bash
+
+echo "--------------------Update kubeconfig with cluster name--------------------"
+aws eks update-kubeconfig --name ${EKS_CLUSTER_NAME}
+
+echo "--------------------Create IAM OIDC & associate with cluster--------------------"
+eksctl utils associate-iam-oidc-provider --cluster ${EKS_CLUSTER_NAME} --approve
+
+echo "--------------------Add Kasten Helm Repo--------------------"
+helm repo add kasten https://charts.kasten.io/ || true
+
+echo "--------------------Create Kasten namespace--------------------"
+kubectl create namespace $NAMESPACE || true 
+
+echo "--------------------Install Kasten--------------------"
+helm install k10 kasten/k10 --namespace=$NAMESPACE \
+    --set secrets.awsAccessKeyId="${KASTEN_AWS_ACCESS_KEY}" \
+    --set secrets.awsSecretAccessKey="${KASTEN_AWS_SECRET_KEY}" \
+    --set secrets.awsIamRole="${KASTEN_IAM_ARN}" 
+
+echo "--------------------Wait Pod is Starting--------------------"
+sleep 1m
+
+echo "--------------------Update Helm Repo & Upgrade K10--------------------"
+helm repo update && \
+    helm get values k10 --output yaml --namespace=kasten-io > k10_val.yaml && \
+    helm upgrade k10 kasten/k10 --namespace=kasten-io -f k10_val.yaml \
+    --set secrets.awsAccessKeyId="${KASTEN_AWS_ACCESS_KEY}" \
+    --set secrets.awsSecretAccessKey="${KASTEN_AWS_SECRET_KEY}"
+
+#This will set external gateway so we can access K10 via loadbalancer 
+echo "--------------------Access K10 via LoadBalancer--------------------"
+helm upgrade k10 kasten/k10 --namespace=kasten-io \
+    --reuse-values \
+    --set externalGateway.create=true \
+    --set auth.tokenAuth.enabled=true \
+    --set secrets.awsAccessKeyId="${KASTEN_AWS_ACCESS_KEY}" \
+    --set secrets.awsSecretAccessKey="${KASTEN_AWS_SECRET_KEY}" 
+
+#This step to reveal the loadbalancer URL which will be used to access K10 via browser 
+echo "--------------------Reveal K10 URL--------------------"
+echo "Use the following URL to access Kasten from your browser: MAKE SURE YOU ADD /k10/# in the end of the URL"
+kubectl get service gateway-ext -n kasten-io | awk '{print $4}'
+
+echo "--------------------Remove Old Token--------------------"
+rm -rf token.txt
+
+#This is going to reveal K82 Token that will be used once we access K10 on browser 
+echo "--------------------Find the K8s Token in token.txt--------------------"
+echo "$token"

Cluster-Backup/Kasten-EKS/Terraform/main.tf

@@ -0,0 +1,7 @@
+module s3 {
+  source = "./modules/s3_bucket"
+}
+
+module users {
+    source = "./modules/users"
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/s3_bucket/output.tf

@@ -0,0 +1,9 @@
+output "s3_bucket_id" {
+    value = aws_s3_bucket.s3.id
+    description = "The S3 Bucket Name"
+}
+
+output "s3_bucket_arn" {
+    value = aws_s3_bucket.s3.arn
+    description = "The S3 Bucket ARN"
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/s3_bucket/s3_bucket.tf

@@ -0,0 +1,12 @@
+resource "aws_s3_bucket" "s3" {
+  bucket = var.BUCKETNAME
+
+  tags = {
+    Name        = var.BUCKETNAME
+  }
+}
+
+resource "aws_s3_bucket_acl" "s3-acl" {
+  bucket = aws_s3_bucket.s3.id
+  acl    = "private"
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/s3_bucket/vars.tf

@@ -0,0 +1,4 @@
+variable "BUCKETNAME" {
+  type    = string
+  default = "k10-cluster-backup"
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/users/output.tf

@@ -0,0 +1,20 @@
+output "user_name" {
+  value       = aws_iam_user.k10-user.name
+  description = "IAM user name"
+}
+
+output "user_arn" {
+  value       = aws_iam_user.k10-user.arn
+  description = "The ARN assigned by AWS for this user"
+}
+
+output "access_key_id" {
+  value       = aws_iam_access_key.k10-user.id
+  description = "The access key ID"
+}
+
+output "secret_access_key" {
+  sensitive   = true
+  value       = aws_iam_access_key.k10-user.secret
+  description = "The secret access key. This will be written to the state file in plain-text"
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/users/users.tf

@@ -0,0 +1,28 @@
+resource "aws_iam_user" "k10-user" {
+  name = var.user
+  path = "/system/"
+
+  tags = {
+    tag-key = var.user
+  }
+}
+
+resource "aws_iam_access_key" "k10-user" {
+  user = aws_iam_user.k10-user.name
+}
+
+resource "aws_iam_user_policy" "k10-user_policy" {
+  name = var.user_policy
+  user = aws_iam_user.k10-user.name
+
+  policy = jsonencode({
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Action: "s3:*",
+        Resource: "arn:aws:s3:::${var.BUCKETNAME}/*"
+      }
+    ]
+  })
+}

Cluster-Backup/Kasten-EKS/Terraform/modules/users/vars.tf

@@ -0,0 +1,16 @@
+variable "BUCKETNAME" {
+  type    = string
+  default = "k10-cluster-backup"
+}
+
+variable "user" {
+  type        = string
+  default     = "k10-user"
+  description = "IAM user name"
+}
+
+variable "user_policy" {
+  type        = string
+  default     = "k10-user-policy"
+  description = "IAM user policy for accessing the S3 Bucket"
+}

Cluster-Backup/Kasten-EKS/Terraform/output.tf

@@ -0,0 +1,30 @@
+output "s3_bucket_id" {
+  value       = module.s3.s3_bucket_id
+  description = "description"
+}
+
+output "s3_bucket_arn" {
+  value       = module.s3.s3_bucket_arn
+  description = "description"
+}
+
+output "user_name" {
+  value       = module.users.user_name
+  description = "IAM user name"
+}
+
+output "user_arn" {
+  value       = module.users.user_arn
+  description = "The ARN assigned by AWS for this user"
+}
+
+output "access_key_id" {
+  value       = module.users.access_key_id
+  description = "The access key ID"
+}
+
+output "secret_access_key" {
+  sensitive   = true
+  value       = module.users.secret_access_key
+  description = "The secret access key. This will be written to the state file in plain-text"
+}

Cluster-Backup/Kasten-EKS/Terraform/provider.tf

@@ -0,0 +1,4 @@
+provider "aws" {
+  profile = "default"
+  region  = var.region
+}

Cluster-Backup/Kasten-EKS/Terraform/variables.tf

@@ -0,0 +1,4 @@
+variable "region" {
+  type    = string 
+  default = "eu-central-1"
+}

Cluster-Backup/Kasten-EKS/aws-eks-app-deploy.yaml

@@ -0,0 +1,18 @@
+---
+- name: "Create EKS and deploy simple app "
+  hosts: localhost
+  tasks:
+    - name: Create cluster
+      command: eksctl create cluster --name simple-project-cluster --nodes-min=2
+
+    - name: Update kubeconfig
+      command: aws eks update-kubeconfig --name simple-project-cluster
+
+    - name: Create new namespace
+      command: kubectl create ns simple-project
+
+    - name: Kubernetes deployment
+      command: kubectl apply -f simple-project/k8s/deployment.yaml
+
+    - name: Kubernetes services
+      command: kubectl apply -f simple-project/k8s/services.yaml

Cluster-Backup/Kasten-EKS/aws-eks-app-remove.yaml

@@ -0,0 +1,19 @@
+---
+- name: "Delete deployment and eks"
+  hosts: localhost
+  tasks:
+    - name: Delete simple-project deployment if exists
+      command: kubectl delete deploy comingsoon-page-img-deployment -n simple-project
+      ignore_errors: yes
+
+    - name: Delete simple-project service if exists
+      command: kubectl delete service comingsoon-page -n simple-project
+      ignore_errors: yes
+
+    - name: Delete namespace if exists
+      command: kubectl delete namespace simple-project
+      ignore_errors: yes
+
+    - name: Delete Cluster if exists
+      command: eksctl delete cluster --name simple-project-cluster --region eu-central-1
+      ignore_errors: yes

Cluster-Backup/Kasten-EKS/simple-project/.dockerignore

@@ -0,0 +1,2 @@
+k8s
+Dockerfile

Cluster-Backup/Kasten-EKS/simple-project/Dockerfile

@@ -0,0 +1,2 @@
+FROM nginx:1.15.0-alpine
+COPY . /usr/share/nginx/html/

Cluster-Backup/Kasten-EKS/simple-project/README

@@ -0,0 +1 @@
+# CominSoon