[FEATURE] - Port spotInstance feature from furyctl-next branch (#70)

* Port spotInstance feature from furyctl-next branch, and all the minor changes on the copyright side * Fix addlicense drone step * Update drone.yml, enable e2e, fix release process * Upgrade vpc-and-vpn eks module to v1.9.1
sighupio · May 3, 2022 · 8bd61f7 · 8bd61f7
1 parent 51d553d
commit 8bd61f7
Show file tree

Hide file tree

Showing 47 changed files with 293 additions and 254 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -7,7 +7,7 @@ name: license
 
 steps:
   - name: check
-    image: golang
+    image: golang:1.17-alpine
     pull: always
     commands:
       - go get -u github.com/google/addlicense
@@ -17,6 +17,9 @@ steps:
 kind: pipeline
 name: Build Test and Release
 
+node:
+  runner: internal
+
 depends_on:
   - license
 
@@ -89,84 +92,84 @@ steps:
     name: integration-vsphere
     commands:
       - bats -t ./automated-tests/integration/vsphere/tests.sh
-# TODO fix e2e tests
-#  - &e2e
-#    name: e2e-gcp
-#    image: quay.io/sighup/e2e-furyctl:v1.2.1_v0.2.2_v1.20.1_20.04
-#    pull: always
-#    privileged: true # Required to connect to the VPN
-#    depends_on:
-#      - build
-#    environment:
-#      FURYCTL_TOKEN:
-#        from_secret: FURYCTL_TOKEN
-#
-#      TERRAFORM_TF_STATES_BUCKET_NAME:
-#        from_secret: TERRAFORM_TF_STATES_BUCKET_NAME
-#      AWS_ACCESS_KEY_ID:
-#        from_secret: AWS_ACCESS_KEY_ID
-#      AWS_SECRET_ACCESS_KEY:
-#        from_secret: AWS_SECRET_ACCESS_KEY
-#      AWS_REGION:
-#        from_secret: AWS_REGION
-#      AWS_DEFAULT_REGION:
-#        from_secret: AWS_REGION
-#
-#      GCP_CREDENTIALS:
-#        from_secret: GCP_CREDENTIALS
-#      GCP_CREDENTIALS_PATH: /tmp/terraform-credentials.json
-#
-#      GOOGLE_PROJECT:
-#        from_secret: GOOGLE_PROJECT
-#      CLOUDSDK_CORE_PROJECT:
-#        from_secret: GOOGLE_PROJECT
-#      GOOGLE_REGION:
-#        from_secret: GOOGLE_REGION
-#
-#      VSPHERE_USER:
-#        from_secret: VSPHERE_USER
-#      VSPHERE_PASSWORD:
-#        from_secret: VSPHERE_PASSWORD
-#      VSPHERE_SERVER:
-#        from_secret: VSPHERE_SERVER
-#      VSPHERE_DATACENTER:
-#        from_secret: VSPHERE_DATACENTER
-#      VSPHERE_DATASTORE:
-#        from_secret: VSPHERE_DATASTORE
-#      VSPHERE_HOST:
-#        from_secret: VSPHERE_HOST
-#      VSPHERE_NET:
-#        from_secret: VSPHERE_NET
-#      VSPHERE_TEMPLATE_PREFIX:
-#        from_secret: VSPHERE_TEMPLATE_PREFIX
-#    commands:
-#      - echo $${GCP_CREDENTIALS} | base64 -d > $${GCP_CREDENTIALS_PATH}
-#      - export GOOGLE_APPLICATION_CREDENTIALS=$${GCP_CREDENTIALS_PATH}
-#      - bats -t ./automated-tests/e2e-tests/gcp-gke/tests.sh
-#    when:
-#      event:
-#        - tag
-#
-#  - <<: *e2e
-#    name: e2e-aws
-#    commands:
-#      - bats -t ./automated-tests/e2e-tests/aws-eks/tests.sh
-#
-#  - <<: *e2e
-#    name: e2e-vsphere
-#    commands:
-#      - bats -t ./automated-tests/e2e-tests/vsphere/tests.sh
-#
-#  - <<: *e2e
-#    name: e2e-vsphere-destroy
-#    depends_on:
-#      - e2e-vsphere
-#    failure: ignore
-#    commands:
-#      - echo " Our vsphere environment is not so stable"
-#      - echo " The destroy phase fails randomly because of disconnection events between vsphere nodes and the API"
-#      - echo " Ignore errors on this step"
-#      - bats -t ./automated-tests/e2e-tests/vsphere/tests-destroy.sh
+
+  - &e2e
+    name: e2e-gcp
+    image: quay.io/sighup/e2e-furyctl:v1.2.1_v0.2.2_v1.20.1_20.04
+    pull: always
+    privileged: true # Required to connect to the VPN
+    depends_on:
+      - build
+    environment:
+      FURYCTL_TOKEN:
+        from_secret: FURYCTL_TOKEN
+
+      TERRAFORM_TF_STATES_BUCKET_NAME:
+        from_secret: TERRAFORM_TF_STATES_BUCKET_NAME
+      AWS_ACCESS_KEY_ID:
+        from_secret: AWS_ACCESS_KEY_ID
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: AWS_SECRET_ACCESS_KEY
+      AWS_REGION:
+        from_secret: AWS_REGION
+      AWS_DEFAULT_REGION:
+        from_secret: AWS_REGION
+
+      GCP_CREDENTIALS:
+        from_secret: GCP_CREDENTIALS
+      GCP_CREDENTIALS_PATH: /tmp/terraform-credentials.json
+
+      GOOGLE_PROJECT:
+        from_secret: GOOGLE_PROJECT
+      CLOUDSDK_CORE_PROJECT:
+        from_secret: GOOGLE_PROJECT
+      GOOGLE_REGION:
+        from_secret: GOOGLE_REGION
+
+      VSPHERE_USER:
+        from_secret: VSPHERE_USER
+      VSPHERE_PASSWORD:
+        from_secret: VSPHERE_PASSWORD
+      VSPHERE_SERVER:
+        from_secret: VSPHERE_SERVER
+      VSPHERE_DATACENTER:
+        from_secret: VSPHERE_DATACENTER
+      VSPHERE_DATASTORE:
+        from_secret: VSPHERE_DATASTORE
+      VSPHERE_HOST:
+        from_secret: VSPHERE_HOST
+      VSPHERE_NET:
+        from_secret: VSPHERE_NET
+      VSPHERE_TEMPLATE_PREFIX:
+        from_secret: VSPHERE_TEMPLATE_PREFIX
+    commands:
+      - echo $${GCP_CREDENTIALS} | base64 -d > $${GCP_CREDENTIALS_PATH}
+      - export GOOGLE_APPLICATION_CREDENTIALS=$${GCP_CREDENTIALS_PATH}
+      - bats -t ./automated-tests/e2e-tests/gcp-gke/tests.sh
+    when:
+      event:
+        - tag
+
+  - <<: *e2e
+    name: e2e-aws
+    commands:
+      - bats -t ./automated-tests/e2e-tests/aws-eks/tests.sh
+
+  - <<: *e2e
+    name: e2e-vsphere
+    commands:
+      - bats -t ./automated-tests/e2e-tests/vsphere/tests.sh
+
+  - <<: *e2e
+    name: e2e-vsphere-destroy
+    depends_on:
+      - e2e-vsphere
+    failure: ignore
+    commands:
+      - echo " Our vsphere environment is not so stable"
+      - echo " The destroy phase fails randomly because of disconnection events between vsphere nodes and the API"
+      - echo " Ignore errors on this step"
+      - bats -t ./automated-tests/e2e-tests/vsphere/tests-destroy.sh
 
   - name: build-release
     image: ghcr.io/goreleaser/goreleaser:v1.4.1
@@ -177,9 +180,9 @@ steps:
       - integration-gcp-gke
       - integration-aws-eks
       - integration-vsphere
-#      - e2e-gcp
-#      - e2e-aws
-#      - e2e-vsphere
+      - e2e-aws
+      - e2e-gcp
+      - e2e-vsphere
     environment:
       GITHUB_TOKEN:
         from_secret: GITHUB_TOKEN
@@ -190,5 +193,8 @@ steps:
       - GO111MODULE=on goreleaser check
       - GO111MODULE=on goreleaser --debug release --rm-dist
     when:
-      event:
-        - tag
+      ref:
+        exclude:
+          - refs/tags/v**-rc**
+        include:
+          - refs/tags/v**
diff --git a/automated-tests/e2e-tests/aws-eks/cluster.tpl.yml b/automated-tests/e2e-tests/aws-eks/cluster.tpl.yml
@@ -25,6 +25,12 @@ spec:
         ports: 53-53
         tags:
           allow: dns
+    - name: my-spot-pool
+      minSize: 1
+      maxSize: 1
+      volumeSize: 50
+      instanceType: t3.micro
+      spotInstance: true
 provisioner: eks
 executor:
   state:

diff --git a/automated-tests/e2e-tests/aws-eks/tests.sh b/automated-tests/e2e-tests/aws-eks/tests.sh
@@ -197,12 +197,11 @@ fi
     [ "$status" -eq 0 ]
 }
 
-@test "kubectl" {
+@test "kubectl get pods" {
     info
     cluster_info(){
         export KUBECONFIG=./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig
         kubectl get pods -A >&3
-        kubectl get nodes -o wide >&3
     }
     run cluster_info
 
@@ -213,6 +212,37 @@ fi
     [ "$status" -eq 0 ]
 }
 
+@test "kubectl get nodes" {
+    info
+    cluster_info(){
+        export KUBECONFIG=./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig
+        kubectl get nodes -o wide --show-labels >&3
+    }
+    run cluster_info
+
+    if [[ $status -ne 0 ]]; then
+        echo "$output" >&3
+        cat ./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig >&3
+    fi
+    [ "$status" -eq 0 ]
+}
+
+@test "kubectl get nodes verify spot presence" {
+    info
+    test(){
+        export KUBECONFIG=./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig
+        data=$(kubectl get nodes --show-labels | grep "node.kubernetes.io/lifecycle=spot")
+        if [ "${data}" == "" ]; then return 1; fi
+    }
+    loop_it test 60 5
+    status=${loop_it_result}
+    if [[ $status -ne 0 ]]; then
+        echo "$output" >&3
+        cat ./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig >&3
+    fi
+    [ "$status" -eq 0 ]
+}
+
 @test "Cluster destroy" {
     info
     destroy(){

diff --git a/automated-tests/e2e-tests/gcp-gke/cluster.tpl.yml b/automated-tests/e2e-tests/gcp-gke/cluster.tpl.yml
@@ -6,7 +6,7 @@ kind: Cluster
 metadata:
   name: e2e-${CI_BUILD_NUMBER}
 spec:
-  version: 1.20.9-gke.701
+  version: 1.21.10-gke.400
   network: e2e-${CI_BUILD_NUMBER}
   subnetworks:
     - e2e-${CI_BUILD_NUMBER}-cluster-subnet
@@ -30,6 +30,12 @@ spec:
         ports: 53-53
         tags:
           allow: dns
+    - name: my-preemptible-pool
+      minSize: 1
+      maxSize: 1
+      volumeSize: 50
+      instanceType: n1-standard-1
+      spotInstance: true
 provisioner: gke
 executor:
   state:

diff --git a/automated-tests/e2e-tests/gcp-gke/tests.sh b/automated-tests/e2e-tests/gcp-gke/tests.sh
@@ -210,6 +210,22 @@ fi
     [ "$status" -eq 0 ]
 }
 
+@test "kubectl get nodes verify spot presence" {
+    info
+    test(){
+        export KUBECONFIG=./automated-tests/e2e-tests/gcp-gke/cluster/secrets/kubeconfig
+        data=$(kubectl get nodes --show-labels | grep "cloud.google.com/gke-preemptible=true")
+        if [ "${data}" == "" ]; then return 1; fi
+    }
+    loop_it test 60 5
+    status=${loop_it_result}
+    if [[ $status -ne 0 ]]; then
+        echo "$output" >&3
+        cat ./automated-tests/e2e-tests/aws-eks/cluster/secrets/kubeconfig >&3
+    fi
+    [ "$status" -eq 0 ]
+}
+
 @test "Cluster destroy" {
     info
     destroy(){

diff --git a/automated-tests/e2e-tests/vsphere/cluster.tpl.yml b/automated-tests/e2e-tests/vsphere/cluster.tpl.yml
@@ -23,19 +23,19 @@ spec:
   boundary: true
   lbNode:
     count: 1
-    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-oraclelinux7.9-template-v20210413"
+    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-ubuntu20LTS-template-v20210115"
   masterNode:
     count: 1
     cpu: 2
     memSize: 8192
     diskSize: 100
-    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-oraclelinux7.9-template-v20210413"
+    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-ubuntu20LTS-template-v20210115"
   infraNode:
     count: 1
     cpu: 2
     memSize: 8192
     diskSize: 100
-    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-oraclelinux7.9-template-v20210413"
+    template: "${VSPHERE_TEMPLATE_PREFIX}/sighup-ubuntu20LTS-template-v20210115"
   nodePools: []
   clusterPODCIDR: 172.21.0.0/16
   clusterSVCCIDR: 172.23.0.0/16

diff --git a/automated-tests/integration/aws-eks/cluster.yml b/automated-tests/integration/aws-eks/cluster.yml
@@ -19,3 +19,10 @@ spec:
     maxSize: 1
     volumeSize: 50
     instanceType: t3.micro
+  - name: my-spot-node-pool
+    minSize: 1
+    maxSize: 1
+    spotInstance: true
+    volumeSize: 50
+    instanceType: t3.micro
+
diff --git a/automated-tests/integration/gcp-gke/cluster.yml b/automated-tests/integration/gcp-gke/cluster.yml
@@ -21,3 +21,9 @@ spec:
     maxSize: 1
     volumeSize: 50
     instanceType: n1-standard-1
+  - name: my-spot-node-pool
+    minSize: 1
+    maxSize: 1
+    volumeSize: 50
+    spotInstance: true
+    instanceType: n1-standard-1
diff --git a/cmd/bootstrap.go b/cmd/bootstrap.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2020 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 

diff --git a/cmd/cluster.go b/cmd/cluster.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2020 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 

diff --git a/cmd/completion.go b/cmd/completion.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2021 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 

diff --git a/cmd/config.go b/cmd/config.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2021 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 

diff --git a/cmd/download.go b/cmd/download.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2020 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 

diff --git a/cmd/init.go b/cmd/init.go
@@ -1,4 +1,4 @@
-// Copyright (c) 2020 SIGHUP s.r.l All rights reserved.
+// Copyright (c) 2022 SIGHUP s.r.l All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.