tools: Adds sonobuoy conformance testing scripts

Adds sonobuoy conformance testing scripts that spins up EKS clusters
and several thar worker nodes then runs sonobuoy conformance testing,
after its done, it retrieves the results and spins down the EKS cluster.

tools/conformance-test/README.md

@@ -0,0 +1,29 @@
+# Thar Kubernetes Conformance Testing
+`setup-test-cluster.sh` sets up an EKS cluster and generates an env file containing cluster information and the user data used to launch Thar worker nodes.
+
+`run-conformance-test.sh` uses the generated env file to launch specified number of worker nodes in the described EKS cluster and runs Kubernetes conformance tests.
+Once the tests are done, the script outputs the test results in the current directory and cleans up the worker nodes.
+
+`clean-up-test-cluster` deletes the EKS test cluster specified by the env file and any files generated by `setup-test-cluster.sh`.
+
+## Running conformance tests against Thar nodes using these scripts:
+1. Run `setup-test-cluster.sh` to set up a test cluster:
+   ```
+   setup-test-cluster.sh --region us-west-2 --cluster-name my-test-cluster
+   ```
+   Once the setup completes successfully, there should be a user data file and an env file in the current directory.
+2. Run `run-conformance-test.sh` to launch Thar worker nodes and run the Kubernetes conformance test:
+   ```
+   run-conformance-test.sh --node-ami ami-07245e9300b9290c1 \
+     --cluster-env-file my-test-cluster.env 
+     --instance-type m5.large \
+     --num-nodes 3
+   ```
+3. The Sonobuoy Kubernetes conformance test results can be examined with:
+   ```
+   sonobuoy results my-test-cluster-conformance-test-results/TIMESTAMP.tar.gz
+   ```
+4. To clean up the test cluster and other associated resources, run `clean-up-test-cluster.sh`:
+   ```
+   clean-up-test-cluster.sh --cluster-env-file my-test-cluster.env
+   ```

tools/conformance-test/clean-up-test-cluster.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+
+# Check for required tools
+if ! command -v eksctl >/dev/null; then
+  echo "* Can't find executable 'eksctl'" >&2
+  exit 2
+fi
+
+# Helper functions
+
+usage() {
+  cat >&2 <<EOF
+${0##*/}
+                --cluster-env-file my-test-cluster.env
+
+Cleans up the EKS cluster described by the specified env file and any files generated by 'setup-test-cluster.sh'
+
+Required:
+   --cluster-env-file           Path to the env file containing EKS cluster information targetted for clean up
+EOF
+}
+
+required_arg() {
+  local arg="${1:?}"
+  local value="${2}"
+  if [ -z "${value}" ]; then
+    echo "ERROR: ${arg} is required" >&2
+    exit 2
+  fi
+}
+
+parse_args() {
+  while [ ${#} -gt 0 ] ; do
+    case "${1}" in
+      --cluster-env-file ) shift; ENV_FILE="${1}" ;;
+
+      --help ) usage; exit 0 ;;
+      *)
+        echo "ERROR: Unknown argument: ${1}" >&2
+        usage
+        exit 2
+        ;;
+    esac
+    shift
+  done
+
+  # Required arguments
+  required_arg "--cluster-env-file" "${ENV_FILE}"
+}
+
+exit_on_error() {
+  local rc="${1:?}"
+  local msg="${2:?}"
+
+  if [ "${rc}" -ne 0 ]; then
+    echo "${msg}" >&2
+    exit 1
+  fi
+}
+
+# Initial setup and checks
+parse_args "${@}"
+
+# Load the env file created by the 'setup-test-cluster' script, if it doesn't exist, exit
+if [ -f "${ENV_FILE}" ]; then
+  . "${ENV_FILE}"
+else
+  echo "* Failed to open env file at ${ENV_FILE}." >&2
+  exit 1
+fi
+
+echo "Removing security group dependencies."
+aws ec2 revoke-security-group-ingress \
+  --region "${REGION}" \
+  --group-id "${NODEGROUP_SG}" \
+  --protocol tcp \
+  --port 1-1024 \
+  --source-group "${CONTROLPLANE_SG}"
+exit_on_error ${?} "* Failed to remove nodegroup sg ingress rules"
+
+aws ec2 revoke-security-group-egress \
+  --region "${REGION}" \
+  --group-id "${CONTROLPLANE_SG}" \
+  --protocol tcp \
+  --port 1-1024 \
+  --source-group "${NODEGROUP_SG}"
+exit_on_error ${?} "* Failed to remove control plane sg egress rules"
+
+echo "Deleting the test cluster."
+eksctl delete cluster -r "${REGION}" -n "${CLUSTER_NAME}"
+exit_on_error ${?} "* Failed to delete ${CLUSTER_NAME} with eksctl, there might be leftover CloudFormation stacks that needs to be deleted. Look for eksctl-${CLUSTER_NAME}-*" >&2
+
+echo "Deleting env file, userdata file, and kubeconfig file"
+rm -f "${ENV_FILE}"
+rm -f "${USERDATA_FILE}"
+rm -f "${KUBECONFIG_FILE}"
+
+echo "Clean up done."

tools/conformance-test/run-conformance-test.sh

@@ -0,0 +1,190 @@
+#!/usr/bin/env bash
+
+# Check for required tools
+for tool in jq sonobuoy aws kubectl; do
+  if ! command -v ${tool} > /dev/null; then
+    echo "* Can't find executable '${tool}'" >&2
+    exit 2
+  fi
+done
+
+DEFAULT_KUBE_CONFORMANCE_VERSION=v1.14.7
+DEFAULT_NUM_NODES=3
+
+# Helper functions
+
+usage() {
+  cat >&2 <<EOF
+${0##*/}
+                --node-ami <AMI ID>
+                --instance-type m5.large
+                --cluster-env-file my-test-cluster.env
+                [ --kube-conformance-version v1.14.6 ]
+
+Spins up worker nodes to join the EKS cluster described in the env file and then runs Sonobuoy Kubernetes conformance tests against said cluster.
+Once the tests are done, retrieves the results and cleans up the created worker node instances.
+
+Required:
+   --node-ami                   The AMI ID of the AMI to use for the worker nodes
+   --instance-type              Instance type launched for worker nodes
+   --cluster-env-file           Path to the env file containing cluster information for setting up worker nodes. Typically generated by 'setup-test-cluster.sh'.
+
+Optional:
+   --kube-conformance-version   The version of the conformance image to use for conformance testing (default "${DEFAULT_KUBE_CONFORMANCE_VERSION}")
+   --num-nodes                  The number of Thar worker nodes to launch (default ${DEFAULT_NUM_NODES})
+EOF
+}
+
+required_arg() {
+  local arg="${1:?}"
+  local value="${2}"
+  if [ -z "${value}" ]; then
+    echo "ERROR: ${arg} is required" >&2
+    exit 2
+  fi
+}
+
+parse_args() {
+  while [ ${#} -gt 0 ] ; do
+    case "${1}" in
+      --node-ami ) shift; NODE_AMI="${1}" ;;
+      --instance-type ) shift; INSTANCE_TYPE="${1}" ;;
+      --cluster-env-file ) shift; ENV_FILE="${1}" ;;
+      --kube-conformance-version ) shift; KUBE_CONFORMANCE_VERSION="${1}" ;;
+      --num-nodes ) shift; NUM_NODES="${1}" ;;
+
+      --help ) usage; exit 0 ;;
+      *)
+        echo "ERROR: Unknown argument: ${1}" >&2
+        usage
+        exit 2
+        ;;
+    esac
+    shift
+  done
+
+  KUBE_CONFORMANCE_VERSION="${KUBE_CONFORMANCE_VERSION:-${DEFAULT_KUBE_CONFORMANCE_VERSION}}"
+  NUM_NODES="${NUM_NODES:-${DEFAULT_NUM_NODES}}"
+
+  # Required arguments
+  required_arg "--node-ami" "${NODE_AMI}"
+  required_arg "--instance-type" "${INSTANCE_TYPE}"
+  required_arg "--cluster-env-file" "${ENV_FILE}"
+}
+
+cleanup() {
+  if [ ${#instance_ids[@]} -ne 0 ]; then
+    echo "Cleaning up Thar worker node instances"
+    for instance_id in "${instance_ids[@]}"; do
+      aws ec2 terminate-instances \
+        --output text \
+        --region "${REGION}" \
+        --instance-ids "${instance_id}"
+    done
+    unset instance_ids
+  fi
+
+ # Wait at most 20 minutes for Sonobuoy to delete its namespace
+  if [ -n "${sonobuoy_run_attempted}" ]; then
+    echo "Cleaning up Sonobuoy namespace, may take up to 20 minutes"
+    ${SONOBUOY} delete --wait=20
+    exit_on_error ${?} "* Failed to delete Sonobuoy namespace."
+    unset sonobuoy_run_attempted
+  fi
+}
+
+trap 'cleanup' EXIT SIGINT SIGTERM
+
+exit_on_error() {
+  local rc="${1:?}"
+  local msg="${2:?}"
+
+  if [ "${rc}" -ne 0 ]; then
+    echo "${msg}" >&2
+    exit 1
+  fi
+}
+
+# Initial setup and checks
+parse_args "${@}"
+
+# Load the env file created by the `setup-test-cluster` script, if it doesn't exist, exit
+if [ -f "${ENV_FILE}" ]; then
+  . "${ENV_FILE}"
+else
+  echo "* Failed to open env file at ${ENV_FILE}." >&2
+  exit 1
+fi
+
+echo "Launching ${NUM_NODES} Thar worker nodes"
+unset instance_ids
+counter=0
+while [ ${counter} -lt "${NUM_NODES}" ]; do
+  instance_id=$(aws ec2 run-instances \
+    --subnet-id "${SUBNET_ID}" \
+    --security-group-ids "${NODEGROUP_SG}" "${CLUSTERSHARED_SG}" \
+    --image-id "${NODE_AMI}" \
+    --instance-type "${INSTANCE_TYPE}" \
+    --region "${REGION}" \
+    --tag-specifications "ResourceType=instance,Tags=[{Key=kubernetes.io/cluster/${CLUSTER_NAME},Value=owned}]" \
+    --user-data file://"${USERDATA_FILE}" \
+    --iam-instance-profile Name="${INSTANCE_PROFILE}" \
+    --query "Instances[*].InstanceId" \
+    --output text)
+  if [ -n "${instance_id}" ]; then
+    instance_ids=("${instance_ids[@]}" "${instance_id}")
+  fi
+  sleep 1
+  ((counter+=1))
+done
+actual_num_nodes=${#instance_ids[@]}
+if [ "${actual_num_nodes}" -ne "${NUM_NODES}" ]; then
+  echo "* Failed to launch requested number of Thar instances: launched ${actual_num_nodes} out of ${NUM_NODES}." >&2
+  exit 1
+fi
+
+echo "Waiting for all Thar worker nodes to become 'Ready' in ${CLUSTER_NAME} cluster"
+KUBECTL="kubectl --kubeconfig ${KUBECONFIG_FILE}"
+MAX_ATTEMPTS=30
+attempts=0
+sleep 30
+while true; do
+  ((attempts+=1))
+  if [ "${attempts}" -gt ${MAX_ATTEMPTS} ]; then
+    echo "* Retry limit (${MAX_ATTEMPTS}) reached! Worker nodes are not becoming ready in cluster ${CLUSTER_NAME}" >&2
+    exit 1
+  fi
+  sleep 5
+  nodes=$(${KUBECTL} get nodes --no-headers)
+  exit_on_error ${?} "* Failed to get node information for ${CLUSTER_NAME} cluster"
+
+  found=$(${KUBECTL} get nodes --no-headers -o name | wc -l)
+  ready=$(echo -n "${nodes}" | grep -c -w "Ready")
+  echo "ready: ${ready}"
+
+  if [ "${found}" -eq "${actual_num_nodes}" ] && [ "${ready}" -eq "${actual_num_nodes}" ]; then
+    break
+  fi
+done
+
+echo "Starting Sonobuoy Kubernetes conformance test! Test may take up to 60 minutes to finish"
+sonobuoy_run_attempted=true
+SONOBUOY="sonobuoy --kubeconfig ${KUBECONFIG_FILE}"
+${SONOBUOY} run \
+  --mode certified-conformance \
+  --kube-conformance-image-version "${KUBE_CONFORMANCE_VERSION}" \
+  --wait
+exit_on_error ${?} "* Failed to run Sonobuoy Kubernetes conformance tests"
+${SONOBUOY} status
+exit_on_error ${?} "* Failed to retrieve conformance test status"
+results_file=$(${SONOBUOY} retrieve "${CLUSTER_NAME}"-conformance-test-results)
+exit_on_error ${?} "* Failed to retrieve Sonobuoy Kubernetes conformance test results"
+sonobuoy results "${results_file}"
+exit_on_error ${?} "* Failed to examine sonobuoy results in ${results_file}"
+echo "Sonobuoy test results available at ${results_file}"
+
+# Exit non-zero if any of the Kubernetes conformance tests fail
+(set -o pipefail; \
+  sonobuoy results "${results_file}" \
+  | awk '$2 == "failed" || $2 == "unknown" {exit 1}')
+exit_on_error ${?} "* Found conformance test failures or conformance tests failed to run (unknown status)."