kubevirtci

#!/bin/bash

set -e

export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.30}
export CAPK_GUEST_K8S_VERSION=${CAPK_GUEST_K8S_VERSION:-v1.30.1}
export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2405151527-09bcd71}
export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh)
export KUBEVIRT_DEPLOY_PROMETHEUS=false
export KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
export KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-15360M}
export KUBEVIRT_DEPLOY_CDI="true"
export METALLB_VERSION="v0.14.3"
export CAPK_RELEASE_VERSION="v0.1.8"
export CLUSTERCTL_VERSION="v1.6.1"
export CALICO_VERSION="v3.27.0"
export KUBEVIRT_VERSION="v1.1.1"
export NODE_VM_IMAGE_TEMPLATE=${NODE_VM_IMAGE_TEMPLATE:-quay.io/capk/ubuntu-2204-container-disk:v1.30.1}

_default_bin_path=./hack/tools/bin
_default_tmp_path=./hack/tools/bin/tmp
_default_clusterctl_path=./hack/tools/bin/clusterctl
_default_virtctl_path=./hack/tools/bin/virtctl

export CLUSTERCTL_PATH=${CLUSTERCTL_PATH:-${_default_clusterctl_path}}
export TENANT_CLUSTER_NAME=${TENANT_CLUSTER_NAME:-kvcluster}
export TENANT_CLUSTER_NAMESPACE=${TENANT_CLUSTER_NAMESPACE:-kvcluster}

_kubectl=cluster-up/cluster-up/kubectl.sh
_ssh_infra=cluster-up/cluster-up/ssh.sh

_action=$1
shift


function kubevirtci::usage() {
	echo "Usage:

	./kubevirtci <cammand>

	Commands:

	  up                                Start a cluster with kubevirt, cert-manager and capi
	  sync                              Build and deploy current capk from source (must be executed from within capk source tree)
	  down                              Destroy the cluster
    build                             Build
	  refresh                           Build current capk and trigger creating new capk pods
	  clean-cache                       Removes all files cached by kubevirtci

	  kubeconfig                        Return the kubeconfig of the cluster
	  kubectl <kubectl options>         Interact with the cluster
	  kubectl-tenant <kubectl options>  Interact with the tenant cluster
	  virtctl <virtctl options>         Run virtctl commands against the cluster
	  clusterctl <clusterctl options>   Run clusterctl commands against the cluster

	  install-capk                      Installs capk from published release manifests
	  install-metallb                   Installs metallb into the infra cluster
	  install-calico                    Installs calico cni into tenant cluster
	  curl-lb <lb name> [lb namespace]  Curls lb service within infra cluster

	  ssh-infra <node name>             SSH into one of the infra nodes (like node01)
	  ssh-tenant <vmi> [vmi namespace]  SSH into one of the guest nodes
	  create-cluster                    Create new kubernetes tenant cluster
	  create-external-cluster           Create new kubernetes tenant cluster simulated as running on external infra
	  destroy-cluster                   Destroy the tenant cluster

	  help                              Print usage
	"
}

function kubevirtci::kubeconfig() {
	cluster-up/cluster-up/kubeconfig.sh
}

function kubevirtci::fetch_kubevirtci() {
	[[ -d cluster-up ]] || git clone https://github.com/kubevirt/kubevirtci.git cluster-up
	(cd cluster-up && git checkout main > /dev/null 2>&1 && git pull > /dev/null && git checkout ${KUBEVIRTCI_TAG} > /dev/null 2>&1)
	mkdir -p ./hack/tools/bin/
	if [ ! -f "${_default_clusterctl_path}" ]; then
		echo >&2 "Downloading clusterctl version ${CLUSTERCTL_VERSION}..."
		curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/${CLUSTERCTL_VERSION}/clusterctl-linux-amd64 -o ${_default_clusterctl_path}
		chmod u+x ${_default_clusterctl_path}
	fi
	if [ ! -f "${_default_virtctl_path}" ]; then
		echo >&2 "Downloading virtctl ..."
		curl -L https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/virtctl-${KUBEVIRT_VERSION}-linux-amd64 -o ${_default_virtctl_path}
		chmod u+x ${_default_virtctl_path}
	fi
}

function kubevirtci::up() {
	make cluster-up -C cluster-up
	export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh)
	echo "installing kubevirt..."
	${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
	${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
	echo "installing capi..."
#The version of the url needs to match what clusterctl is expecting.
	cat << EOF > ${_default_bin_path}/clusterctl_config.yaml
---
cert-manager:
  url: "https://github.com/cert-manager/cert-manager/releases/1.13.2/cert-manager.yaml"
EOF
	$CLUSTERCTL_PATH init -v 4 --config=${_default_bin_path}/clusterctl_config.yaml
	echo "waiting for kubevirt to become ready, this can take a few minutes. You can safely abort this step, the cluster is ready ..."
	${_kubectl} -n kubevirt wait kv kubevirt --for condition=Available --timeout=5m
}

function kubevirtci::down() {
	make cluster-down -C cluster-up
}

function kubevirtci::build() {
	export REGISTRY="127.0.0.1:$(cluster-up/cluster-up/cli.sh ports registry)"
	make image-build
	make image-push
}

function kubevirtci::ssh_tenant() {
	if [ -z "$1" ]; then
		echo "No VM supplied"
	fi
	vmi_name=$1
	vmi_namespace=${2:-$TENANT_CLUSTER_NAMESPACE}
	if [ ! -z "$2" ]; then
		shift
	fi
	shift
	mkdir -p $_default_tmp_path

	echo "vmi $vmi_name namespace $vmi_namespace"

	${_kubectl} get secret -n $TENANT_CLUSTER_NAMESPACE kvcluster-ssh-keys -o jsonpath='{.data}' | grep key | awk -F '"' '{print $4}' | base64 -d > ${_default_tmp_path}/key.pem

	chmod 600 ${_default_tmp_path}/key.pem
	ssh -o IdentitiesOnly=yes -o "StrictHostKeyChecking=no" -o "ProxyCommand=$_default_virtctl_path port-forward --stdio=true $vmi_name.$vmi_namespace 22" capk@$vmi_name.$vmi_namespace -i ${_default_tmp_path}/key.pem "$@"

	rm ${_default_tmp_path}/key.pem
}

function kubevirtci::refresh() {
	${_kubectl} delete pods --all -n capk-system
}

function kubevirtci::install() {
	export MANIFEST_IMG="registry:5000/capk-manager-amd64"
	export MANIFEST_TAG="dev"
	make generate-manifests
	${_kubectl} kustomize config/kubevirtci | ${_kubectl} delete -f - || true
	${_kubectl} kustomize config/kubevirtci | ${_kubectl} apply -f -
	${_kubectl} wait -n capk-system --for=condition=Available=true deployment/capk-controller-manager --timeout=10m
}

function kubevirtci::generate_kubeconfig() {
        make clusterkubevirtadm-linux
        bin/clusterkubevirtadm-linux-amd64 apply credentials --namespace ${TENANT_CLUSTER_NAMESPACE} 
        bin/clusterkubevirtadm-linux-amd64 get kubeconfig --namespace=${TENANT_CLUSTER_NAMESPACE} --output-kubeconfig=kubeconfig-e2e
        sed -i -r 's/127.0.0.1:[0-9]+/192.168.66.101:6443/g' kubeconfig-e2e
}

function kubevirtci::destroy_cluster() {
	${_kubectl} delete cluster -n ${TENANT_CLUSTER_NAMESPACE} ${TENANT_CLUSTER_NAME} --ignore-not-found
}


function kubevirtci::create_cluster() {
	export IMAGE_REPO=k8s.gcr.io
	export CRI_PATH="/var/run/containerd/containerd.sock"
	template=templates/cluster-template.yaml
	if [ ! -f $template ]; then
		#can't pass the url directly because clusterctl doesn't resolve the redirects properly.
		tmp_template=$(mktemp -d)
		template="$tmp_template/cluster-template.yaml"
		echo "saving template to $template"
		curl -L "https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt/releases/download/$CAPK_RELEASE_VERSION/cluster-template.yaml" --output $template
	fi

	echo "Using cluster template $template"

	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f -

	echo "Wait for tenant cluster to be ready"
	${_kubectl} wait cluster -n ${TENANT_CLUSTER_NAMESPACE} kvcluster --for=condition=Ready --timeout=10m

	echo "Wait for tenant cluster kubernetes apiserver up"
	kubevirtci::retry_until_success kubevirtci::kubectl_tenant get pods -n kube-system

	echo "Waiting for worker VM in tenant cluster namespace"
	kubevirtci::retry_until_success kubevirtci::vm_matches "${TENANT_CLUSTER_NAME}-md-"

	if [ $tmp_template ]; then
		rm -rf $tmp_template
	fi
}


function kubevirtci::create_external_cluster() {
	export IMAGE_REPO=k8s.gcr.io
	export CRI_PATH="/var/run/containerd/containerd.sock"

	${_kubectl} delete secret external-infra-kubeconfig -n capk-system --ignore-not-found
	${_kubectl} create secret generic external-infra-kubeconfig -n capk-system --from-file=kubeconfig=kubeconfig-e2e --from-literal=namespace=${TENANT_CLUSTER_NAMESPACE}
	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from templates/cluster-template-ext-infra.yaml | ${_kubectl} apply -f -
}

function kubevirtci::create_tenant_namespace {
	${_kubectl} apply -f - <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: ${TENANT_CLUSTER_NAMESPACE}
EOF
}

function kubevirtci::install_capk_release {
	${_kubectl} apply -f https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt/releases/download/${CAPK_RELEASE_VERSION}/infrastructure-components.yaml

	${_kubectl} wait -n capk-system --for=condition=Available=true deployment/capk-controller-manager --timeout=10m

	echo "capk release $CAPK_RELEASE_VERSION installed!"
}


function kubevirtci::install_calico {
	kubevirtci::kubectl_tenant apply -f https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml
	echo "Waiting for calico pods rollout"
	kubevirtci::kubectl_tenant rollout status ds/calico-node -n kube-system --timeout=5m

}

function kubevirtci::install_metallb {
	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/namespace.yaml
	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/metallb.yaml

	echo "waiting for metallb to come online"
	${_kubectl} -n metallb-system wait deployment controller --for condition=Available --timeout=5m

	mkdir -p ${_default_tmp_path}
	local metal_config=${_default_tmp_path}/metallb-config.yaml

	cat << EOF > $metal_config
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 192.168.66.201-192.168.66.250
EOF

	${_kubectl} apply -f ${metal_config}

	rm $metal_config
	echo "metallb installed!"
}

function kubevirtci::curl_lb {
	mkdir -p ${_default_tmp_path}
	job_yaml=${_default_tmp_path}/curl-test-pod.yaml
	if [ -f ${job_yaml} ]; then
		${_kubectl} delete -f ${job_yaml} --ignore-not-found
	fi

	lb_name=$1
	lb_namespace=${2:-$TENANT_CLUSTER_NAMESPACE}

	$_kubectl get service $lb_name -n $lb_namespace

	lb_ip=$($_kubectl get service $lb_name -n $lb_namespace -o yaml | grep "ip:" | awk '{print $3}')
	lb_port=$($_kubectl get service $lb_name -n $lb_namespace -o yaml | grep "port:" | awk '{print $2}')


	cat << EOF > $job_yaml
---
apiVersion: batch/v1
kind: Job
metadata:
  name: curl-test-job
  namespace: ${lb_namespace}
spec:
  template:
    spec:
      containers:
      - name: fedora
        image: fedora:35
        command:
          - curl
          - "${lb_ip}:${lb_port}"
      restartPolicy: Never
  backoffLimit: 4
EOF

	${_kubectl} create -f $job_yaml
	echo "-----------Waiting for curl job to complete"
	${_kubectl} wait job curl-test-job -n default --for condition=Complete --timeout=5m

	pod_name=$($_kubectl get pods --selector=job-name=curl-test-job --output=jsonpath='{.items[*].metadata.name}')

	echo "-----------CURL LOG FOR POD $pod_name"
	$_kubectl logs -n default $pod_name 2>/dev/null

	${_kubectl} delete -f ${job_yaml} --ignore-not-found > /dev/null 2>&1
}


function kubevirtci::kubectl_tenant {
    vms_list=$(${_kubectl} get vm -n ${TENANT_CLUSTER_NAMESPACE} --no-headers -o custom-columns=":metadata.name")
    for vm in $vms_list
    do
	if [[ "$vm" == kvcluster-control-plane* ]]; then
            control_plane_vm_name=$vm
	fi
    done
    if [ -n "${control_plane_vm_name}" ]; then
	echo "Found control plane VM: ${control_plane_vm_name} in namespace ${TENANT_CLUSTER_NAMESPACE}"
    else
	echo "control-plane vm is not found in namespace ${TENANT_CLUSTER_NAMESPACE} (looking for regex kvcluster-control-plane*)"
	exit 1
    fi
    ${_default_virtctl_path} port-forward -n ${TENANT_CLUSTER_NAMESPACE} vm/${control_plane_vm_name} 64443:6443 > /dev/null 2>&1 &
    trap 'kill $(jobs -p) > /dev/null 2>&1' EXIT
    rm -f .${TENANT_CLUSTER_NAME}-kubeconfig
    $CLUSTERCTL_PATH get kubeconfig ${TENANT_CLUSTER_NAME} -n ${TENANT_CLUSTER_NAMESPACE} > .${TENANT_CLUSTER_NAME}-kubeconfig
    sleep 0.1
    kubectl --kubeconfig .${TENANT_CLUSTER_NAME}-kubeconfig --insecure-skip-tls-verify --server https://localhost:64443 "$@"
}

function kubevirtci::retry_until_success {
    local timeout=30
    local interval=1
    until $@; do
        ((timeout--)) && ((timeout==0)) && echo "condition not met" && exit 1
        echo "waiting for \"$@\""
        sleep $interval 
    done
}

function kubevirtci::vm_matches {
    local vm_name=$1
    ${_kubectl} get vm -n ${TENANT_CLUSTER_NAMESPACE} --no-headers -o custom-columns=":metadata.name" | grep -q $vm_name
}

kubevirtci::fetch_kubevirtci
case ${_action} in
"up")
	kubevirtci::up
	;;
"down")
	kubevirtci::down
	;;
"build")
	kubevirtci::build
	;;
"refresh")
	kubevirtci::build
	kubevirtci::refresh
	;;
"sync")
	kubevirtci::build
	kubevirtci::install
	;;
"install-capk")
	kubevirtci::install_capk_release
	;;
"install-metallb")
	kubevirtci::install_metallb
	;;
"install-calico")
	kubevirtci::install_calico
	;;
"curl-lb")
	kubevirtci::curl_lb "$@"
	;;
"kubeconfig")
	kubevirtci::kubeconfig
	;;
"kubectl")
	${_kubectl} "$@"
	;;
"kubectl-tenant")
	kubevirtci::kubectl_tenant "$@"
	;;
"virtctl")
	${_default_virtctl_path} "$@"
	;;
"ssh-infra")
	$_ssh_infra "$@"
	;;
"ssh-tenant")
	kubevirtci::ssh_tenant "$@"
	;;
"clusterctl")
	$CLUSTERCTL_PATH "$@"
	;;
"create-cluster")
	kubevirtci::create_tenant_namespace
	kubevirtci::create_cluster
	;;
"create-external-cluster")
	kubevirtci::create_tenant_namespace
	kubevirtci::generate_kubeconfig
	kubevirtci::create_external_cluster
	;;
"destroy-cluster")
	kubevirtci::destroy_cluster
	;;

"clean-cache")
	rm ${_default_clusterctl_path}
	rm ${_default_virtctl_path}
	rm -rf ${_default_tmp_path}
	;;
"help")
	kubevirtci::usage
	;;
*)
	echo "Error: Unknown kubevirtci command"
	echo ""
	kubevirtci::usage
	exit 1
	;;
esac