test(robot): add uninstallation check test case

ref: longhorn/longhorn#9222

Signed-off-by: Chris <chris.chien@suse.com>

e2e/keywords/backup.resource

@@ -26,3 +26,7 @@ Check volume ${volume_id} data is backup ${backup_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_name} =    get_backup_name    ${backup_id}
     check_restored_volume_checksum    ${volume_name}    ${backup_name}
+
+Check backup synced from backupstore
+    ${backups} =    list_all_backups
+    assert_all_backups_exist    ${backups}    ${backups_before_uninstall}

e2e/keywords/longhorn.resource

@@ -3,6 +3,8 @@ Documentation       Longhorn Keywords
 
 Library             ../libs/keywords/instancemanager_keywords.py
 Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/longhorn_deploy_keywords.py
+Library             ../libs/keywords/backup_keywords.py
 
 *** Variables ***
 @{longhorn_workloads}
@@ -43,3 +45,14 @@ Check Longhorn workload pods ${condition} annotated with ${key}
     Run Keyword IF    '${condition}' == 'not'    Should Not Be True    ${is_annotated}
     ...    ELSE IF    '${condition}' == 'is'    Should Be True    ${is_annotated}
     ...    ELSE    Fail    Invalid condition ${condition}
+
+Uninstall Longhorn ${LONGHORN_BRANCH}
+    ${backups_before_uninstall} =     list_all_backups
+    uninstall_longhorn    ${LONGHORN_BRANCH}
+    Set Test Variable    ${backups_before_uninstall}
+
+Check all Longhorn CRD removed
+    check_longhorn_crd_removed
+
+Install Longhorn ${LONGHORN_BRANCH}
+    install_longhorn    ${LONGHORN_BRANCH}

e2e/libs/backup/backup.py

@@ -27,6 +27,12 @@ def get_backup_volume(self, volume_name):
     def list(self, volume_name):
         return self.backup.list(volume_name)
 
+    def list_all(self):
+        return self.backup.list_all()
+
+    def assert_all_backups_exist(self, source_backups, target_backup):
+        return self.backup.assert_all_backups_exist(source_backups, target_backup)
+
     def verify_no_error(self, volume_name):
         backup_volume = self.get_backup_volume(volume_name)
         assert not backup_volume['messages'], \

e2e/libs/backup/base.py

@@ -71,6 +71,10 @@ def wait_for_backup_completed(self, volume_name, snapshot_name):
     def list(self, volume_name):
         return NotImplemented
 
+    @abstractmethod
+    def list_all(self):
+        return NotImplemented
+
     @abstractmethod
     def delete(self, volume_name, backup_id):
         return NotImplemented

e2e/libs/backup/rest.py

@@ -2,6 +2,7 @@
 from utility.utility import logging
 from utility.utility import get_longhorn_client
 from utility.utility import get_retry_count_and_interval
+from utility.utility import get_all_crs
 from node_exec import NodeExec
 from volume import Rest as RestVolume
 from snapshot import Snapshot as RestSnapshot
@@ -109,6 +110,20 @@ def list(self, volume_name):
             backup_volume = self.get_backup_volume(volume_name)
             return backup_volume.backupList().data
 
+    def list_all(self):
+        return get_all_crs(group="longhorn.io",
+                      version="v1beta2",
+                      namespace="longhorn-system",
+                      plural="backups",
+                      )
+
+    def assert_all_backups_exist(self, source_backups, target_backup):
+        target_backup_names = {(item["metadata"]["name"]) for item in target_backup["items"]}
+
+        for item in source_backups["items"]:            
+            backup_name = item["metadata"]["name"]
+            assert backup_name in target_backup_names, f"Error: Backup {backup_name} not found in {target_backup_names}"
+
     def delete(self, volume_name, backup_id):
         return NotImplemented
 

e2e/libs/k8s/k8s.py

@@ -2,6 +2,7 @@
 import subprocess
 import asyncio
 from kubernetes import client
+from kubernetes.client.rest import ApiException
 from workload.pod import create_pod
 from workload.pod import delete_pod
 from workload.pod import new_pod_manifest
@@ -71,3 +72,84 @@ def wait_all_pods_evicted(node_name):
         time.sleep(retry_interval)
 
     assert evicted, 'failed to evict pods'
+
+def wait_namespaced_job_complete(job_label, namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.BatchV1Api()
+    for i in range(retry_count):
+        target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label)        
+        if len(target_job.items) > 0:
+            running_jobs = []
+            for job in target_job.items:
+                conditions = job.status.conditions
+                if conditions:
+                    logging(f"=== {conditions}")
+                    for condition in conditions:
+                        logging(f"{condition.type}  {condition.status}")
+                        if condition.type == "Complete" and condition.status == "True":
+                            print(f"Job {job.metadata.name} is complete.")
+                            running_jobs.append(job)
+                            break
+            if len(running_jobs) == len(target_job.items):
+                return
+
+        logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...")
+        time.sleep(retry_interval)
+
+    assert False, 'Job not complete'
+
+def wait_namespace_terminated(namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.CoreV1Api()
+    for i in range(retry_count):
+        try:
+            target_namespace = api.read_namespace(name=namespace)
+            target_namespace_status = target_namespace.status.phase
+            logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...")
+        except ApiException:
+            return
+
+        time.sleep(retry_interval)
+
+    assert False, f'namespace {target_namespace.metadata.name} not terminated'
+
+def get_all_custom_resources():
+    api = client.ApiextensionsV1Api()
+    crds = api.list_custom_resource_definition()
+
+    return crds
+
+def delete_namespace(namespace):
+    api = client.CoreV1Api()
+    try:
+        api.delete_namespace(name=namespace)
+    except ApiException as e:
+        assert e.status == 404
+
+def create_namespace(namespace):
+    api = client.CoreV1Api()
+    namespace = client.V1Namespace(
+        metadata=client.V1ObjectMeta(name=namespace)
+    )
+
+    api.create_namespace(body=namespace)
+
+def get_pod_logs(namespace, pod_label):
+    api = client.CoreV1Api()
+    logs= ""
+    try:
+        pods = api.list_namespaced_pod(namespace, label_selector=pod_label)
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace)                
+    except client.exceptions.ApiException as e:
+        logging(f"Exception when calling CoreV1Api: {e}")
+
+    logging(f'{logs}')
+    return logs
+
+def list_namespace_pods(namespace):
+    v1 = client.CoreV1Api()
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    return pods

e2e/libs/keywords/backup_keywords.py

@@ -31,3 +31,10 @@ def cleanup_backups(self):
         if get_backupstore():
             self.backup.cleanup_system_backups()
             self.backup.cleanup_backup_volumes()
+
+    def list_all_backups(self):
+        all_backups = self.backup.list_all()
+        return all_backups
+
+    def assert_all_backups_exist(self, source_backups, target_backup):
+        self.backup.assert_all_backups_exist(source_backups, target_backup)

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -0,0 +1,14 @@
+from longhorn_deploy import LonghornDeploy
+class longhorn_deploy_keywords:
+
+    def __init__(self):
+        self.longhorn = LonghornDeploy()
+
+    def uninstall_longhorn(self, longhorn_branch):        
+        self.longhorn.uninstall(longhorn_branch)
+
+    def check_longhorn_crd_removed(self):
+        self.longhorn.check_longhorn_crd_removed()
+
+    def install_longhorn(self, longhorn_branch):
+        self.longhorn.install(longhorn_branch)

e2e/libs/keywords/uninstallation_keywords.py

@@ -0,0 +1,12 @@
+from uninstallation import Uninstallation
+
+class uninstallation_keywords:
+
+    def __init__(self):
+        self.uninstallation = Uninstallation()
+
+    def uninstall_longhorn(self, longhor_branch, install_method):
+        self.uninstallation.uninstall_longhorn(longhor_branch, install_method)
+
+    def check_longhorn_crd_removed(self):
+        self.uninstallation.check_longhorn_crd_removed()

e2e/libs/longhorn.py

@@ -941,3 +941,4 @@ def _get_timeout(timeout):
 
 def from_env(prefix='CATTLE_', **kw):
     return gdapi_from_env(prefix=prefix, factory=Client, **kw)
+

e2e/libs/longhorn_deploy/__init__.py

@@ -0,0 +1,3 @@
+from longhorn_deploy.longhorn_deploy import LonghornDeploy
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart

e2e/libs/longhorn_deploy/base.py

@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from k8s import k8s
+from kubernetes.client.rest import ApiException
+from utility.constant import LONGHORN_NAMESPACE
+from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL
+import time
+from utility.utility import logging
+from node_exec import NodeExec
+from node import Node
+
+class Base(ABC):
+
+    @abstractmethod
+    def install(self):
+        return NotImplemented
+
+    @abstractmethod
+    def uninstall(self, longhorn_branch=None):
+        return NotImplemented
+
+    def check_longhorn_crd_removed(self):
+        all_crd = k8s.get_all_custom_resources()
+        for crd in all_crd.items:            
+            assert "longhorn.io" not in crd.metadata.name
+
+    def check_longhorn_uninstall_pod_log(self):
+        logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
+        assert "error" not in logs
+        assert "level=fatal" not in logs
+
+    def wait_longhorn_status_running(self):
+        RETRY_COUNTS = 10  
+        RETRY_INTERVAL = 60        
+        retries = 0
+
+        while True:
+            try:
+                pods = k8s.list_namespace_pods(LONGHORN_NAMESPACE)
+
+                csi_pods = [pod for pod in pods.items if 'csi-' in pod.metadata.name]
+                engine_image_pods = [pod for pod in pods.items if 'engine-image-' in pod.metadata.name]                    
+                non_running_pods = [pod for pod in pods.items if pod.status.phase != 'Running']
+
+                if csi_pods and engine_image_pods and not non_running_pods:
+                    logging(f"Longhorn is fully running.")
+                    break
+                else:
+                    logging("Longhorn is still installing ... re-checking in 1m")
+            except ApiException as e:
+                logging("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
+
+            time.sleep(RETRY_INTERVAL)
+            retries += 1
+
+            if retries == RETRY_COUNTS:
+                logging("Error: longhorn installation timeout")
+                return 1  
+
+    def expose_longhorn_ui(self):        
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        cmd = "kubectl expose --type=NodePort deployment longhorn-ui -n longhorn-system "\
+              "--port 8000 --name longhorn-ui-nodeport "\
+              "--overrides '{\"apiVersion\": \"v1\",\"spec\":{\"ports\": [{\"port\":8000,\"protocol\":\"TCP\",\"targetPort\":8000,\"nodePort\":30000}]}}'"
+
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "expose longhorn-ui failed"        

e2e/libs/longhorn_deploy/longhorn_deploy.py

@@ -0,0 +1,24 @@
+from longhorn_deploy.base import Base
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart
+import os
+
+class LonghornDeploy(Base):
+
+    _method = os.getenv("INSTALL_METHOD")
+
+    def __init__(self):
+
+        if self._method == "kubectl":        
+            self.longhorn = LonghornKubectl()
+        elif self._method == "helm":            
+            self.longhorn = LonghornHelmChart()
+
+    def uninstall(self, longhorn_branch):
+        return self.longhorn.uninstall(longhorn_branch)
+
+    def check_longhorn_crd_removed(self):
+        return self.longhorn.check_longhorn_crd_removed()
+
+    def install(self, longhorn_branch):
+        return self.longhorn.install(longhorn_branch)

e2e/libs/longhorn_deploy/longhorn_helm_chart.py

@@ -0,0 +1,40 @@
+from longhorn_deploy.base import Base
+from node import Node
+from node_exec import NodeExec
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+import os
+
+class LonghornHelmChart(Base):
+
+    def uninstall(self, longhorn_branch):        
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        cmd = f"export KUBECONFIG={os.getenv("KUBECONFIG")} && helm uninstall longhorn -n {LONGHORN_NAMESPACE}"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm uninstall command failed"        
+
+        k8s.delete_namespace(namespace=LONGHORN_NAMESPACE)
+        k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
+
+    def install(self, longhorn_branch):        
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        k8s.create_namespace(LONGHORN_NAMESPACE)
+
+        cmd = f"export KUBECONFIG={os.getenv("KUBECONFIG")} && helm repo add longhorn https://charts.longhorn.io"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm repo add longhorn command failed"
+
+        cmd = f"export KUBECONFIG={os.getenv("KUBECONFIG")} && helm repo update"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm repo update command failed"
+
+        cmd = f"export KUBECONFIG={os.getenv("KUBECONFIG")} && helm install longhorn longhorn/longhorn --version {longhorn_branch} -n {LONGHORN_NAMESPACE}"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm install command failed"
+
+        self.wait_longhorn_status_running()
+        self.expose_longhorn_ui()