test(robot): add uninstallation check test case

ref: longhorn/longhorn#9222

Signed-off-by: Chris <chris.chien@suse.com>

e2e/keywords/backup.resource

@@ -26,3 +26,6 @@ Check volume ${volume_id} data is backup ${backup_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_name} =    get_backup_name    ${backup_id}
     check_restored_volume_checksum    ${volume_name}    ${backup_name}
+
+Check backup synced from backupstore
+    assert_all_backups_before_uninstall_exist    ${backups_before_uninstall}

e2e/keywords/longhorn.resource

@@ -3,6 +3,8 @@ Documentation       Longhorn Keywords
 
 Library             ../libs/keywords/instancemanager_keywords.py
 Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/longhorn_deploy_keywords.py
+Library             ../libs/keywords/backup_keywords.py
 
 *** Variables ***
 @{longhorn_workloads}
@@ -43,3 +45,14 @@ Check Longhorn workload pods ${condition} annotated with ${key}
     Run Keyword IF    '${condition}' == 'not'    Should Not Be True    ${is_annotated}
     ...    ELSE IF    '${condition}' == 'is'    Should Be True    ${is_annotated}
     ...    ELSE    Fail    Invalid condition ${condition}
+
+Uninstall Longhorn
+    ${backups_before_uninstall} =     list_all_backups
+    uninstall_longhorn_system
+    Set Test Variable    ${backups_before_uninstall}
+
+Check all Longhorn CRD removed
+    check_longhorn_crd_removed
+
+Install Longhorn
+    install_longhorn_system

e2e/keywords/volume.resource

@@ -260,3 +260,18 @@ Upgrade volume ${volume_id} engine to compatible engine image
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     upgrade_engine_image    ${volume_name}    ${compatible_engine_image_name}
     wait_for_engine_image_upgrade_completed    ${volume_name}    ${compatible_engine_image_name}
+Check volume ${volume_id} data is backup ${backup_id} created in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${current_checksum} =    get_volume_checksum    ${volume_name}
+    ${backup_data} =    get_backup_data_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    Should Be Equal    ${current_checksum}    ${backup_data}
+
+Create DR volume ${volume_id} from backup ${backup_id} in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    create_volume   ${volume_name}    frontend=    Standby=True    fromBackup=${backup_url}
+
+Wait for volume ${volume_id} restoration from backup ${backup_id} in another cluster completed
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_name} =    get_backup_name_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    wait_for_volume_restoration_completed    ${volume_name}    ${backup_name}

e2e/libs/backup/backup.py

@@ -21,12 +21,21 @@ def create(self, volume_name, backup_id):
     def get(self, backup_id, volume_name):
         return self.backup.get(backup_id, volume_name)
 
+    def get_from_list(self, backup_list, backup_id):
+        return self.backup.get_from_list(backup_list, backup_id)
+
     def get_backup_volume(self, volume_name):
         return self.backup.get_backup_volume(volume_name)
 
     def list(self, volume_name):
         return self.backup.list(volume_name)
 
+    def list_all(self):
+        return self.backup.list_all()
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        return self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall)
+
     def verify_no_error(self, volume_name):
         backup_volume = self.get_backup_volume(volume_name)
         assert not backup_volume['messages'], \

e2e/libs/backup/base.py

@@ -71,6 +71,10 @@ def wait_for_backup_completed(self, volume_name, snapshot_name):
     def list(self, volume_name):
         return NotImplemented
 
+    @abstractmethod
+    def list_all(self):
+        return NotImplemented
+
     @abstractmethod
     def delete(self, volume_name, backup_id):
         return NotImplemented

e2e/libs/backup/rest.py

@@ -2,6 +2,7 @@
 from utility.utility import logging
 from utility.utility import get_longhorn_client
 from utility.utility import get_retry_count_and_interval
+from utility.utility import get_all_crs
 from node_exec import NodeExec
 from volume import Rest as RestVolume
 from snapshot import Snapshot as RestSnapshot
@@ -51,6 +52,12 @@ def get(self, backup_id, volume_name):
                 return backup
         return None
 
+    def get_from_list(self, backup_list, backup_id):
+        for backup in backup_list["items"]:
+            if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
+                return backup
+        return None
+
     def get_by_snapshot(self, volume_name, snapshot_name):
         """
         look for a backup from snapshot on the backupstore
@@ -109,6 +116,35 @@ def list(self, volume_name):
             backup_volume = self.get_backup_volume(volume_name)
             return backup_volume.backupList().data
 
+    def list_all(self):
+        return get_all_crs(group="longhorn.io",
+                      version="v1beta2",
+                      namespace="longhorn-system",
+                      plural="backups",
+                      )
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        for i in range(self.retry_count):
+            time.sleep(self.retry_interval)
+            try:
+                current_backups = self.list_all()
+                assert len(current_backups["items"]) == len(backups_before_uninstall["items"])
+                break
+            except Exception as e:
+                logging(f"{e}")
+                continue
+
+        target_backup_names = {(item["metadata"]["name"]) for item in backups_before_uninstall["items"]}
+        for item in current_backups["items"]:
+            backup_name = item["metadata"]["name"]
+            assert backup_name in target_backup_names, f"Error: Backup {backup_name} not found in {target_backup_names}"
+
+            for i in range(self.retry_count):
+                time.sleep(self.retry_interval)
+                volume_name = item["status"]["volumeName"]
+                if self.get_backup_volume(volume_name) != None:
+                    break
+
     def delete(self, volume_name, backup_id):
         return NotImplemented
 

e2e/libs/k8s/k8s.py

@@ -2,6 +2,7 @@
 import subprocess
 import asyncio
 from kubernetes import client
+from kubernetes.client.rest import ApiException
 from workload.pod import create_pod
 from workload.pod import delete_pod
 from workload.pod import new_pod_manifest
@@ -71,3 +72,74 @@ def wait_all_pods_evicted(node_name):
         time.sleep(retry_interval)
 
     assert evicted, 'failed to evict pods'
+
+def wait_namespaced_job_complete(job_label, namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.BatchV1Api()
+    for i in range(retry_count):
+        target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label)
+        if len(target_job.items) > 0:
+            running_jobs = []
+            for job in target_job.items:
+                conditions = job.status.conditions
+                if conditions:
+                    for condition in conditions:
+                        logging(f"{condition.type}  {condition.status}")
+                        if condition.type == "Complete" and condition.status == "True":
+                            print(f"Job {job.metadata.name} is complete.")
+                            running_jobs.append(job)
+                            break
+            if len(running_jobs) == len(target_job.items):
+                return
+
+        logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...")
+        time.sleep(retry_interval)
+
+    assert False, 'Job not complete'
+
+def wait_namespace_terminated(namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.CoreV1Api()
+    for i in range(retry_count):
+        try:
+            target_namespace = api.read_namespace(name=namespace)
+            target_namespace_status = target_namespace.status.phase
+            logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...")
+        except ApiException:
+            return
+        time.sleep(retry_interval)
+
+    assert False, f'namespace {target_namespace.metadata.name} not terminated'
+
+def get_all_custom_resources():
+    api = client.ApiextensionsV1Api()
+    crds = api.list_custom_resource_definition()
+
+    return crds
+
+def get_pod_logs(namespace, pod_label):
+    api = client.CoreV1Api()
+    logs= ""
+    try:
+        pods = api.list_namespaced_pod(namespace, label_selector=pod_label)
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace)
+    except client.exceptions.ApiException as e:
+        logging(f"Exception when calling CoreV1Api: {e}")
+
+    logging(f'{logs}')
+    return logs
+
+def list_namespace_pods(namespace):
+    v1 = client.CoreV1Api()
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    return pods
+
+def delete_namespace(namespace):
+    api = client.CoreV1Api()
+    try:
+        api.delete_namespace(name=namespace)
+    except ApiException as e:
+        assert e.status == 404

e2e/libs/keywords/backup_keywords.py

@@ -20,6 +20,18 @@ def get_backup_name(self, backup_id, volume_name=None):
     def get_backup_url(self, backup_id, volume_name=None):
         return self.backup.get(backup_id, volume_name).url
 
+    def get_backup_url_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup["status"]["url"]
+
+    def get_backup_data_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['annotations']["test.longhorn.io/data-checksum"]
+
+    def get_backup_name_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['name']
+
     def delete_backup_volume(self, volume_name):
         return self.backup.delete_backup_volume(volume_name)
 
@@ -31,3 +43,10 @@ def cleanup_backups(self):
         if get_backupstore():
             self.backup.cleanup_system_backups()
             self.backup.cleanup_backup_volumes()
+
+    def list_all_backups(self):
+        all_backups = self.backup.list_all()
+        return all_backups
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall)

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -0,0 +1,14 @@
+from longhorn_deploy import LonghornDeploy
+class longhorn_deploy_keywords:
+
+    def __init__(self):
+        self.longhorn = LonghornDeploy()
+
+    def uninstall_longhorn_system(self):
+        self.longhorn.uninstall()
+
+    def check_longhorn_crd_removed(self):
+        self.longhorn.check_longhorn_crd_removed()
+
+    def install_longhorn_system(self):
+        self.longhorn.install()

e2e/libs/keywords/volume_keywords.py

@@ -311,3 +311,6 @@ def upgrade_engine_image(self, volume_name, engine_image_name):
 
     def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
         self.volume.wait_for_engine_image_upgrade_completed(volume_name, engine_image_name)
+
+    def get_volume_checksum(self, volume_name):
+        return self.volume.get_checksum(volume_name)

e2e/libs/longhorn_deploy/__init__.py

@@ -0,0 +1,3 @@
+from longhorn_deploy.longhorn_deploy import LonghornDeploy
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart

e2e/libs/longhorn_deploy/base.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL
+from utility.constant import LONGHORN_INSTALL_SCRIPT_PATH
+from utility.constant import LONGHORN_INSTALL_TIMEOUT
+import time
+import subprocess
+import os
+from utility.utility import logging
+from utility.utility import get_retry_count_and_interval
+
+class Base(ABC):
+
+    def __init__(self):
+        self.retry_count, self.retry_interval = get_retry_count_and_interval()
+
+    @abstractmethod
+    def install(self):
+        return NotImplemented
+
+    @abstractmethod
+    def uninstall(self, longhorn_branch=None):
+        return NotImplemented
+
+    def check_longhorn_crd_removed(self):
+        all_crd = k8s.get_all_custom_resources()
+        for crd in all_crd.items:
+            assert "longhorn.io" not in crd.metadata.name
+
+    def check_longhorn_uninstall_pod_log(self):
+        logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
+        assert "error" not in logs
+        assert "level=fatal" not in logs
+
+    def install_longhorn(self):
+        current_path=os.getcwd()
+        full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH)
+        process = subprocess.Popen(['bash', full_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        for i in range(LONGHORN_INSTALL_TIMEOUT):
+            if process.poll() is not None:
+                return
+            else:
+                output = process.stdout.readline()
+                logging(f"{output.strip()}")
+            time.sleep(self.retry_interval)
+
+        assert False, f"Error occurred while installing longhorn"

e2e/libs/longhorn_deploy/longhorn_deploy.py

@@ -0,0 +1,22 @@
+from longhorn_deploy.base import Base
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart
+import os
+
+class LonghornDeploy(Base):
+
+    def __init__(self):
+
+        if self._method == "manifest":
+            self.longhorn = LonghornKubectl()
+        elif self._method == "helm":
+            self.longhorn = LonghornHelmChart()
+
+    def uninstall(self):
+        return self.longhorn.uninstall()
+
+    def check_longhorn_crd_removed(self):
+        return self.longhorn.check_longhorn_crd_removed()
+
+    def install(self):
+        return self.longhorn.install()

e2e/libs/longhorn_deploy/longhorn_helm_chart.py

@@ -0,0 +1,23 @@
+from longhorn_deploy.base import Base
+from node import Node
+from node_exec import NodeExec
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+
+import os
+
+class LonghornHelmChart(Base):
+
+    def uninstall(self):
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        cmd = f'export KUBECONFIG={os.getenv("KUBECONFIG")} && helm uninstall longhorn -n {LONGHORN_NAMESPACE}'
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm uninstall command failed"
+
+        k8s.delete_namespace(namespace=LONGHORN_NAMESPACE)
+        k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
+
+    def install(self):
+        self.install_longhorn()