test(robot): add uninstallation check test case

ref: longhorn/longhorn#9222

Signed-off-by: Chris <chris.chien@suse.com>

e2e/Dockerfile

@@ -26,6 +26,14 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_
     curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar
 
 ADD e2e /e2e
+ADD pipelines/utilities/longhorn_ui.sh ./pipelines/utilities/longhorn_ui.sh
+ADD pipelines/utilities/create_longhorn_namespace.sh ./pipelines/utilities/create_longhorn_namespace.sh
+ADD pipelines/utilities/install_backupstores.sh ./pipelines/utilities/install_backupstores.sh
+ADD pipelines/utilities/longhorn_status.sh  ./pipelines/utilities/longhorn_status.sh
+ADD pipelines/utilities/longhorn_helm_chart.sh ./pipelines/utilities/longhorn_helm_chart.sh
+ADD pipelines/utilities/create_aws_secret.sh ./pipelines/utilities/create_aws_secret.sh
+ADD pipelines/utilities/longhorn_manifest.sh ./pipelines/utilities/longhorn_manifest.sh
+
 WORKDIR /e2e
 
 RUN pip install -r requirements.txt

e2e/keywords/backup.resource

@@ -26,3 +26,6 @@ Check volume ${volume_id} data is backup ${backup_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_name} =    get_backup_name    ${backup_id}
     check_restored_volume_checksum    ${volume_name}    ${backup_name}
+
+Check backup synced from backupstore
+    assert_all_backups_before_uninstall_exist    ${backups_before_uninstall}

e2e/keywords/longhorn.resource

@@ -4,6 +4,8 @@ Documentation       Longhorn Keywords
 Library             ../libs/keywords/instancemanager_keywords.py
 Library             ../libs/keywords/workload_keywords.py
 Library             ../libs/keywords/k8s_keywords.py
+Library             ../libs/keywords/longhorn_deploy_keywords.py
+Library             ../libs/keywords/backup_keywords.py
 
 *** Variables ***
 @{longhorn_workloads}
@@ -52,4 +54,15 @@ Check instance-manager pod is not running on drained node
 Check instance-manager pod is running on node ${node_id}
     ${node_name} =    get_node_by_index    ${node_id}
     ${pod} =     get_instance_manager_on_node    ${node_name}
-    Should Not Be Equal    ${pod}    ${None}
+    Should Not Be Equal    ${pod}    ${None}
+
+Uninstall Longhorn
+    ${backups_before_uninstall} =     list_all_backups
+    uninstall_longhorn_system
+    Set Test Variable    ${backups_before_uninstall}
+
+Check all Longhorn CRD removed
+    check_longhorn_crd_removed
+
+Install Longhorn
+    install_longhorn_system

e2e/keywords/volume.resource

@@ -266,3 +266,18 @@ Upgrade volume ${volume_id} engine to compatible engine image
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     upgrade_engine_image    ${volume_name}    ${compatible_engine_image_name}
     wait_for_engine_image_upgrade_completed    ${volume_name}    ${compatible_engine_image_name}
+Check volume ${volume_id} data is backup ${backup_id} created in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${current_checksum} =    get_volume_checksum    ${volume_name}
+    ${backup_data} =    get_backup_data_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    Should Be Equal    ${current_checksum}    ${backup_data}
+
+Create DR volume ${volume_id} from backup ${backup_id} in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    create_volume   ${volume_name}    frontend=    Standby=True    fromBackup=${backup_url}
+
+Wait for volume ${volume_id} restoration from backup ${backup_id} in another cluster completed
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_name} =    get_backup_name_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    wait_for_volume_restoration_completed    ${volume_name}    ${backup_name}

e2e/libs/backup/backup.py

@@ -21,12 +21,21 @@ def create(self, volume_name, backup_id):
     def get(self, backup_id, volume_name):
         return self.backup.get(backup_id, volume_name)
 
+    def get_from_list(self, backup_list, backup_id):
+        return self.backup.get_from_list(backup_list, backup_id)
+
     def get_backup_volume(self, volume_name):
         return self.backup.get_backup_volume(volume_name)
 
     def list(self, volume_name):
         return self.backup.list(volume_name)
 
+    def list_all(self):
+        return self.backup.list_all()
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        return self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall)
+
     def verify_no_error(self, volume_name):
         backup_volume = self.get_backup_volume(volume_name)
         assert not backup_volume['messages'], \

e2e/libs/backup/base.py

@@ -71,6 +71,10 @@ def wait_for_backup_completed(self, volume_name, snapshot_name):
     def list(self, volume_name):
         return NotImplemented
 
+    @abstractmethod
+    def list_all(self):
+        return NotImplemented
+
     @abstractmethod
     def delete(self, volume_name, backup_id):
         return NotImplemented

e2e/libs/backup/rest.py

@@ -2,6 +2,7 @@
 from utility.utility import logging
 from utility.utility import get_longhorn_client
 from utility.utility import get_retry_count_and_interval
+from utility.utility import get_all_crs
 from node_exec import NodeExec
 from volume import Rest as RestVolume
 from snapshot import Snapshot as RestSnapshot
@@ -51,6 +52,12 @@ def get(self, backup_id, volume_name):
                 return backup
         return None
 
+    def get_from_list(self, backup_list, backup_id):
+        for backup in backup_list["items"]:
+            if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
+                return backup
+        return None
+
     def get_by_snapshot(self, volume_name, snapshot_name):
         """
         look for a backup from snapshot on the backupstore
@@ -109,6 +116,35 @@ def list(self, volume_name):
             backup_volume = self.get_backup_volume(volume_name)
             return backup_volume.backupList().data
 
+    def list_all(self):
+        return get_all_crs(group="longhorn.io",
+                      version="v1beta2",
+                      namespace="longhorn-system",
+                      plural="backups",
+                      )
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        for i in range(self.retry_count):
+            time.sleep(self.retry_interval)
+            try:
+                current_backups = self.list_all()
+                assert len(current_backups["items"]) == len(backups_before_uninstall["items"])
+                break
+            except Exception as e:
+                logging(f"{e}")
+                continue
+
+        target_backup_names = {(item["metadata"]["name"]) for item in backups_before_uninstall["items"]}
+        for item in current_backups["items"]:
+            backup_name = item["metadata"]["name"]
+            assert backup_name in target_backup_names, f"Error: Backup {backup_name} not found in {target_backup_names}"
+
+            for i in range(self.retry_count):
+                time.sleep(self.retry_interval)
+                volume_name = item["status"]["volumeName"]
+                if self.get_backup_volume(volume_name) != None:
+                    break
+
     def delete(self, volume_name, backup_id):
         return NotImplemented
 

e2e/libs/k8s/k8s.py

@@ -3,6 +3,7 @@
 import asyncio
 import os
 from kubernetes import client
+from kubernetes.client.rest import ApiException
 from workload.pod import create_pod
 from workload.pod import delete_pod
 from workload.pod import new_pod_manifest
@@ -94,3 +95,78 @@ def check_instance_manager_pdb_not_exist(instance_manager):
     exec_cmd = ["kubectl", "get", "pdb", "-n", "longhorn-system"]
     res = subprocess_exec_cmd(exec_cmd)
     assert instance_manager not in res.decode('utf-8')
+def wait_namespaced_job_complete(job_label, namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.BatchV1Api()
+    for i in range(retry_count):
+        target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label)
+        if len(target_job.items) > 0:
+            running_jobs = []
+            for job in target_job.items:
+                conditions = job.status.conditions
+                if conditions:
+                    for condition in conditions:
+                        logging(f"{condition.type}  {condition.status}")
+                        if condition.type == "Complete" and condition.status == "True":
+                            print(f"Job {job.metadata.name} is complete.")
+                            running_jobs.append(job)
+                            break
+            if len(running_jobs) == len(target_job.items):
+                return
+
+        logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...")
+        time.sleep(retry_interval)
+
+    assert False, 'Job not complete'
+
+def wait_namespace_terminated(namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.CoreV1Api()
+    for i in range(retry_count):
+        try:
+            target_namespace = api.read_namespace(name=namespace)
+            target_namespace_status = target_namespace.status.phase
+            logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...")
+        except ApiException as e:
+            if e.status == 404:
+                logging(f"Namespace {namespace} successfully terminated.")
+                return
+            else:
+                logging(f"Error while fetching namespace {namespace} status: {e}")
+
+        time.sleep(retry_interval)
+
+    assert False, f'namespace {target_namespace.metadata.name} not terminated'
+
+def get_all_custom_resources():
+    api = client.ApiextensionsV1Api()
+    crds = api.list_custom_resource_definition()
+
+    return crds
+
+def get_pod_logs(namespace, pod_label):
+    api = client.CoreV1Api()
+    logs= ""
+    try:
+        pods = api.list_namespaced_pod(namespace, label_selector=pod_label)
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace)
+    except client.exceptions.ApiException as e:
+        logging(f"Exception when calling CoreV1Api: {e}")
+
+    logging(f'{logs}')
+    return logs
+
+def list_namespace_pods(namespace):
+    v1 = client.CoreV1Api()
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    return pods
+
+def delete_namespace(namespace):
+    api = client.CoreV1Api()
+    try:
+        api.delete_namespace(name=namespace)
+    except ApiException as e:
+        assert e.status == 404

e2e/libs/keywords/backup_keywords.py

@@ -20,6 +20,18 @@ def get_backup_name(self, backup_id, volume_name=None):
     def get_backup_url(self, backup_id, volume_name=None):
         return self.backup.get(backup_id, volume_name).url
 
+    def get_backup_url_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup["status"]["url"]
+
+    def get_backup_data_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['annotations']["test.longhorn.io/data-checksum"]
+
+    def get_backup_name_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['name']
+
     def delete_backup_volume(self, volume_name):
         return self.backup.delete_backup_volume(volume_name)
 
@@ -31,3 +43,10 @@ def cleanup_backups(self):
         if get_backupstore():
             self.backup.cleanup_system_backups()
             self.backup.cleanup_backup_volumes()
+
+    def list_all_backups(self):
+        all_backups = self.backup.list_all()
+        return all_backups
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall)

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -0,0 +1,14 @@
+from longhorn_deploy import LonghornDeploy
+class longhorn_deploy_keywords:
+
+    def __init__(self):
+        self.longhorn = LonghornDeploy()
+
+    def uninstall_longhorn_system(self):
+        self.longhorn.uninstall()
+
+    def check_longhorn_crd_removed(self):
+        self.longhorn.check_longhorn_crd_removed()
+
+    def install_longhorn_system(self):
+        self.longhorn.install()

e2e/libs/keywords/volume_keywords.py

@@ -311,3 +311,6 @@ def upgrade_engine_image(self, volume_name, engine_image_name):
 
     def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name):
         self.volume.wait_for_engine_image_upgrade_completed(volume_name, engine_image_name)
+
+    def get_volume_checksum(self, volume_name):
+        return self.volume.get_checksum(volume_name)

e2e/libs/longhorn_deploy/__init__.py

@@ -0,0 +1,3 @@
+from longhorn_deploy.longhorn_deploy import LonghornDeploy
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart

e2e/libs/longhorn_deploy/base.py

@@ -0,0 +1,45 @@
+from abc import ABC, abstractmethod
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL
+from utility.constant import LONGHORN_INSTALL_SCRIPT_PATH
+from utility.constant import LONGHORN_INSTALL_TIMEOUT
+import subprocess
+import os
+from utility.utility import get_retry_count_and_interval
+from utility.utility import logging
+
+class Base(ABC):
+
+    def __init__(self):
+        self.retry_count, self.retry_interval = get_retry_count_and_interval()
+
+    @abstractmethod
+    def install(self):
+        return NotImplemented
+
+    @abstractmethod
+    def uninstall(self, longhorn_branch=None):
+        return NotImplemented
+
+    def check_longhorn_crd_removed(self):
+        all_crd = k8s.get_all_custom_resources()
+        for crd in all_crd.items:
+            assert "longhorn.io" not in crd.metadata.name
+
+    def check_longhorn_uninstall_pod_log(self):
+        logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
+        assert "error" not in logs
+        assert "level=fatal" not in logs
+
+    def install_longhorn(self):
+        current_path=os.getcwd()
+        full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH)
+
+        try:
+            output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT)
+            logging(output)
+        except subprocess.CalledProcessError as e:
+            logging(f"Error: {e.stderr}")
+        except subprocess.TimeoutExpired as e:
+            logging(f"Command timed out after {e.timeout} seconds")

e2e/libs/longhorn_deploy/longhorn_deploy.py

@@ -0,0 +1,24 @@
+from longhorn_deploy.base import Base
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart
+import os
+
+class LonghornDeploy(Base):
+
+    _method = os.getenv("LONGHORN_INSTALL_METHOD", "manifest")
+
+    def __init__(self):
+
+        if self._method == "manifest":
+            self.longhorn = LonghornKubectl()
+        elif self._method == "helm":
+            self.longhorn = LonghornHelmChart()
+
+    def uninstall(self):
+        return self.longhorn.uninstall()
+
+    def check_longhorn_crd_removed(self):
+        return self.longhorn.check_longhorn_crd_removed()
+
+    def install(self):
+        return self.longhorn.install()

e2e/libs/longhorn_deploy/longhorn_helm_chart.py

@@ -0,0 +1,23 @@
+from longhorn_deploy.base import Base
+from node import Node
+from node_exec import NodeExec
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+
+import os
+
+class LonghornHelmChart(Base):
+
+    def uninstall(self):
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        cmd = f'export KUBECONFIG={os.getenv("KUBECONFIG")} && helm uninstall longhorn -n {LONGHORN_NAMESPACE}'
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply helm uninstall command failed"
+
+        k8s.delete_namespace(namespace=LONGHORN_NAMESPACE)
+        k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
+
+    def install(self):
+        self.install_longhorn()