test(robot): add uninstallation check test case

ref: longhorn/longhorn#9222

Signed-off-by: Chris <chris.chien@suse.com>

e2e/keywords/backup.resource

@@ -26,3 +26,6 @@ Check volume ${volume_id} data is backup ${backup_id}
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     ${backup_name} =    get_backup_name    ${backup_id}
     check_restored_volume_checksum    ${volume_name}    ${backup_name}
+
+Check backup synced from backupstore    
+    assert_all_backups_exist    ${backups_before_uninstall}    

e2e/keywords/common.resource

@@ -30,7 +30,7 @@ Set test environment
     update_setting    v2-data-engine    true
     ${worker_nodes}=    get_worker_nodes
     FOR    ${worker_node}    IN    @{worker_nodes}
-        add_disk    ${worker_node}    block    /dev/xvdh
+        add_disk    ${worker_node}    block    /dev/loop3
     END
 
 Cleanup test resources

e2e/keywords/longhorn.resource

@@ -3,6 +3,8 @@ Documentation       Longhorn Keywords
 
 Library             ../libs/keywords/instancemanager_keywords.py
 Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/longhorn_deploy_keywords.py
+Library             ../libs/keywords/backup_keywords.py
 
 *** Variables ***
 @{longhorn_workloads}
@@ -43,3 +45,14 @@ Check Longhorn workload pods ${condition} annotated with ${key}
     Run Keyword IF    '${condition}' == 'not'    Should Not Be True    ${is_annotated}
     ...    ELSE IF    '${condition}' == 'is'    Should Be True    ${is_annotated}
     ...    ELSE    Fail    Invalid condition ${condition}
+
+Uninstall Longhorn ${LONGHORN_BRANCH}
+    ${backups_before_uninstall} =     list_all_backups
+    uninstall_longhorn    ${LONGHORN_BRANCH}
+    Set Test Variable    ${backups_before_uninstall}
+
+Check all Longhorn CRD removed
+    check_longhorn_crd_removed
+
+Install Longhorn ${LONGHORN_BRANCH}
+    install_longhorn    ${LONGHORN_BRANCH}

e2e/keywords/volume.resource

@@ -254,3 +254,19 @@ Record volume ${volume_id} replica names
 Check volume ${volume_id} replica names are as recorded
     ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
     check_volume_replica_names_recorded    ${volume_name}
+
+Check volume ${volume_id} data is correct in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${current_checksum} =    get_volume_checksum    ${volume_name}
+    ${backup_data} =    get_backup_data_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    Should Be Equal    ${current_checksum}    ${backup_data}
+
+Create DR volume ${volume_id} from backup ${backup_id} in another cluster
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_url} =    get_backup_url_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    create_volume   ${volume_name}    frontend=    Standby=True    fromBackup=${backup_url}
+
+Wait for volume ${volume_id} restoration from backup ${backup_id} in another cluster completed
+    ${volume_name} =    generate_name_with_suffix    volume    ${volume_id}
+    ${backup_name} =    get_backup_name_from_backup_list    ${backups_before_uninstall}    ${backup_id}
+    wait_for_volume_restoration_completed    ${volume_name}    ${backup_name}

e2e/libs/backup/backup.py

@@ -21,12 +21,21 @@ def create(self, volume_name, backup_id):
     def get(self, backup_id, volume_name):
         return self.backup.get(backup_id, volume_name)
 
+    def get_from_list(self, backup_list, backup_id):
+        return self.backup.get_from_list(backup_list, backup_id)
+
     def get_backup_volume(self, volume_name):
         return self.backup.get_backup_volume(volume_name)
 
     def list(self, volume_name):
         return self.backup.list(volume_name)
 
+    def list_all(self):
+        return self.backup.list_all()
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        return self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall)
+
     def verify_no_error(self, volume_name):
         backup_volume = self.get_backup_volume(volume_name)
         assert not backup_volume['messages'], \

e2e/libs/backup/base.py

@@ -71,6 +71,10 @@ def wait_for_backup_completed(self, volume_name, snapshot_name):
     def list(self, volume_name):
         return NotImplemented
 
+    @abstractmethod
+    def list_all(self):
+        return NotImplemented
+
     @abstractmethod
     def delete(self, volume_name, backup_id):
         return NotImplemented

e2e/libs/backup/rest.py

@@ -2,6 +2,7 @@
 from utility.utility import logging
 from utility.utility import get_longhorn_client
 from utility.utility import get_retry_count_and_interval
+from utility.utility import get_all_crs
 from node_exec import NodeExec
 from volume import Rest as RestVolume
 from snapshot import Snapshot as RestSnapshot
@@ -51,6 +52,16 @@ def get(self, backup_id, volume_name):
                 return backup
         return None
 
+    def get_from_list(self, backup_list, backup_id):
+        for backup in backup_list["items"]:
+            logging(f"===========")
+            logging(f"{backup_id}")
+            logging(f"{backup}")
+            logging(f"backup['metadata']['annotations']['test.longhorn.io/backup-id']")
+            if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id:
+                return backup
+        return None
+
     def get_by_snapshot(self, volume_name, snapshot_name):
         """
         look for a backup from snapshot on the backupstore
@@ -109,6 +120,34 @@ def list(self, volume_name):
             backup_volume = self.get_backup_volume(volume_name)
             return backup_volume.backupList().data
 
+    def list_all(self):
+        return get_all_crs(group="longhorn.io",
+                      version="v1beta2",
+                      namespace="longhorn-system",
+                      plural="backups",
+                      )
+
+    def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall):
+        for i in range(self.retry_count):            
+            try:
+                current_backups = self.list_all()
+                assert len(current_backups["items"] == len(backups_before_uninstall["items"]))
+                break
+            except:
+                continue
+
+        target_backup_names = {(item["metadata"]["name"]) for item in backups_before_uninstall["items"]}
+        for item in current_backups["items"]:            
+            backup_name = item["metadata"]["name"]
+            assert backup_name in target_backup_names, f"Error: Backup {backup_name} not found in {target_backup_names}"
+
+            """
+            for i in range(self.retry_count):
+                volume_name = item["status"]["volumeName"]
+                if self.get_backup_volume(volume_name) != None:
+                    break
+            """
+
     def delete(self, volume_name, backup_id):
         return NotImplemented
 

e2e/libs/k8s/k8s.py

@@ -2,6 +2,7 @@
 import subprocess
 import asyncio
 from kubernetes import client
+from kubernetes.client.rest import ApiException
 from workload.pod import create_pod
 from workload.pod import delete_pod
 from workload.pod import new_pod_manifest
@@ -71,3 +72,84 @@ def wait_all_pods_evicted(node_name):
         time.sleep(retry_interval)
 
     assert evicted, 'failed to evict pods'
+
+def wait_namespaced_job_complete(job_label, namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.BatchV1Api()
+    for i in range(retry_count):
+        target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label)        
+        if len(target_job.items) > 0:
+            running_jobs = []
+            for job in target_job.items:
+                conditions = job.status.conditions
+                if conditions:
+                    logging(f"=== {conditions}")
+                    for condition in conditions:
+                        logging(f"{condition.type}  {condition.status}")
+                        if condition.type == "Complete" and condition.status == "True":
+                            print(f"Job {job.metadata.name} is complete.")
+                            running_jobs.append(job)
+                            break
+            if len(running_jobs) == len(target_job.items):
+                return
+
+        logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...")
+        time.sleep(retry_interval)
+
+    assert False, 'Job not complete'
+
+def wait_namespace_terminated(namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.CoreV1Api()
+    for i in range(retry_count):
+        try:
+            target_namespace = api.read_namespace(name=namespace)
+            target_namespace_status = target_namespace.status.phase
+            logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...")
+        except ApiException:
+            return
+
+        time.sleep(retry_interval)
+
+    assert False, f'namespace {target_namespace.metadata.name} not terminated'
+
+def get_all_custom_resources():
+    api = client.ApiextensionsV1Api()
+    crds = api.list_custom_resource_definition()
+
+    return crds
+
+def delete_namespace(namespace):
+    api = client.CoreV1Api()
+    try:
+        api.delete_namespace(name=namespace)
+    except ApiException as e:
+        assert e.status == 404
+
+def create_namespace(namespace):
+    api = client.CoreV1Api()
+    namespace = client.V1Namespace(
+        metadata=client.V1ObjectMeta(name=namespace)
+    )
+
+    api.create_namespace(body=namespace)
+
+def get_pod_logs(namespace, pod_label):
+    api = client.CoreV1Api()
+    logs= ""
+    try:
+        pods = api.list_namespaced_pod(namespace, label_selector=pod_label)
+        for pod in pods.items:
+            pod_name = pod.metadata.name
+            logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace)                
+    except client.exceptions.ApiException as e:
+        logging(f"Exception when calling CoreV1Api: {e}")
+
+    logging(f'{logs}')
+    return logs
+
+def list_namespace_pods(namespace):
+    v1 = client.CoreV1Api()
+    pods = v1.list_namespaced_pod(namespace=namespace)
+
+    return pods

e2e/libs/keywords/backup_keywords.py

@@ -20,6 +20,18 @@ def get_backup_name(self, backup_id, volume_name=None):
     def get_backup_url(self, backup_id, volume_name=None):
         return self.backup.get(backup_id, volume_name).url
 
+    def get_backup_url_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup["status"]["url"]
+
+    def get_backup_data_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['annotations']["test.longhorn.io/data-checksum"]
+
+    def get_backup_name_from_backup_list(self, backup_list, backup_id):
+        backup = self.backup.get_from_list(backup_list, backup_id)
+        return backup['metadata']['name']
+
     def delete_backup_volume(self, volume_name):
         return self.backup.delete_backup_volume(volume_name)
 
@@ -31,3 +43,10 @@ def cleanup_backups(self):
         if get_backupstore():
             self.backup.cleanup_system_backups()
             self.backup.cleanup_backup_volumes()
+
+    def list_all_backups(self):
+        all_backups = self.backup.list_all()
+        return all_backups
+
+    def assert_all_backups_exist(self, source_backups, target_backup):
+        self.backup.assert_all_backups_exist(source_backups, target_backup)

e2e/libs/keywords/longhorn_deploy_keywords.py

@@ -0,0 +1,14 @@
+from longhorn_deploy import LonghornDeploy
+class longhorn_deploy_keywords:
+
+    def __init__(self):
+        self.longhorn = LonghornDeploy()
+
+    def uninstall_longhorn(self, longhorn_branch):        
+        self.longhorn.uninstall(longhorn_branch)
+
+    def check_longhorn_crd_removed(self):
+        self.longhorn.check_longhorn_crd_removed()
+
+    def install_longhorn(self, longhorn_branch):
+        self.longhorn.install(longhorn_branch)

e2e/libs/keywords/volume_keywords.py

@@ -93,7 +93,7 @@ def get_node_ids_by_replica_locality(self, volume_name, replica_locality):
 
     def write_volume_random_data(self, volume_name, size_in_mb, data_id=0):
         logging(f'Writing {size_in_mb} MB random data to volume {volume_name}')
-        checksum = self.volume.write_random_data(volume_name, size_in_mb, data_id)
+        checksum = self.volume.write_random_data(volume_name, size_in_mb, data_id)        
 
     def keep_writing_data(self, volume_name):
         logging(f'Keep writing data to volume {volume_name}')
@@ -305,3 +305,6 @@ def check_volume_replica_names_recorded(self, volume_name):
             f"Volume {volume_name} replica names mismatched:\n" \
             f"Want: {expected_replica_names}\n" \
             f"Got: {actual_replica_names}"
+
+    def get_volume_checksum(self, volume_name):
+        return self.volume.get_checksum(volume_name)

e2e/libs/longhorn_deploy/__init__.py

@@ -0,0 +1,3 @@
+from longhorn_deploy.longhorn_deploy import LonghornDeploy
+from longhorn_deploy.longhorn_kubectl import LonghornKubectl
+from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart

e2e/libs/longhorn_deploy/base.py

@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from k8s import k8s
+from kubernetes.client.rest import ApiException
+from utility.constant import LONGHORN_NAMESPACE
+from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL
+import time
+from utility.utility import logging
+from node_exec import NodeExec
+from node import Node
+
+class Base(ABC):
+
+    @abstractmethod
+    def install(self):
+        return NotImplemented
+
+    @abstractmethod
+    def uninstall(self, longhorn_branch=None):
+        return NotImplemented
+
+    def check_longhorn_crd_removed(self):
+        all_crd = k8s.get_all_custom_resources()
+        for crd in all_crd.items:            
+            assert "longhorn.io" not in crd.metadata.name
+
+    def check_longhorn_uninstall_pod_log(self):
+        logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
+        assert "error" not in logs
+        assert "level=fatal" not in logs
+
+    def wait_longhorn_status_running(self):
+        RETRY_COUNTS = 10  
+        RETRY_INTERVAL = 60        
+        retries = 0
+
+        while True:
+            try:
+                pods = k8s.list_namespace_pods(LONGHORN_NAMESPACE)
+
+                csi_pods = [pod for pod in pods.items if 'csi-' in pod.metadata.name]
+                engine_image_pods = [pod for pod in pods.items if 'engine-image-' in pod.metadata.name]                    
+                non_running_pods = [pod for pod in pods.items if pod.status.phase != 'Running']
+
+                if csi_pods and engine_image_pods and not non_running_pods:
+                    logging(f"Longhorn is fully running.")
+                    break
+                else:
+                    logging("Longhorn is still installing ... re-checking in 1m")
+            except ApiException as e:
+                logging("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
+
+            time.sleep(RETRY_INTERVAL)
+            retries += 1
+
+            if retries == RETRY_COUNTS:
+                logging("Error: longhorn installation timeout")
+                return 1  
+
+    def expose_longhorn_ui(self):        
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        cmd = "kubectl expose --type=NodePort deployment longhorn-ui -n longhorn-system "\
+              "--port 8000 --name longhorn-ui-nodeport "\
+              "--overrides '{\"apiVersion\": \"v1\",\"spec\":{\"ports\": [{\"port\":8000,\"protocol\":\"TCP\",\"targetPort\":8000,\"nodePort\":30000}]}}'"
+
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "expose longhorn-ui failed"