test(robot): add uninstallation check test case

ref: longhorn/longhorn#9222

Signed-off-by: Chris <chris.chien@suse.com>

e2e/keywords/common.resource

@@ -53,3 +53,8 @@ Cleanup test resources
 Cleanup test resources include off nodes
     Power on off node
     Cleanup test resources
+
+Cleanup test resources after uninstall
+    cleanup_control_plane_network_latency
+    cleanup_node_exec
+    cleanup_stress_helper

e2e/keywords/longhorn.resource

@@ -3,6 +3,7 @@ Documentation       Longhorn Keywords
 
 Library             ../libs/keywords/instancemanager_keywords.py
 Library             ../libs/keywords/workload_keywords.py
+Library             ../libs/keywords/uninstallation_keywords.py
 
 *** Variables ***
 @{longhorn_workloads}
@@ -43,3 +44,9 @@ Check Longhorn workload pods ${condition} annotated with ${key}
     Run Keyword IF    '${condition}' == 'not'    Should Not Be True    ${is_annotated}
     ...    ELSE IF    '${condition}' == 'is'    Should Be True    ${is_annotated}
     ...    ELSE    Fail    Invalid condition ${condition}
+
+Uninstall Longhorn ${LONGHORN_BRANCH} by ${INSTALL_METHOD}
+    uninstall_longhorn    ${LONGHORN_BRANCH}    ${INSTALL_METHOD}
+
+Check all Longhorn CRD removed
+    check_longhorn_crd_removed

e2e/libs/k8s/k8s.py

@@ -2,6 +2,7 @@
 import subprocess
 import asyncio
 from kubernetes import client
+from kubernetes.client.rest import ApiException
 from workload.pod import create_pod
 from workload.pod import delete_pod
 from workload.pod import new_pod_manifest
@@ -71,3 +72,70 @@ def wait_all_pods_evicted(node_name):
         time.sleep(retry_interval)
 
     assert evicted, 'failed to evict pods'
+
+def wait_namespaced_job_complete(job_label, namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.BatchV1Api()
+    for i in range(retry_count):
+        target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label)        
+        if len(target_job.items) > 0:
+            running_jobs = []
+            for job in target_job.items:
+                conditions = job.status.conditions
+                if conditions:
+                    logging(f"=== {conditions}")
+                    for condition in conditions:
+                        logging(f"{condition.type}  {condition.status}")
+                        if condition.type == "Complete" and condition.status == "True":
+                            print(f"Job {job.metadata.name} is complete.")
+                            running_jobs.append(job)
+                            break
+            if len(running_jobs) == len(target_job.items):
+                return
+
+        logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...")
+        time.sleep(retry_interval)
+
+    assert False, 'Job not complete'
+
+def wait_namespace_terminated(namespace):
+    retry_count, retry_interval = get_retry_count_and_interval()
+    api = client.CoreV1Api()
+    for i in range(retry_count):
+        try:
+            target_namespace = api.read_namespace(name=namespace)
+            target_namespace_status = target_namespace.status.phase
+            logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...")
+        except:
+            return
+
+        time.sleep(retry_interval)
+
+    assert False, f'namespace {target_namespace.metadata.name} not terminated'
+
+def get_all_custom_resources():
+    api = client.ApiextensionsV1Api()
+    crds = api.list_custom_resource_definition()
+
+    return crds
+
+def delete_namespace(namespace):
+    api = client.CoreV1Api()
+    try:
+        api.delete_namespace(name=namespace)
+    except ApiException as e:
+        assert e.status == 404        
+
+def get_pod_logs(namespace, pod_label):
+    api = client.CoreV1Api()
+    logs= ""
+    try:
+        pods = api.list_namespaced_pod(namespace, label_selector=pod_label)
+        for pod in pods.items:
+            pod_name = pod.metadata.name                        
+            logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace)                
+    except client.exceptions.ApiException as e:
+        logging(f"Exception when calling CoreV1Api: {e}")
+
+    logging(f'{logs}')
+    return logs

e2e/libs/keywords/uninstallation_keywords.py

@@ -0,0 +1,12 @@
+from uninstallation import Uninstallation
+
+class uninstallation_keywords:
+
+    def __init__(self):
+        self.uninstallation = Uninstallation()
+
+    def uninstall_longhorn(self, longhor_branch, install_method):
+        self.uninstallation.uninstall_longhorn(longhor_branch, install_method)
+
+    def check_longhorn_crd_removed(self):
+        self.uninstallation.check_longhorn_crd_removed()

e2e/libs/uninstallation/__init__.py

@@ -0,0 +1 @@
+from uninstallation.uninstallation import Uninstallation

e2e/libs/uninstallation/uninstallation.py

@@ -0,0 +1,57 @@
+from utility.utility import logging
+from node import Node
+from node_exec import NodeExec
+from k8s import k8s
+from utility.constant import LONGHORN_NAMESPACE
+from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL
+from kubernetes import client
+
+import os
+
+class Uninstallation:
+
+    def uninstall_longhorn(self, longhorn_branch, install_method):
+        control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane")
+        control_plane_node = control_plane_nodes[0]
+
+        if install_method == "kubectl":
+            self.uninstall_longhorn_by_kubectl(control_plane_node, longhorn_branch)
+
+        elif install_method == "helm":
+            self.uninstall_longhorn_by_helm(control_plane_node)
+
+    def check_longhorn_crd_removed(self):
+        all_crd = k8s.get_all_custom_resources()
+        for crd in all_crd.items:            
+            assert "longhorn.io" not in crd.metadata.name
+
+    def uninstall_longhorn_by_kubectl(self, control_plane_node, longhorn_branch):
+        cmd = f"kubectl create -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/uninstall/uninstall.yaml"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "apply uninstall yaml failed"            
+        k8s.wait_namespaced_job_complete(job_label=LONGHORN_UNINSTALL_JOB_LABEL, namespace=LONGHORN_NAMESPACE)
+        self.check_longhorn_uninstall_pod_log()
+
+        cmd =f"kubectl delete -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/deploy/longhorn.yaml"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)            
+        assert res, "delete remaining components failed"
+
+        cmd= f"kubectl delete -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/uninstall/uninstall.yaml"
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "delete uninstallation components failed"
+        k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
+
+    def uninstall_longhorn_by_helm(self, control_plane_node):
+        kubeconfig_path = os.getenv('KUBECONFIG')
+
+        cmd = f"export KUBECONFIG={kubeconfig_path} && helm uninstall longhorn -n {LONGHORN_NAMESPACE}"            
+        res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd)
+        assert res, "helm uninstallation longhorn failed"
+
+        k8s.delete_namespace(LONGHORN_NAMESPACE)
+        k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE)
+
+    def check_longhorn_uninstall_pod_log(self):
+        logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL)
+        assert "level=error" not in logs
+        assert "level=fatal" not in logs

e2e/libs/utility/constant.py

@@ -18,3 +18,5 @@
 
 DISK_BEING_SYNCING = "being syncing and please retry later"
 NODE_UPDATE_RETRY_INTERVAL = 6
+
+LONGHORN_UNINSTALL_JOB_LABEL="job-name=longhorn-uninstall"

e2e/tests/test_cases/uninstallation_checks.robot

@@ -0,0 +1,65 @@
+*** Settings ***
+Documentation    Uninstallation Checks
+
+Test Tags    manual_test_case
+
+Resource    ../keywords/common.resource
+Resource    ../keywords/setting.resource
+Resource    ../keywords/volume.resource
+Resource    ../keywords/persistentvolume.resource
+Resource    ../keywords/persistentvolumeclaim.resource
+Resource    ../keywords/workload.resource
+Resource    ../keywords/backup.resource
+Resource    ../keywords/snapshot.resource
+Resource    ../keywords/backupstore.resource
+Resource    ../keywords/longhorn.resource
+
+Test Setup    Set test environment
+Test Teardown    Cleanup test resources after uninstall
+
+*** Variables ***
+${LOOP_COUNT}    1
+${RETRY_COUNT}    300
+${RETRY_INTERVAL}    1
+${DATA_ENGINE}    v1
+${LONGHORN_BRANCH}    masater
+${INSTALL_METHOD}    kubectl
+
+*** Test Cases ***
+Uninstallation Checks
+    [Documentation]    Uninstallation Checks    
+    ...    Prerequisites
+    ...    - Have a setup of Longhorn installed on a kubernetes cluster.
+    ...    - Have few volumes backups stored on S3/NFS backup store.
+    ...    - Have one DR volume created (not activated) in another cluster with a volume in current cluster.
+    ...
+    ...    Test steps
+    ...    1. Uninstall Longhorn.
+    ...    2. Check the logs of the job longhorn-uninstall, make sure there is no error(skip this step if using helm).
+    ...    3. Check all the components of Longhorn from the namespace longhorn-system are uninstalled. E.g. Longhorn manager, Longhorn driver, Longhorn UI, instance manager, engine image, CSI driver etc.
+    ...    4. Check all the CRDs are removed kubectl get crds | grep longhorn.
+    ...    
+    ...    Not implemented Steps
+    ...    5. Check the backup stores, the backups taken should NOT be removed.
+    ...    6. Activate the DR volume in the other cluster and check the data.
+
+    Given Set setting deleting-confirmation-flag to true
+    And Create volume 0 with    dataEngine=v1
+    And Attach volume 0
+    And Wait for volume 0 healthy
+    And Create volume 1 with    dataEngine=v2
+    And Attach volume 1
+    And Wait for volume 1 healthy
+
+    When Create backup 0 for volume 0
+    And Create backup 1 for volume 1
+    Then Verify backup list contains no error for volume 0
+    And Verify backup list contains no error for volume 1
+    And Verify backup list contains backup 0 of volume 0
+    And Verify backup list contains backup 1 of volume 1
+
+    When Create DR volume 2 from backup 0
+    And Wait for volume 2 restoration from backup 0 completed
+
+    Then Uninstall Longhorn ${LONGHORN_BRANCH} by ${INSTALL_METHOD}
+    And Check Longhorn CRD removed