From 73bee6c0ca1daa819c098a67a63b82b5adcf8c0f Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 21 Aug 2024 17:20:05 +0800 Subject: [PATCH] test(robot): add uninstallation check test case ref: longhorn/longhorn#9222 Signed-off-by: Chris --- e2e/Dockerfile | 8 ++ e2e/keywords/backup.resource | 3 + e2e/keywords/longhorn.resource | 15 +++- e2e/keywords/volume.resource | 15 ++++ e2e/libs/backup/backup.py | 9 ++ e2e/libs/backup/base.py | 4 + e2e/libs/backup/rest.py | 36 ++++++++ e2e/libs/k8s/k8s.py | 76 +++++++++++++++++ e2e/libs/keywords/backup_keywords.py | 19 +++++ e2e/libs/keywords/longhorn_deploy_keywords.py | 14 ++++ e2e/libs/keywords/volume_keywords.py | 3 + e2e/libs/longhorn_deploy/__init__.py | 3 + e2e/libs/longhorn_deploy/base.py | 45 ++++++++++ e2e/libs/longhorn_deploy/longhorn_deploy.py | 24 ++++++ .../longhorn_deploy/longhorn_helm_chart.py | 23 +++++ e2e/libs/longhorn_deploy/longhorn_kubectl.py | 34 ++++++++ e2e/libs/utility/constant.py | 4 + e2e/libs/utility/utility.py | 10 +++ .../negative/uninstallation_checks.robot | 80 ++++++++++++++++++ e2e/utilities/longhorn-install.sh | 84 +++++++++++++++++++ pipelines/utilities/run_longhorn_e2e_test.sh | 18 ++++ 21 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 e2e/libs/keywords/longhorn_deploy_keywords.py create mode 100644 e2e/libs/longhorn_deploy/__init__.py create mode 100644 e2e/libs/longhorn_deploy/base.py create mode 100644 e2e/libs/longhorn_deploy/longhorn_deploy.py create mode 100644 e2e/libs/longhorn_deploy/longhorn_helm_chart.py create mode 100644 e2e/libs/longhorn_deploy/longhorn_kubectl.py create mode 100644 e2e/tests/negative/uninstallation_checks.robot create mode 100644 e2e/utilities/longhorn-install.sh diff --git a/e2e/Dockerfile b/e2e/Dockerfile index e594282407..4e27974484 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -26,6 +26,14 @@ RUN curl -sO https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_ curl -L https://github.com/jonelo/jacksum/releases/download/v3.4.0/jacksum-3.4.0.jar --output /jacksum.jar ADD e2e /e2e +ADD pipelines/utilities/longhorn_ui.sh ./pipelines/utilities/longhorn_ui.sh +ADD pipelines/utilities/create_longhorn_namespace.sh ./pipelines/utilities/create_longhorn_namespace.sh +ADD pipelines/utilities/install_backupstores.sh ./pipelines/utilities/install_backupstores.sh +ADD pipelines/utilities/longhorn_status.sh ./pipelines/utilities/longhorn_status.sh +ADD pipelines/utilities/longhorn_helm_chart.sh ./pipelines/utilities/longhorn_helm_chart.sh +ADD pipelines/utilities/create_aws_secret.sh ./pipelines/utilities/create_aws_secret.sh +ADD pipelines/utilities/longhorn_manifest.sh ./pipelines/utilities/longhorn_manifest.sh + WORKDIR /e2e RUN pip install -r requirements.txt diff --git a/e2e/keywords/backup.resource b/e2e/keywords/backup.resource index ba3819bba5..853556fd69 100644 --- a/e2e/keywords/backup.resource +++ b/e2e/keywords/backup.resource @@ -26,3 +26,6 @@ Check volume ${volume_id} data is backup ${backup_id} ${volume_name} = generate_name_with_suffix volume ${volume_id} ${backup_name} = get_backup_name ${backup_id} check_restored_volume_checksum ${volume_name} ${backup_name} + +Check backup synced from backupstore + assert_all_backups_before_uninstall_exist ${backups_before_uninstall} diff --git a/e2e/keywords/longhorn.resource b/e2e/keywords/longhorn.resource index b181bec88a..163baf9806 100644 --- a/e2e/keywords/longhorn.resource +++ b/e2e/keywords/longhorn.resource @@ -4,6 +4,8 @@ Documentation Longhorn Keywords Library ../libs/keywords/instancemanager_keywords.py Library ../libs/keywords/workload_keywords.py Library ../libs/keywords/k8s_keywords.py +Library ../libs/keywords/longhorn_deploy_keywords.py +Library ../libs/keywords/backup_keywords.py *** Variables *** @{longhorn_workloads} @@ -52,4 +54,15 @@ Check instance-manager pod is not running on drained node Check instance-manager pod is running on node ${node_id} ${node_name} = get_node_by_index ${node_id} ${pod} = get_instance_manager_on_node ${node_name} - Should Not Be Equal ${pod} ${None} \ No newline at end of file + Should Not Be Equal ${pod} ${None} + +Uninstall Longhorn + ${backups_before_uninstall} = list_all_backups + uninstall_longhorn_system + Set Test Variable ${backups_before_uninstall} + +Check all Longhorn CRD removed + check_longhorn_crd_removed + +Install Longhorn + install_longhorn_system diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index 1887b3b1fc..337f0ae240 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -266,3 +266,18 @@ Upgrade volume ${volume_id} engine to compatible engine image ${volume_name} = generate_name_with_suffix volume ${volume_id} upgrade_engine_image ${volume_name} ${compatible_engine_image_name} wait_for_engine_image_upgrade_completed ${volume_name} ${compatible_engine_image_name} +Check volume ${volume_id} data is backup ${backup_id} created in another cluster + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${current_checksum} = get_volume_checksum ${volume_name} + ${backup_data} = get_backup_data_from_backup_list ${backups_before_uninstall} ${backup_id} + Should Be Equal ${current_checksum} ${backup_data} + +Create DR volume ${volume_id} from backup ${backup_id} in another cluster + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${backup_url} = get_backup_url_from_backup_list ${backups_before_uninstall} ${backup_id} + create_volume ${volume_name} frontend= Standby=True fromBackup=${backup_url} + +Wait for volume ${volume_id} restoration from backup ${backup_id} in another cluster completed + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${backup_name} = get_backup_name_from_backup_list ${backups_before_uninstall} ${backup_id} + wait_for_volume_restoration_completed ${volume_name} ${backup_name} diff --git a/e2e/libs/backup/backup.py b/e2e/libs/backup/backup.py index dc9aac4038..511f89bfd0 100644 --- a/e2e/libs/backup/backup.py +++ b/e2e/libs/backup/backup.py @@ -21,12 +21,21 @@ def create(self, volume_name, backup_id): def get(self, backup_id, volume_name): return self.backup.get(backup_id, volume_name) + def get_from_list(self, backup_list, backup_id): + return self.backup.get_from_list(backup_list, backup_id) + def get_backup_volume(self, volume_name): return self.backup.get_backup_volume(volume_name) def list(self, volume_name): return self.backup.list(volume_name) + def list_all(self): + return self.backup.list_all() + + def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall): + return self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall) + def verify_no_error(self, volume_name): backup_volume = self.get_backup_volume(volume_name) assert not backup_volume['messages'], \ diff --git a/e2e/libs/backup/base.py b/e2e/libs/backup/base.py index 4111f9a2fd..1a85123686 100644 --- a/e2e/libs/backup/base.py +++ b/e2e/libs/backup/base.py @@ -71,6 +71,10 @@ def wait_for_backup_completed(self, volume_name, snapshot_name): def list(self, volume_name): return NotImplemented + @abstractmethod + def list_all(self): + return NotImplemented + @abstractmethod def delete(self, volume_name, backup_id): return NotImplemented diff --git a/e2e/libs/backup/rest.py b/e2e/libs/backup/rest.py index 00f10360bd..82552cd30f 100644 --- a/e2e/libs/backup/rest.py +++ b/e2e/libs/backup/rest.py @@ -2,6 +2,7 @@ from utility.utility import logging from utility.utility import get_longhorn_client from utility.utility import get_retry_count_and_interval +from utility.utility import get_all_crs from node_exec import NodeExec from volume import Rest as RestVolume from snapshot import Snapshot as RestSnapshot @@ -51,6 +52,12 @@ def get(self, backup_id, volume_name): return backup return None + def get_from_list(self, backup_list, backup_id): + for backup in backup_list["items"]: + if backup['metadata']['annotations']['test.longhorn.io/backup-id'] == backup_id: + return backup + return None + def get_by_snapshot(self, volume_name, snapshot_name): """ look for a backup from snapshot on the backupstore @@ -109,6 +116,35 @@ def list(self, volume_name): backup_volume = self.get_backup_volume(volume_name) return backup_volume.backupList().data + def list_all(self): + return get_all_crs(group="longhorn.io", + version="v1beta2", + namespace="longhorn-system", + plural="backups", + ) + + def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall): + for i in range(self.retry_count): + time.sleep(self.retry_interval) + try: + current_backups = self.list_all() + assert len(current_backups["items"]) == len(backups_before_uninstall["items"]) + break + except Exception as e: + logging(f"{e}") + continue + + target_backup_names = {(item["metadata"]["name"]) for item in backups_before_uninstall["items"]} + for item in current_backups["items"]: + backup_name = item["metadata"]["name"] + assert backup_name in target_backup_names, f"Error: Backup {backup_name} not found in {target_backup_names}" + + for i in range(self.retry_count): + time.sleep(self.retry_interval) + volume_name = item["status"]["volumeName"] + if self.get_backup_volume(volume_name) != None: + break + def delete(self, volume_name, backup_id): return NotImplemented diff --git a/e2e/libs/k8s/k8s.py b/e2e/libs/k8s/k8s.py index 3002c83e5f..faff415d7c 100644 --- a/e2e/libs/k8s/k8s.py +++ b/e2e/libs/k8s/k8s.py @@ -3,6 +3,7 @@ import asyncio import os from kubernetes import client +from kubernetes.client.rest import ApiException from workload.pod import create_pod from workload.pod import delete_pod from workload.pod import new_pod_manifest @@ -94,3 +95,78 @@ def check_instance_manager_pdb_not_exist(instance_manager): exec_cmd = ["kubectl", "get", "pdb", "-n", "longhorn-system"] res = subprocess_exec_cmd(exec_cmd) assert instance_manager not in res.decode('utf-8') +def wait_namespaced_job_complete(job_label, namespace): + retry_count, retry_interval = get_retry_count_and_interval() + api = client.BatchV1Api() + for i in range(retry_count): + target_job = api.list_namespaced_job(namespace=namespace, label_selector=job_label) + if len(target_job.items) > 0: + running_jobs = [] + for job in target_job.items: + conditions = job.status.conditions + if conditions: + for condition in conditions: + logging(f"{condition.type} {condition.status}") + if condition.type == "Complete" and condition.status == "True": + print(f"Job {job.metadata.name} is complete.") + running_jobs.append(job) + break + if len(running_jobs) == len(target_job.items): + return + + logging(f"Waiting for job with label {job_label} complete, retry ({i}) ...") + time.sleep(retry_interval) + + assert False, 'Job not complete' + +def wait_namespace_terminated(namespace): + retry_count, retry_interval = get_retry_count_and_interval() + api = client.CoreV1Api() + for i in range(retry_count): + try: + target_namespace = api.read_namespace(name=namespace) + target_namespace_status = target_namespace.status.phase + logging(f"Waiting for namespace {target_namespace.metadata.name} terminated, current status is {target_namespace_status} retry ({i}) ...") + except ApiException as e: + if e.status == 404: + logging(f"Namespace {namespace} successfully terminated.") + return + else: + logging(f"Error while fetching namespace {namespace} status: {e}") + + time.sleep(retry_interval) + + assert False, f'namespace {target_namespace.metadata.name} not terminated' + +def get_all_custom_resources(): + api = client.ApiextensionsV1Api() + crds = api.list_custom_resource_definition() + + return crds + +def get_pod_logs(namespace, pod_label): + api = client.CoreV1Api() + logs= "" + try: + pods = api.list_namespaced_pod(namespace, label_selector=pod_label) + for pod in pods.items: + pod_name = pod.metadata.name + logs = logs + api.read_namespaced_pod_log(name=pod_name, namespace=namespace) + except client.exceptions.ApiException as e: + logging(f"Exception when calling CoreV1Api: {e}") + + logging(f'{logs}') + return logs + +def list_namespace_pods(namespace): + v1 = client.CoreV1Api() + pods = v1.list_namespaced_pod(namespace=namespace) + + return pods + +def delete_namespace(namespace): + api = client.CoreV1Api() + try: + api.delete_namespace(name=namespace) + except ApiException as e: + assert e.status == 404 diff --git a/e2e/libs/keywords/backup_keywords.py b/e2e/libs/keywords/backup_keywords.py index b0d9f7e76f..bbb8ebc6c6 100644 --- a/e2e/libs/keywords/backup_keywords.py +++ b/e2e/libs/keywords/backup_keywords.py @@ -20,6 +20,18 @@ def get_backup_name(self, backup_id, volume_name=None): def get_backup_url(self, backup_id, volume_name=None): return self.backup.get(backup_id, volume_name).url + def get_backup_url_from_backup_list(self, backup_list, backup_id): + backup = self.backup.get_from_list(backup_list, backup_id) + return backup["status"]["url"] + + def get_backup_data_from_backup_list(self, backup_list, backup_id): + backup = self.backup.get_from_list(backup_list, backup_id) + return backup['metadata']['annotations']["test.longhorn.io/data-checksum"] + + def get_backup_name_from_backup_list(self, backup_list, backup_id): + backup = self.backup.get_from_list(backup_list, backup_id) + return backup['metadata']['name'] + def delete_backup_volume(self, volume_name): return self.backup.delete_backup_volume(volume_name) @@ -31,3 +43,10 @@ def cleanup_backups(self): if get_backupstore(): self.backup.cleanup_system_backups() self.backup.cleanup_backup_volumes() + + def list_all_backups(self): + all_backups = self.backup.list_all() + return all_backups + + def assert_all_backups_before_uninstall_exist(self, backups_before_uninstall): + self.backup.assert_all_backups_before_uninstall_exist(backups_before_uninstall) diff --git a/e2e/libs/keywords/longhorn_deploy_keywords.py b/e2e/libs/keywords/longhorn_deploy_keywords.py new file mode 100644 index 0000000000..ac170f4d5f --- /dev/null +++ b/e2e/libs/keywords/longhorn_deploy_keywords.py @@ -0,0 +1,14 @@ +from longhorn_deploy import LonghornDeploy +class longhorn_deploy_keywords: + + def __init__(self): + self.longhorn = LonghornDeploy() + + def uninstall_longhorn_system(self): + self.longhorn.uninstall() + + def check_longhorn_crd_removed(self): + self.longhorn.check_longhorn_crd_removed() + + def install_longhorn_system(self): + self.longhorn.install() diff --git a/e2e/libs/keywords/volume_keywords.py b/e2e/libs/keywords/volume_keywords.py index 726f693e4d..2757fc4bd8 100644 --- a/e2e/libs/keywords/volume_keywords.py +++ b/e2e/libs/keywords/volume_keywords.py @@ -311,3 +311,6 @@ def upgrade_engine_image(self, volume_name, engine_image_name): def wait_for_engine_image_upgrade_completed(self, volume_name, engine_image_name): self.volume.wait_for_engine_image_upgrade_completed(volume_name, engine_image_name) + + def get_volume_checksum(self, volume_name): + return self.volume.get_checksum(volume_name) diff --git a/e2e/libs/longhorn_deploy/__init__.py b/e2e/libs/longhorn_deploy/__init__.py new file mode 100644 index 0000000000..a6bdd48e39 --- /dev/null +++ b/e2e/libs/longhorn_deploy/__init__.py @@ -0,0 +1,3 @@ +from longhorn_deploy.longhorn_deploy import LonghornDeploy +from longhorn_deploy.longhorn_kubectl import LonghornKubectl +from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart diff --git a/e2e/libs/longhorn_deploy/base.py b/e2e/libs/longhorn_deploy/base.py new file mode 100644 index 0000000000..1ba6468cbf --- /dev/null +++ b/e2e/libs/longhorn_deploy/base.py @@ -0,0 +1,45 @@ +from abc import ABC, abstractmethod +from k8s import k8s +from utility.constant import LONGHORN_NAMESPACE +from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL +from utility.constant import LONGHORN_INSTALL_SCRIPT_PATH +from utility.constant import LONGHORN_INSTALL_TIMEOUT +import subprocess +import os +from utility.utility import get_retry_count_and_interval +from utility.utility import logging + +class Base(ABC): + + def __init__(self): + self.retry_count, self.retry_interval = get_retry_count_and_interval() + + @abstractmethod + def install(self): + return NotImplemented + + @abstractmethod + def uninstall(self, longhorn_branch=None): + return NotImplemented + + def check_longhorn_crd_removed(self): + all_crd = k8s.get_all_custom_resources() + for crd in all_crd.items: + assert "longhorn.io" not in crd.metadata.name + + def check_longhorn_uninstall_pod_log(self): + logs = k8s.get_pod_logs(LONGHORN_NAMESPACE, LONGHORN_UNINSTALL_JOB_LABEL) + assert "error" not in logs + assert "level=fatal" not in logs + + def install_longhorn(self): + current_path=os.getcwd() + full_path = os.path.join(current_path, LONGHORN_INSTALL_SCRIPT_PATH) + + try: + output = subprocess.check_output(['bash', full_path], timeout=LONGHORN_INSTALL_TIMEOUT) + logging(output) + except subprocess.CalledProcessError as e: + logging(f"Error: {e.stderr}") + except subprocess.TimeoutExpired as e: + logging(f"Command timed out after {e.timeout} seconds") diff --git a/e2e/libs/longhorn_deploy/longhorn_deploy.py b/e2e/libs/longhorn_deploy/longhorn_deploy.py new file mode 100644 index 0000000000..2023e7d73d --- /dev/null +++ b/e2e/libs/longhorn_deploy/longhorn_deploy.py @@ -0,0 +1,24 @@ +from longhorn_deploy.base import Base +from longhorn_deploy.longhorn_kubectl import LonghornKubectl +from longhorn_deploy.longhorn_helm_chart import LonghornHelmChart +import os + +class LonghornDeploy(Base): + + _method = os.getenv("LONGHORN_INSTALL_METHOD", "manifest") + + def __init__(self): + + if self._method == "manifest": + self.longhorn = LonghornKubectl() + elif self._method == "helm": + self.longhorn = LonghornHelmChart() + + def uninstall(self): + return self.longhorn.uninstall() + + def check_longhorn_crd_removed(self): + return self.longhorn.check_longhorn_crd_removed() + + def install(self): + return self.longhorn.install() diff --git a/e2e/libs/longhorn_deploy/longhorn_helm_chart.py b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py new file mode 100644 index 0000000000..65d49587fb --- /dev/null +++ b/e2e/libs/longhorn_deploy/longhorn_helm_chart.py @@ -0,0 +1,23 @@ +from longhorn_deploy.base import Base +from node import Node +from node_exec import NodeExec +from k8s import k8s +from utility.constant import LONGHORN_NAMESPACE + +import os + +class LonghornHelmChart(Base): + + def uninstall(self): + control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") + control_plane_node = control_plane_nodes[0] + + cmd = f'export KUBECONFIG={os.getenv("KUBECONFIG")} && helm uninstall longhorn -n {LONGHORN_NAMESPACE}' + res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd) + assert res, "apply helm uninstall command failed" + + k8s.delete_namespace(namespace=LONGHORN_NAMESPACE) + k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) + + def install(self): + self.install_longhorn() diff --git a/e2e/libs/longhorn_deploy/longhorn_kubectl.py b/e2e/libs/longhorn_deploy/longhorn_kubectl.py new file mode 100644 index 0000000000..f5a7834562 --- /dev/null +++ b/e2e/libs/longhorn_deploy/longhorn_kubectl.py @@ -0,0 +1,34 @@ +from longhorn_deploy.base import Base +from node import Node +from node_exec import NodeExec +from k8s import k8s +from utility.constant import LONGHORN_NAMESPACE +from utility.constant import LONGHORN_UNINSTALL_JOB_LABEL + +import os + +class LonghornKubectl(Base): + + def uninstall(self): + longhorn_branch = os.getenv("LONGHORN_REPO_BRANCH") + + control_plane_nodes = Node.list_node_names_by_role(self, role="control-plane") + control_plane_node = control_plane_nodes[0] + + cmd = f"kubectl create -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/uninstall/uninstall.yaml" + res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd) + assert res, "apply uninstall yaml failed" + k8s.wait_namespaced_job_complete(job_label=LONGHORN_UNINSTALL_JOB_LABEL, namespace=LONGHORN_NAMESPACE) + self.check_longhorn_uninstall_pod_log() + + cmd = f"kubectl delete -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/deploy/longhorn.yaml" + res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd) + assert res, "delete remaining components failed" + + cmd= f"kubectl delete -f https://raw.githubusercontent.com/longhorn/longhorn/{longhorn_branch}/uninstall/uninstall.yaml" + res = NodeExec.get_instance().issue_cmd(control_plane_node, cmd) + assert res, "delete uninstallation components failed" + k8s.wait_namespace_terminated(namespace=LONGHORN_NAMESPACE) + + def install(self): + self.install_longhorn() diff --git a/e2e/libs/utility/constant.py b/e2e/libs/utility/constant.py index 1db2c67a66..ac1c6e49ea 100644 --- a/e2e/libs/utility/constant.py +++ b/e2e/libs/utility/constant.py @@ -18,3 +18,7 @@ DISK_BEING_SYNCING = "being syncing and please retry later" NODE_UPDATE_RETRY_INTERVAL = 6 + +LONGHORN_UNINSTALL_JOB_LABEL="job-name=longhorn-uninstall" +LONGHORN_INSTALL_SCRIPT_PATH="utilities/longhorn-install.sh" +LONGHORN_INSTALL_TIMEOUT = 600 diff --git a/e2e/libs/utility/utility.py b/e2e/libs/utility/utility.py index 70c6ab3a4f..8169a6c5f1 100644 --- a/e2e/libs/utility/utility.py +++ b/e2e/libs/utility/utility.py @@ -164,6 +164,16 @@ def get_cr(group, version, namespace, plural, name): logging(f"Getting namespaced custom object error: {e}") time.sleep(retry_interval) +def get_all_crs(group, version, namespace, plural): + api = client.CustomObjectsApi() + retry_count, retry_interval = get_retry_count_and_interval() + for _ in range(retry_count): + try: + resp = api.list_namespaced_custom_object(group, version, namespace, plural) + return resp + except ApiException as e: + logging(f"Getting namespaced custom object error: {e}") + time.sleep(retry_interval) def filter_cr(group, version, namespace, plural, field_selector="", label_selector=""): api = client.CustomObjectsApi() diff --git a/e2e/tests/negative/uninstallation_checks.robot b/e2e/tests/negative/uninstallation_checks.robot new file mode 100644 index 0000000000..0d35658e28 --- /dev/null +++ b/e2e/tests/negative/uninstallation_checks.robot @@ -0,0 +1,80 @@ +*** Settings *** +Documentation Uninstallation Checks + +Test Tags negative + +Resource ../keywords/common.resource +Resource ../keywords/setting.resource +Resource ../keywords/volume.resource +Resource ../keywords/persistentvolume.resource +Resource ../keywords/persistentvolumeclaim.resource +Resource ../keywords/workload.resource +Resource ../keywords/backup.resource +Resource ../keywords/snapshot.resource +Resource ../keywords/backupstore.resource +Resource ../keywords/longhorn.resource +Library ../libs/keywords/setting_keywords.py + +Test Setup Set test environment +Test Teardown Cleanup test resources + +*** Variables *** +${LOOP_COUNT} 1 +${RETRY_COUNT} 300 +${RETRY_INTERVAL} 1 + +*** Test Cases *** +Uninstallation Checks + [Documentation] Uninstallation Checks + ... Prerequisites + ... - Have a setup of Longhorn installed on a kubernetes cluster. + ... - Have few volumes backups stored on S3/NFS backup store. + ... - Have one DR volume created (not activated) + ... + ... Test steps + ... 1. Uninstall Longhorn. + ... 2. Check the logs of the job longhorn-uninstall, make sure there is no error(skip this step if using helm). + ... 3. Check all the components of Longhorn from the namespace longhorn-system are uninstalled. E.g. Longhorn manager, Longhorn driver, Longhorn UI, instance manager, engine image, CSI driver etc. + ... 4. Check all the CRDs are removed kubectl get crds | grep longhorn. + ... 5. Check the backup stores, the backups taken should NOT be removed. + ... 6. Create the DR volume in the other cluster and check the data. + ... + ... Important + ... - This test case need have set environment variable manually first if not run on Jenkins + ... - LONGHORN_INSTALL_METHOD : helm or manifest + ... - LONGHORN_REPO_BRANCH (ex:master) + ... - CUSTOM_LONGHORN_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_ENGINE_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE (if not using master-head) + ... - CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE (if not using master-head) + + Given Set setting deleting-confirmation-flag to true + And Create volume 0 with dataEngine=v1 + And Attach volume 0 + And Wait for volume 0 healthy + And Write data 0 to volume 0 + And Create volume 1 with dataEngine=v2 + And Attach volume 1 + And Wait for volume 1 healthy + + When Create backup 0 for volume 0 + And Create backup 1 for volume 1 + Then Verify backup list contains no error for volume 0 + And Verify backup list contains no error for volume 1 + And Verify backup list contains backup 0 of volume 0 + And Verify backup list contains backup 1 of volume 1 + + Then Uninstall Longhorn + And Check Longhorn CRD removed + + # Assume this is another Longhorn cluster + Then Install Longhorn + And set_backupstore + And Check backup synced from backupstore + When Create DR volume 0 from backup 0 in another cluster + And Wait for volume 0 restoration from backup 0 in another cluster completed + And Activate DR volume 0 + And Attach volume 0 + And Wait for volume 0 healthy + Then Check volume 0 data is backup 0 created in another cluster diff --git a/e2e/utilities/longhorn-install.sh b/e2e/utilities/longhorn-install.sh new file mode 100644 index 0000000000..9e3d23c9e0 --- /dev/null +++ b/e2e/utilities/longhorn-install.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -x + +source ../pipelines/utilities/longhorn_ui.sh +source ../pipelines/utilities/create_longhorn_namespace.sh +source ../pipelines/utilities/install_backupstores.sh +source ../pipelines/utilities/longhorn_status.sh +source ../pipelines/utilities/longhorn_helm_chart.sh +source ../pipelines/utilities/create_aws_secret.sh +source ../pipelines/utilities/longhorn_manifest.sh + +# create and clean tmpdir +TMPDIR="/tmp/longhorn" +mkdir -p ${TMPDIR} +rm -rf "${TMPDIR}/" + +LONGHORN_NAMESPACE="longhorn-system" + +install_longhorn_by_chart(){ + CUSTOM_LONGHORN_MANAGER_IMAGE=${CUSTOM_LONGHORN_MANAGER_IMAGE:-"longhornio/longhorn-manager:master-head"} + CUSTOM_LONGHORN_ENGINE_IMAGE=${CUSTOM_LONGHORN_ENGINE_IMAGE:-"longhornio/longhorn-engine:master-head"} + + CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE=${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE:-""} + CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE=${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE:-""} + CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE=${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE:-""} + + HELM_VARIABLES=("--set enableGoCoverDir=true") + CUSTOM_LONGHORN_MANAGER_IMAGE_REPO=$(echo $CUSTOM_LONGHORN_MANAGER_IMAGE | cut -d ":" -f 1) + CUSTOM_LONGHORN_MANAGER_IMAGE_TAG=$(echo $CUSTOM_LONGHORN_MANAGER_IMAGE | cut -d ":" -f 2) + HELM_VARIABLES+=("--set image.longhorn.manager.repository=${CUSTOM_LONGHORN_MANAGER_IMAGE_REPO}") + HELM_VARIABLES+=("--set image.longhorn.manager.tag=${CUSTOM_LONGHORN_MANAGER_IMAGE_TAG}") + + CUSTOM_LONGHORN_ENGINE_IMAGE_REPO=$(echo $CUSTOM_LONGHORN_ENGINE_IMAGE | cut -d ":" -f 1) + CUSTOM_LONGHORN_ENGINE_IMAGE_TAG=$(echo $CUSTOM_LONGHORN_ENGINE_IMAGE | cut -d ":" -f 2) + HELM_VARIABLES+=("--set image.longhorn.engine.repository=${CUSTOM_LONGHORN_ENGINE_IMAGE_REPO}") + HELM_VARIABLES+=("--set image.longhorn.engine.tag=${CUSTOM_LONGHORN_ENGINE_IMAGE_TAG}") + + # replace images if custom image is specified. + if [[ ! -z ${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE} ]]; then + CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE_REPO=$(echo $CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE | cut -d ":" -f 1) + CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE_TAG=$(echo $CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE | cut -d ":" -f 2) + HELM_VARIABLES+=("--set image.longhorn.instanceManager.repository=${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE_REPO}") + HELM_VARIABLES+=("--set image.longhorn.instanceManager.tag=${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE_TAG}") + fi + + if [[ ! -z ${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE} ]]; then + CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE_REPO=$(echo $CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE | cut -d ":" -f 1) + CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE_TAG=$(echo $CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE | cut -d ":" -f 2) + HELM_VARIABLES+=("--set image.longhorn.shareManager.repository=${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE_REPO}") + HELM_VARIABLES+=("--set image.longhorn.shareManager.tag=${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE_TAG}") + fi + + if [[ ! -z ${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE} ]]; then + CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE_REPO=$(echo $CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE | cut -d ":" -f 1) + CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE_TAG=$(echo $CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE | cut -d ":" -f 2) + HELM_VARIABLES+=("--set image.longhorn.backingImageManager.repository=${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE_REPO}") + HELM_VARIABLES+=("--set image.longhorn.backingImageManager.tag=${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE_TAG}") + fi + + HELM_VARIABLE_STR=$( printf " %s" "${HELM_VARIABLES[@]}" ) + echo ${HELM_VARIABLE_STR} + + helm install longhorn "${LONGHORN_REPO_DIR}/chart/" --namespace longhorn-system --create-namespace $HELM_VARIABLE_STR + wait_longhorn_status_running +} + +install_longhorn(){ + create_longhorn_namespace + install_backupstores + if [[ "${LONGHORN_INSTALL_METHOD}" == "helm" ]]; then + LONGHORN_REPO_URI=${LONGHORN_REPO_URI:-"https://github.com/longhorn/longhorn.git"} + LONGHORN_REPO_DIR="${TMPDIR}/longhorn" + get_longhorn_chart + install_longhorn_by_chart + elif [[ "${LONGHORN_INSTALL_METHOD}" == "manifest" ]]; then + generate_longhorn_yaml_manifest "${TF_VAR_tf_workspace}" + install_longhorn_by_manifest "${TF_VAR_tf_workspace}/longhorn.yaml" + fi + setup_longhorn_ui_nodeport + +} + +install_longhorn diff --git a/pipelines/utilities/run_longhorn_e2e_test.sh b/pipelines/utilities/run_longhorn_e2e_test.sh index bf01ec5a0a..4319f64a12 100755 --- a/pipelines/utilities/run_longhorn_e2e_test.sh +++ b/pipelines/utilities/run_longhorn_e2e_test.sh @@ -1,6 +1,7 @@ run_longhorn_e2e_test(){ LONGHORN_TESTS_CUSTOM_IMAGE=${LONGHORN_TESTS_CUSTOM_IMAGE:-"longhornio/longhorn-e2e-test:master-head"} + LONGHORN_INSTALL_METHOD=${LONGHORN_INSTALL_METHOD:-"manifest"} LONGHORN_TESTS_MANIFEST_FILE_PATH="e2e/deploy/test.yaml" @@ -32,6 +33,15 @@ run_longhorn_e2e_test(){ yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env[6].value="'${LONGHORN_TEST_CLOUDPROVIDER}'"' ${LONGHORN_TESTS_MANIFEST_FILE_PATH} + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_REPO_URI", "value": "'${LONGHORN_REPO_URI}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_REPO_BRANCH", "value": "'${LONGHORN_REPO_BRANCH}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_ENGINE_IMAGE", "value": "'${CUSTOM_LONGHORN_ENGINE_IMAGE}'"}' "${CUSTOM_LONGHORN_ENGINE_IMAGE}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE", "value": "'${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + yq e -i 'select(.spec.containers[0] != null).spec.containers[0].env += {"name": "LONGHORN_INSTALL_METHOD", "value": "'${LONGHORN_INSTALL_METHOD}'"}' "${LONGHORN_TESTS_MANIFEST_FILE_PATH}" + LONGHORN_TEST_POD_NAME=`yq e 'select(.spec.containers[0] != null).metadata.name' ${LONGHORN_TESTS_MANIFEST_FILE_PATH}` kubectl apply -f ${LONGHORN_TESTS_MANIFEST_FILE_PATH} @@ -87,6 +97,14 @@ run_longhorn_e2e_test_out_of_cluster(){ -e LAB_ACCESS_KEY="${TF_VAR_lab_access_key}" \ -e LAB_SECRET_KEY="${TF_VAR_lab_secret_key}" \ -e LAB_CLUSTER_ID="$(cat /tmp/cluster_id)" \ + -e LONGHORN_REPO_URI="${LONGHORN_REPO_URI}"\ + -e LONGHORN_REPO_BRANCH="${LONGHORN_REPO_BRANCH}"\ + -e CUSTOM_LONGHORN_MANAGER_IMAGE="${CUSTOM_LONGHORN_MANAGER_IMAGE}"\ + -e CUSTOM_LONGHORN_ENGINE_IMAGE="${CUSTOM_LONGHORN_ENGINE_IMAGE}"\ + -e CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE="${CUSTOM_LONGHORN_INSTANCE_MANAGER_IMAGE}"\ + -e CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE="${CUSTOM_LONGHORN_SHARE_MANAGER_IMAGE}"\ + -e CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE="${CUSTOM_LONGHORN_BACKING_IMAGE_MANAGER_IMAGE}"\ + -e LONGHORN_INSTALL_METHOD="${LONGHORN_INSTALL_METHOD}"\ --mount source="vol-${IMAGE_NAME}",target=/tmp \ "${LONGHORN_TESTS_CUSTOM_IMAGE}" "${ROBOT_COMMAND_ARGS[@]}" docker stop "${CONTAINER_NAME}"