diff --git a/e2e/keywords/common.resource b/e2e/keywords/common.resource index 642aaae18f..17e2902876 100644 --- a/e2e/keywords/common.resource +++ b/e2e/keywords/common.resource @@ -19,6 +19,7 @@ Library ../libs/keywords/setting_keywords.py Library ../libs/keywords/backupstore_keywords.py Library ../libs/keywords/backup_keywords.py Library ../libs/keywords/sharemanager_keywords.py +Library ../libs/keywords/k8s_keywords.py *** Keywords *** Set test environment @@ -35,6 +36,7 @@ Set test environment END Cleanup test resources + uncordon_all_nodes cleanup_control_plane_network_latency reset_node_schedule cleanup_node_exec diff --git a/e2e/keywords/host.resource b/e2e/keywords/host.resource index c15ffe8f0d..0ad7047162 100644 --- a/e2e/keywords/host.resource +++ b/e2e/keywords/host.resource @@ -37,3 +37,8 @@ Restart cluster Power on off node Run keyword And Continue On Failure ... power_on_node_by_name ${powered_off_node} + +Power off node ${node_id} + ${powered_off_node} = get_node_by_index ${node_id} + power_off_node_by_name ${powered_off_node} + Set Test Variable ${powered_off_node} \ No newline at end of file diff --git a/e2e/keywords/k8s.resource b/e2e/keywords/k8s.resource index 75ff214d4e..d553daef4e 100644 --- a/e2e/keywords/k8s.resource +++ b/e2e/keywords/k8s.resource @@ -5,6 +5,7 @@ Library ../libs/keywords/k8s_keywords.py Library ../libs/keywords/workload_keywords.py Library ../libs/keywords/volume_keywords.py Library ../libs/keywords/host_keywords.py +Library ../libs/keywords/node_keywords.py *** Variables *** @@ -51,5 +52,62 @@ Force drain volume of ${workload_kind} ${workload_id} replica node Set Test Variable ${drained_node} Set Test Variable ${last_volume_node} +Force drain node ${node_id} + ${node_name} = get_node_by_index ${node_id} + force_drain_node ${node_name} + +Drain volume of ${workload_kind} ${workload_id} volume node + ${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id} + ${volume_name} = get_workload_volume_name ${workload_name} + ${drained_node} = get_volume_node ${volume_name} + ${last_volume_node} = get_volume_node ${volume_name} + drain_node ${drained_node} + wait_for_all_pods_evicted ${drained_node} + Set Test Variable ${drained_node} + Set Test Variable ${last_volume_node} + Uncordon the drained node uncordon_node ${drained_node} + +Cordon ${workload_kind} ${workload_id} volume node + ${workload_name} = generate_name_with_suffix ${workload_kind} ${workload_id} + ${volume_name} = get_workload_volume_name ${workload_name} + ${volume_node} = get_volume_node ${volume_name} + cordon_node ${volume_node} + check_node_is_not_schedulable ${volume_node} + +Force drain all nodes + FOR ${node_id} IN RANGE 0 3 + ${node_name} = get_node_by_index ${node_id} + force_drain_node ${node_name} + wait_for_all_pods_evicted ${node_name} + END + +Check node ${node_id} cordoned + ${node_name} = get_node_by_index ${node_id} + check_node_cordoned ${node_name} + +Force drain node ${node_id} and wait for ${duration} second + ${drained_node} = get_node_by_index ${node_id} + ${instance_manager_name} = get_instance_manager_on_node ${drained_node} + ${drain_process} = force_drain_node_and_wait ${drained_node} ${duration} + Set Test Variable ${drain_process} + Set Test Variable ${instance_manager_name} + Set Test Variable ${drained_node} + +The drain process not completed + check_drain_process_not_completed ${drain_process} + +The drain process completed + wait_for_all_pods_evicted ${drained_node} + check_drain_process_completed ${drain_process} + +Drain logs should contain + [Arguments] ${log} + ${drain_logs} = get_drain_process_error_log ${drain_process} + Should Contain ${drain_logs} ${log} + +Check PDB not exist + [Arguments] ${instance_manger} + check_instance_manager_pdb_not_exist ${instance_manger} + diff --git a/e2e/keywords/longhorn.resource b/e2e/keywords/longhorn.resource index 377a7e38a9..e774eba5ae 100644 --- a/e2e/keywords/longhorn.resource +++ b/e2e/keywords/longhorn.resource @@ -3,6 +3,7 @@ Documentation Longhorn Keywords Library ../libs/keywords/instancemanager_keywords.py Library ../libs/keywords/workload_keywords.py +Library ../libs/keywords/k8s_keywords.py *** Variables *** @{longhorn_workloads} @@ -43,3 +44,12 @@ Check Longhorn workload pods ${condition} annotated with ${key} Run Keyword IF '${condition}' == 'not' Should Not Be True ${is_annotated} ... ELSE IF '${condition}' == 'is' Should Be True ${is_annotated} ... ELSE Fail Invalid condition ${condition} + +Check instance-manager pod is not running on drained node + ${pod} = get_instance_manager_on_node ${drained_node} + Should Be Equal ${pod} ${None} + +Check instance-manager pod is running on node ${node_id} + ${node_name} = get_node_by_index ${node_id} + ${pod} = get_instance_manager_on_node ${node_name} + Should Not Be Equal ${pod} ${None} \ No newline at end of file diff --git a/e2e/keywords/volume.resource b/e2e/keywords/volume.resource index 085188f074..1887b3b1fc 100644 --- a/e2e/keywords/volume.resource +++ b/e2e/keywords/volume.resource @@ -206,6 +206,12 @@ Wait volume ${volume_id} replica on node ${node_id} stopped ${node_name} = get_node_by_index ${node_id} wait_for_replica_stopped ${volume_name} ${node_name} +Check volume ${volume_id} replica on node ${node_id} exist + ${volume_name} = generate_name_with_suffix volume ${volume_id} + ${node_name} = get_node_by_index ${node_id} + ${replica_name} get_replica_name_on_node ${volume_name} ${node_name} + Should Not Be Equal ${replica_name} ${None} + Check volume ${volume_id} data is intact ${volume_name} = generate_name_with_suffix volume ${volume_id} check_data_checksum ${volume_name} diff --git a/e2e/libs/k8s/k8s.py b/e2e/libs/k8s/k8s.py index ea854b8fa6..4c90d1d4e8 100644 --- a/e2e/libs/k8s/k8s.py +++ b/e2e/libs/k8s/k8s.py @@ -1,6 +1,7 @@ import time import subprocess import asyncio +import os from kubernetes import client from workload.pod import create_pod from workload.pod import delete_pod @@ -9,6 +10,10 @@ from utility.utility import subprocess_exec_cmd from utility.utility import logging from utility.utility import get_retry_count_and_interval +from utility.utility import check_popen_process_not_completed +from utility.utility import check_popen_process_completed +from utility.utility import get_popen_process_error_log +from robot.libraries.BuiltIn import BuiltIn async def restart_kubelet(node_name, downtime_in_sec=10): manifest = new_pod_manifest( @@ -71,3 +76,45 @@ def wait_all_pods_evicted(node_name): time.sleep(retry_interval) assert evicted, 'failed to evict pods' + +def check_node_cordoned(node_name): + api = client.CoreV1Api() + node = api.read_node(node_name) + assert node.spec.unschedulable is True, f"node {node_name} is not cordoned." + +def force_drain_node_and_wait(node_name, duration): + _, retry_interval = get_retry_count_and_interval() + exec_cmd = ["kubectl", "drain", node_name, "--force", "--ignore-daemonsets", "--delete-emptydir-data"] + drain_process = subprocess.Popen(exec_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + for i in range(int(duration)): + logging(f"Performing {exec_cmd} and wait... counts={i}") + if drain_process.poll() is not None: + raise AssertionError(f"Drain node {node_name} completed, but it was expected not to complete!") + time.sleep(retry_interval) + + return drain_process + +def check_drain_process_not_completed(drain_process): + check_popen_process_not_completed(drain_process) + +def check_drain_process_completed(drain_process): + check_popen_process_completed(drain_process) + +def get_drain_process_error_log(drain_process): + return get_popen_process_error_log(drain_process) + +def get_instance_manager_on_node(node_name): + data_engine = BuiltIn().get_variable_value("${DATA_ENGINE}") + pods = get_all_pods_on_node(node_name) + for pod in pods: + labels = pod.metadata.labels + if labels.get("longhorn.io/data-engine") == data_engine and \ + labels.get("longhorn.io/component") == "instance-manager": + return pod.metadata.name + return None + +def check_instance_manager_pdb_not_exist(instance_manager): + exec_cmd = ["kubectl", "get", "pdb", "-n", "longhorn-system"] + res = subprocess_exec_cmd(exec_cmd) + assert instance_manager not in res.decode('utf-8') diff --git a/e2e/libs/keywords/host_keywords.py b/e2e/libs/keywords/host_keywords.py index bd4f6c7ef8..b2a4fe64c2 100644 --- a/e2e/libs/keywords/host_keywords.py +++ b/e2e/libs/keywords/host_keywords.py @@ -52,3 +52,6 @@ def power_off_volume_node(self, volume_name): def power_on_node_by_name(self, node_name): self.host.power_on_node(node_name) + + def power_off_node_by_name(self, node_name): + self.host.power_off_node(node_name) diff --git a/e2e/libs/keywords/k8s_keywords.py b/e2e/libs/keywords/k8s_keywords.py index 625696a8d3..85834dfe23 100644 --- a/e2e/libs/keywords/k8s_keywords.py +++ b/e2e/libs/keywords/k8s_keywords.py @@ -5,8 +5,16 @@ from k8s.k8s import drain_node, force_drain_node from k8s.k8s import cordon_node, uncordon_node from k8s.k8s import wait_all_pods_evicted +from k8s.k8s import get_all_pods_on_node +from k8s.k8s import check_node_cordoned +from k8s.k8s import force_drain_node_and_wait +from k8s.k8s import check_drain_process_not_completed +from k8s.k8s import get_instance_manager_on_node +from k8s.k8s import get_drain_process_error_log +from k8s.k8s import check_instance_manager_pdb_not_exist +from k8s.k8s import check_drain_process_completed from utility.utility import logging - +from node import Node class k8s_keywords: @@ -50,5 +58,38 @@ def force_drain_node(self, node_name): def uncordon_node(self, node_name): uncordon_node(node_name) + def cordon_node(self, node_name): + cordon_node(node_name) + def wait_for_all_pods_evicted(self, node_name): wait_all_pods_evicted(node_name) + + def uncordon_all_nodes(self): + nodes = Node.list_node_names_by_role("worker") + + for node_name in nodes: + uncordon_node(node_name) + + def get_all_pods_on_node(self, node_name): + return get_all_pods_on_node(node_name) + + def check_node_cordoned(self, node_name): + check_node_cordoned(node_name) + + def force_drain_node_and_wait(self, node_name, duration): + return force_drain_node_and_wait(node_name, duration) + + def check_drain_process_not_completed(self, drain_process): + return check_drain_process_not_completed(drain_process) + + def check_drain_process_completed(self, drain_process): + return check_drain_process_completed(drain_process) + + def get_instance_manager_on_node(self, node_name): + return get_instance_manager_on_node(node_name) + + def get_drain_process_error_log(self, drain_process): + return get_drain_process_error_log(drain_process) + + def check_instance_manager_pdb_not_exist(self, instance_manager): + return check_instance_manager_pdb_not_exist(instance_manager) \ No newline at end of file diff --git a/e2e/libs/keywords/node_keywords.py b/e2e/libs/keywords/node_keywords.py index 74ef9055bf..bd53dcfdf6 100644 --- a/e2e/libs/keywords/node_keywords.py +++ b/e2e/libs/keywords/node_keywords.py @@ -40,5 +40,9 @@ def enable_node_scheduling(self, node_name): def reset_node_schedule(self): nodes = self.node.list_node_names_by_role("worker") + for node_name in nodes: self.enable_node_scheduling(node_name) + + def check_node_is_not_schedulable(self, node_name): + self.node.check_node_schedulable(node_name, schedulable="False") diff --git a/e2e/libs/node/node.py b/e2e/libs/node/node.py index 565fe0c971..dc1491986c 100644 --- a/e2e/libs/node/node.py +++ b/e2e/libs/node/node.py @@ -10,6 +10,8 @@ from utility.utility import get_retry_count_and_interval from utility.utility import logging +from k8s.k8s import uncordon_node + class Node: DEFAULT_DISK_PATH = "/var/lib/longhorn/" @@ -159,3 +161,11 @@ def set_default_disk_scheduling(self, node_name, allowScheduling): if disk.path == self.DEFAULT_DISK_PATH: disk.allowScheduling = allowScheduling self.update_disks(node_name, node.disks) + + def check_node_schedulable(self, node_name, schedulable): + node = get_longhorn_client().by_id_node(node_name) + for _ in range(self.retry_count): + if node["conditions"]["Schedulable"]["status"] == schedulable: + break + time.sleep(self.retry_interval) + assert node["conditions"]["Schedulable"]["status"] == schedulable diff --git a/e2e/libs/utility/utility.py b/e2e/libs/utility/utility.py index 8f3521bb6b..c1e1e469c7 100644 --- a/e2e/libs/utility/utility.py +++ b/e2e/libs/utility/utility.py @@ -285,3 +285,25 @@ def get_name_suffix(*args): if arg: suffix += f"-{arg}" return suffix + + +def check_popen_process_not_completed(process): + assert process.poll() is None, f"process {process} terminated which is not expected." + + +def check_popen_process_completed(process): + retry_count, retry_interval = get_retry_count_and_interval() + for i in range(retry_count): + if process.poll() is not None: + return + time.sleep(retry_interval) + assert process.poll() is not None, f"process {process} not terminated which is not expected." + + +def get_popen_process_error_log(drain_process): + if drain_process.poll() is None: + drain_process.terminate() + + stdout, stderr = drain_process.communicate() + logging(f"{stderr.decode('utf-8')}") + return stderr.decode('utf-8') diff --git a/e2e/requirements.txt b/e2e/requirements.txt index 69b127c977..4a345f1cd3 100644 --- a/e2e/requirements.txt +++ b/e2e/requirements.txt @@ -6,4 +6,4 @@ kubernetes==27.2.0 requests==2.32.3 boto3==1.35.19 pyyaml==6.0.2 -minio==5.0.10 +minio==5.0.10 \ No newline at end of file diff --git a/e2e/tests/negative/node_drain.robot b/e2e/tests/negative/node_drain.robot index e220ceb845..1e689b1145 100644 --- a/e2e/tests/negative/node_drain.robot +++ b/e2e/tests/negative/node_drain.robot @@ -10,6 +10,10 @@ Resource ../keywords/k8s.resource Resource ../keywords/deployment.resource Resource ../keywords/workload.resource Resource ../keywords/setting.resource +Resource ../keywords/longhorn.resource +Resource ../keywords/volume.resource +Resource ../keywords/host.resource +Resource ../keywords/node.resource Test Setup Set test environment Test Teardown Cleanup test resources @@ -89,3 +93,142 @@ Force Drain Replica Node While Replica Rebuilding And Wait for deployment 1 pods stable And Check deployment 1 data in file data.txt is intact END + +Drain node with force + [Documentation] Drain node with force + ... 1. Deploy a cluster contains 3 worker nodes N1, N2, N3. + ... 2. Deploy Longhorn. + ... 3. Create a 1-replica deployment with a 3-replica Longhorn volume. The volume is attached to N1. + ... 4. Write some data to the volume and get the md5sum. + ... 5. Force drain and remove N2, which contains one replica only. + ... kubectl drain --delete-emptydir-data=true --force=true --grace-period=-1 --ignore-daemonsets=true --timeout= + ... 6. Wait for the volume Degraded. + ... 7. Force drain and remove N1, which is the node the volume is attached to. + ... kubectl drain --delete-emptydir-data=true --force=true --grace-period=-1 --ignore-daemonsets=true --timeout= + ... + ... 8. Wait for the volume detaching then being recovered. Will get attached to the workload/node. + ... 9. Verify the instance manager pods are gone and not recreated after the drain. + ... 10. Validate the volume content. The data is intact. + Given Create storageclass longhorn-test with dataEngine=${DATA_ENGINE} + And Create persistentvolumeclaim 0 using RWO volume with longhorn-test storageclass + And Create deployment 0 with persistentvolumeclaim 0 + And Wait for volume of deployment 0 healthy + And Write 2048 MB data to file data.txt in deployment 0 + + And Force drain volume of deployment 0 replica node + Then Wait for volume of deployment 0 attached to the original node and degraded + + And Force drain volume of deployment 0 volume node + And Wait for volume of deployment 0 attached to another node and degraded + And Check instance-manager pod is not running on drained node + Then Check deployment 0 data in file data.txt is intact + +Drain node without force + [Documentation] Drain node without force + ... 1. Cordon the node. Longhorn will automatically disable the node scheduling when a Kubernetes node is cordoned. + ... 2. Evict all the replicas from the node. + ... 3. Run the following command to drain the node with force flag set to false. + ... kubectl drain --delete-emptydir-data --force=false --grace-period=-1 --ignore-daemonsets=true --timeout= + ... 4. Observe that the workloads move to another node. The volumes should first detach and attach to workloads once they move to another node. + ... 5. One by one all the pods should get evicted. + ... 6. Verify the instance manager pods are gone and not recreated after the drain. + Given Create storageclass longhorn-test with dataEngine=${DATA_ENGINE} + And Create persistentvolumeclaim 0 using RWO volume with longhorn-test storageclass + And Create deployment 0 with persistentvolumeclaim 0 + And Wait for volume of deployment 0 healthy + And Write 2048 MB data to file data.txt in deployment 0 + + When Cordon deployment 0 volume node + And Delete replica of deployment 0 volume on volume node + And Drain volume of deployment 0 volume node + And Wait for volume of deployment 0 attached to another node and degraded + And Check instance-manager pod is not running on drained node + Then Check deployment 0 data in file data.txt is intact + +Test kubectl drain nodes for PVC/PV/LHV is created through Longhorn API + [Documentation] Test kubectl drain nodes for PVC/PV/LHV is created through Longhorn API + ... Given 1 PVC/PV/LHV created through Longhorn API And LHV is not yet attached/replicated. + ... When kubectl drain nodes. + ... NODE=centos-worker-0 + ... kubectl cordon ${NODE} + ... kubectl drain --force --ignore-daemonsets --delete-emptydir-data --grace-period=10 ${NODE} + ... Then all node should successfully drain. + When Create volume 0 with dataEngine=${DATA_ENGINE} + And Wait for volume 0 detached + And Create persistentvolume for volume 0 + And Create persistentvolumeclaim for volume 0 + And Force drain all nodes + +Stopped replicas on deleted nodes should not be counted as healthy replicas when draining nodes + [Documentation] Stopped replicas on deleted nodes should not be counted as healthy replicas when draining nodes + ... When draining a node, the node will be set as unscheduled and all pods should be evicted. + ... By Longhorn’s default settings, the replica will only be evicted if there is another healthy replica on the running node. + ... Related Issue: + ... - https://github.com/longhorn/longhorn/issues/2237 + ... + ... Given Longhorn with 2 nodes cluster: Node_1, Node_2 + ... And Update the Node Drain Policy setting to block-if-contains-last-replica + ... And Create a 5Gi detached volume with 2 replicas. + ... And Stop Node_1 that contains one of the replicas. + ... When Attempts to drain Node_2 that contains remaining replica. + ... kubectl drain --delete-emptydir-data=true --force=true --grace-period=-1 --ignore-daemonsets=true + ... Then The Node_2 becomes cordoned. + ... And All pods on Node_2 are evicted except the replica instance manager pod. + ... kubectl get pods --field-selector spec.nodeName= -o wide -n longhorn-system + ... And The message like below keeps appearing. + ... evicting pod longhorn-system/instance-manager-r-xxxxxxxx + ... error when evicting pods/"instance-manager-r-xxxxxxxx" -n "longhorn-system" (will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget. + ... And The last healthy replica exists on the Node_2. + [Teardown] Cleanup test resources include off nodes + Given Disable node 0 scheduling + And Set setting node-drain-policy to block-if-contains-last-replica + And Given Create volume 0 with size=5Gi numberOfReplicas=2 dataEngine=${DATA_ENGINE} + And Attach volume 0 to node 1 + And Wait for volume 0 healthy + And Write data to volume 0 + And Detach volume 0 from attached node + And Wait for volume 0 detached + And Power off node 1 + + When Force drain node 2 and wait for 90 second + And The drain process not completed + And Check instance-manager pod is running on node 2 + And Drain logs should contain log=error when evicting pods/\"${instance_manager_name} + And Check volume 0 replica on node 2 exist + +Setting Allow Node Drain with the Last Healthy Replica protects the last healthy replica with Pod Disruption Budget (PDB) + [Documentation] Setting Allow Node Drain with the Last Healthy Replica protects the last healthy replica with Pod Disruption Budget (PDB) + ... Related Issue: + ... - https://github.com/longhorn/longhorn/issues/2237 + ... + ... Given Longhorn with 2 nodes cluster: Node_1, Node_2 + ... And Update the Node Drain Policy setting to block-if-contains-last-replica and confirm with following command: + ... kubectl get settings.longhorn.io/node-drain-policy -n longhorn-system + ... And Create a 5Gi detached volume with 2 replicas. + ... And Stop Node_1 that contains one of the replicas. + ... And Drain Node_2 so that all pods on Node_2 are evicted, but the replica instance manager pod is still on Node_2 because it is protected by PDB. + ... kubectl drain --delete-emptydir-data=true --force=true --grace-period=-1 --ignore-daemonsets=true + ... And Update the Node Drain Policy setting to always-allow and confirm with following command: + ... kubectl get settings.longhorn.io/node-drain-policy -n longhorn-system + ... Then The pod longhorn-system/instance-manager-r-xxxxxxxx will be evicted successfully and the following command can be used to ensure that only daemonset pods such as engine-image, longhorn-csi-plugin and longhorn-manager daemonset pods are running on Node_2: + ... kubectl get pods --field-selector spec.nodeName= -o wide -n longhorn-system + ... And The PDB will be deleted and can be verified with the following command: + ... kubectl get pdb -n longhorn-system + [Teardown] Cleanup test resources include off nodes + Given Disable node 0 scheduling + And Set setting node-drain-policy to block-if-contains-last-replica + And Given Create volume 0 with size=5Gi numberOfReplicas=2 dataEngine=${DATA_ENGINE} + And Attach volume 0 to node 1 + And Wait for volume 0 healthy + And Write data to volume 0 + And Detach volume 0 from attached node + And Wait for volume 0 detached + And Power off node 1 + + When Force drain node 2 and wait for 90 second + And The drain process not completed + And Check instance-manager pod is running on node 2 + + When Set setting node-drain-policy to always-allow + And The drain process completed + And Check PDB not exist instance_manger=${instance_manager_name}