Merge pull request #271 from ricardobranco777/fix_eks

eks: Delete the services associated with the cluster

Makefile

@@ -2,12 +2,16 @@
 CONT_TAG=suse/qac/pcw
 
 .PHONY: all
-all: prepare test
+all: prepare test pylint
 
 .PHONY: prepare
 prepare:
 	pip install -r requirements_test.txt
 
+.PHONY: pylint
+pylint:
+	pylint ocw/lib/*.py cleanup_k8s.py
+
 .PHONY: test
 test:
 	flake8 --max-line-length=130 webui

ocw/lib/eks.py

@@ -86,15 +86,15 @@ def kubectl_client(self, region: str, cluster_name: str):
     def all_clusters(self) -> dict:
         clusters = {}
         for region in EKS.__cluster_regions:
-            self.log_dbg("Checking clusters in {}", region)
+            self.log_dbg(f"Checking clusters in {region}")
             response = self.eks_client(region).list_clusters()
             if 'clusters' in response and len(response['clusters']) > 0:
                 clusters[region] = []
-                self.log_dbg("Found {} clusters in {}", len(response['clusters']), region)
+                self.log_dbg(f"Found {len(response['clusters'])} clusters in {region}")
                 for cluster in response['clusters']:
                     cluster_description = self.eks_client(region).describe_cluster(name=cluster)
                     if 'cluster' not in cluster_description or 'tags' not in cluster_description['cluster']:
-                        self.log_err("Unexpected cluster description: {}", cluster_description)
+                        self.log_err(f"Unexpected cluster description: {cluster_description}")
                     elif TAG_IGNORE not in cluster_description['cluster']['tags']:
                         clusters[region].append(cluster)
                 if len(clusters[region]) == 0:
@@ -105,40 +105,54 @@ def wait_for_empty_nodegroup_list(self, region: str, cluster_name: str, timeout_
         if self.dry_run:
             self.log_info("Skip waiting due to dry-run mode")
             return None
-        self.log_dbg("Waiting empty nodegroup list in {}", cluster_name)
+        self.log_dbg(f"Waiting empty nodegroup list in {cluster_name}")
         end = datetime.now(timezone.utc) + timedelta(minutes=timeout_minutes)
         resp_nodegroup = self.eks_client(region).list_nodegroups(clusterName=cluster_name)
 
         while datetime.now(timezone.utc) < end and len(resp_nodegroup['nodegroups']) > 0:
             time.sleep(20)
             resp_nodegroup = self.eks_client(region).list_nodegroups(clusterName=cluster_name)
             if len(resp_nodegroup['nodegroups']) > 0:
-                self.log_dbg("Still waiting for {} nodegroups to disappear", len(resp_nodegroup['nodegroups']))
+                self.log_dbg(f"Still waiting for {len(resp_nodegroup['nodegroups'])} nodegroups to disappear")
         return None
 
+    def delete_nodegroups(self, region: str, cluster: str) -> None:
+        resp_nodegroup = self.eks_client(region).list_nodegroups(clusterName=cluster)
+        if 'nodegroups' in resp_nodegroup:
+            self.log_dbg(f"Found {len(resp_nodegroup['nodegroups'])} nodegroups for {cluster}")
+        for nodegroup in resp_nodegroup['nodegroups']:
+            if self.dry_run:
+                self.log_info(f"Skipping {nodegroup} nodegroup deletion due to dry-run mode")
+            else:
+                self.log_info(f"Deleting {nodegroup}")
+                self.eks_client(region).delete_nodegroup(
+                    clusterName=cluster, nodegroupName=nodegroup)
+        if 'nodegroups' in resp_nodegroup:
+            self.wait_for_empty_nodegroup_list(region, cluster)
+
+    def delete_services(self, region: str, cluster: str) -> None:
+        services = self.eks_client(region).list_services(clusterName=cluster)['services']
+        for service in services:
+            if self.dry_run:
+                self.log_info(f"Skipping {service} cluster service deletion due to dry-run mode", service)
+            else:
+                self.log_info(f"Deleting {service}", service)
+                self.eks_client(region).delete_service(clusterName=cluster, service=service)
+
     def delete_all_clusters(self) -> None:
         self.log_info("Deleting all clusters!")
         for region in self.__cluster_regions:
             response = self.eks_client(region).list_clusters()
             if len(response['clusters']):
-                self.log_dbg("Found {} cluster(s) in {}", len(response['clusters']), region)
-                for cluster in response['clusters']:
-                    resp_nodegroup = self.eks_client(region).list_nodegroups(clusterName=cluster)
-                    if len(resp_nodegroup['nodegroups']):
-                        self.log_dbg("Found {} nodegroups for {}", len(resp_nodegroup['nodegroups']), cluster)
-                        for nodegroup in resp_nodegroup['nodegroups']:
-                            if self.dry_run:
-                                self.log_info("Skipping {} nodegroup deletion due to dry-run mode", nodegroup)
-                            else:
-                                self.log_info("Deleting {}", nodegroup)
-                                self.eks_client(region).delete_nodegroup(
-                                    clusterName=cluster, nodegroupName=nodegroup)
-                        self.wait_for_empty_nodegroup_list(region, cluster)
-                    if self.dry_run:
-                        self.log_info("Skipping {} cluster deletion due to dry-run mode", cluster)
-                    else:
-                        self.log_info("Finally deleting {} cluster", cluster)
-                        self.eks_client(region).delete_cluster(name=cluster)
+                self.log_dbg(f"Found {len(response['clusters'])} cluster(s) in {region}")
+            for cluster in response['clusters']:
+                self.delete_nodegroups(region, cluster)
+                self.delete_services(region, cluster)
+                if self.dry_run:
+                    self.log_info(f"Skipping {cluster} cluster deletion due to dry-run mode")
+                else:
+                    self.log_info(f"Finally deleting {cluster} cluster")
+                    self.eks_client(region).delete_cluster(name=cluster)
 
     def cleanup_k8s_jobs(self):
         self.log_info("Cleanup k8s jobs in EKS clusters")

tests/test_eks.py

@@ -8,7 +8,7 @@
 from tests.kubernetes import MockedSubprocessReturn, MockedKubernetesClient, MockedKubernetesConfig, MockedKubernetesJob
 
 
-def test_list_clusters(eks_patch, monkeypatch):
+def test_all_clusters(eks_patch, monkeypatch):
     mocked_eks = MockedEKSClient()
     monkeypatch.setattr(EKS, 'eks_client', lambda self, region: mocked_eks)
     all_clusters = eks_patch.all_clusters()
@@ -31,9 +31,14 @@ def test_list_clusters(eks_patch, monkeypatch):
     assert all_clusters == {'region1': ['hastags']}
 
 
-class MockedEKSClient():
+class MockedEKSClient:
     def __init__(self):
         self.clusters_list = []
+        self.deleted_clusters = []
+        self.nodegrups = []
+        self.deleted_nodegroups = []
+        self.services = []
+        self.deleted_services = []
 
     def list_clusters(self):
         return self.clusters_list
@@ -49,6 +54,21 @@ def describe_cluster(self, name=None):
             return {'cluster': {'tags': {'pcw_ignore': '1'}}}
         return None
 
+    def delete_cluster(self, *args, **kwargs):
+        self.deleted_clusters.append(kwargs['name'])
+
+    def delete_nodegroup(self, *args, **kwargs):
+        self.deleted_nodegroups.append(kwargs['nodegroupName'])
+
+    def delete_service(self, *args, **kwargs):
+        self.deleted_services.append(kwargs['service'])
+
+    def list_nodegroups(self, *args, **kwargs):
+        return self.nodegroups
+
+    def list_services(self, *args, **kwargs):
+        return self.services
+
 
 @pytest.fixture
 def eks_patch(monkeypatch):
@@ -96,6 +116,61 @@ def test_create_credentials_file(eks_patch, monkeypatch):
         eks_patch.create_credentials_file()
 
 
+def test_delete_nodegroups(eks_patch, monkeypatch):
+    mocked_eks = MockedEKSClient()
+    mocked_eks.nodegroups = {'nodegroups': ['nodegroup1']}
+    monkeypatch.setattr(EKS, 'eks_client', lambda self, region: mocked_eks)
+    monkeypatch.setattr(EKS, 'wait_for_empty_nodegroup_list', lambda self, *args, **kwargs: None)
+
+    eks_patch.delete_nodegroups('myregion', 'mycluster')
+    assert mocked_eks.deleted_nodegroups == ['nodegroup1']
+
+    # test dry_run
+    eks_patch.dry_run = True
+    mocked_eks.deleted_nodegroups = []
+    eks_patch.delete_nodegroups('myregion', 'mycluster')
+    assert mocked_eks.deleted_nodegroups == []
+
+
+def test_delete_services(eks_patch, monkeypatch):
+    mocked_eks = MockedEKSClient()
+    mocked_eks.services = {'services': ['service1']}
+    monkeypatch.setattr(EKS, 'eks_client', lambda self, region: mocked_eks)
+
+    eks_patch.delete_services('myregion', 'mycluster')
+    assert mocked_eks.deleted_services == ['service1']
+
+    # test dry_run
+    eks_patch.dry_run = True
+    mocked_eks.deleted_services = []
+    eks_patch.delete_services('myregion', 'mycluster')
+    assert mocked_eks.deleted_services == []
+
+
+def test_delete_all_clusters(eks_patch, monkeypatch):
+    mocked_eks = MockedEKSClient()
+    mocked_eks.clusters_list = {'clusters': ['cluster1']}
+    mocked_eks.nodegroups = {'nodegroups': ['nodegroup1']}
+    mocked_eks.services = {'services': ['service1']}
+    monkeypatch.setattr(EKS, 'eks_client', lambda self, region: mocked_eks)
+    monkeypatch.setattr(EKS, 'wait_for_empty_nodegroup_list', lambda self, *args, **kwargs: None)
+
+    eks_patch.delete_all_clusters()
+    assert mocked_eks.deleted_clusters == ['cluster1']
+    assert mocked_eks.deleted_nodegroups == ['nodegroup1']
+    assert mocked_eks.deleted_services == ['service1']
+
+    # test dry_run
+    eks_patch.dry_run = True
+    mocked_eks.deleted_clusters = []
+    mocked_eks.deleted_nodegroups = []
+    mocked_eks.deleted_services = []
+    eks_patch.delete_all_clusters()
+    assert mocked_eks.deleted_clusters == []
+    assert mocked_eks.deleted_nodegroups == []
+    assert mocked_eks.deleted_services == []
+
+
 def test_cleanup_k8s_jobs(eks_patch, monkeypatch):
     mocked_eks = MockedEKSClient()
     mocked_eks.clusters_list = {'clusters': ['cluster1']}