Cleanup old helm-test k8s namespaces

cleanup_k8s.py

@@ -19,11 +19,11 @@ def main():
         providers = PCWConfig.get_providers_for("k8sclusters", namespace)
         try:
             if ProviderChoice.GCE in providers:
-                GKE(namespace).cleanup_k8s_jobs()
+                GKE(namespace).cleanup_k8s()
             if ProviderChoice.EC2 in providers:
-                EKS(namespace).cleanup_k8s_jobs()
+                EKS(namespace).cleanup_k8s()
             if ProviderChoice.AZURE in providers:
-                AKS(namespace).cleanup_k8s_jobs()
+                AKS(namespace).cleanup_k8s()
         except Exception:
             logger.exception("[%s] Cleanup failed!", namespace)
 

ocw/lib/aks.py

@@ -1,7 +1,7 @@
 import os
 import kubernetes
 from ocw.lib.provider import Provider
-from ocw.lib.k8s import clean_jobs
+from ocw.lib.k8s import clean_jobs, clean_helm_namespaces
 from webui.PCWConfig import PCWConfig
 
 
@@ -35,14 +35,18 @@ def kubectl_client(self, resource_group: str, cluster_name: str):
                                    f"for resource-group {resource_group} : {res.stderr}")
 
             kubernetes.config.load_kube_config(config_file=kubeconfig)
-            self.__kubectl_client[cluster_name] = kubernetes.client.BatchV1Api()
+            self.__kubectl_client[cluster_name] = kubernetes.client
 
         return self.__kubectl_client[cluster_name]
 
-    def cleanup_k8s_jobs(self):
+    def cleanup_k8s(self):
         clusters = PCWConfig.get_k8s_clusters_for_provider(self._namespace, "azure")
-        self.log_info(f"Cleanup k8s jobs in AKS clusters. {len(clusters)}  will be queried ")
+        self.log_info(f"Cleanup AKS clusters. {len(clusters)}  will be queried ")
         for cluster in clusters:
-            self.log_info(f"Cleanup k8s jobs in AKS cluster {cluster['cluster_name']}")
-            client = self.kubectl_client(cluster["resource_group"], cluster["cluster_name"])
+            self.log_info(f"Cleaning jobs in AKS cluster {cluster['cluster_name']}")
+            client = self.kubectl_client(cluster["resource_group"], cluster["cluster_name"]).BatchV1Api()
             clean_jobs(self, client, cluster["cluster_name"])
+
+            self.log_info(f"Cleaning namespaces in AKS cluster {cluster['cluster_name']}")
+            client = self.kubectl_client(cluster["resource_group"], cluster["cluster_name"]).CoreV1Api()
+            clean_helm_namespaces(self, client, cluster["cluster_name"])

ocw/lib/eks.py

@@ -6,7 +6,7 @@
 import boto3
 from webui.PCWConfig import PCWConfig, ConfigFile
 from ocw.lib.provider import Provider
-from ocw.lib.k8s import clean_jobs
+from ocw.lib.k8s import clean_jobs, clean_helm_namespaces
 
 TAG_IGNORE = 'pcw_ignore'
 
@@ -79,7 +79,7 @@ def kubectl_client(self, region: str, cluster_name: str):
                 raise RuntimeError(f"Cannot get the kubeconfig for the cluster {cluster_name} on region {region}")
 
             kubernetes.config.load_kube_config(config_file=kubeconfig)
-            self.__kubectl_client[region_cluster] = kubernetes.client.BatchV1Api()
+            self.__kubectl_client[region_cluster] = kubernetes.client
 
         return self.__kubectl_client[region_cluster]
 
@@ -154,12 +154,16 @@ def delete_all_clusters(self) -> None:
                     self.log_info(f"Finally deleting {cluster} cluster")
                     self.eks_client(region).delete_cluster(name=cluster)
 
-    def cleanup_k8s_jobs(self):
-        self.log_info("Cleanup k8s jobs in EKS clusters")
+    def cleanup_k8s(self):
+        self.log_info("Cleanup EKS clusters")
         for region in self.__cluster_regions:
             self.log_dbg(f"Region {region}")
             clusters = self.eks_client(region).list_clusters()['clusters']
             for cluster_name in clusters:
-                self.log_info(f"Cleanup k8s jobs in EKS cluster {cluster_name} in region {region}")
-                client = self.kubectl_client(region, cluster_name)
+                self.log_info(f"Cleaning jobs in EKS cluster {cluster_name} in region {region}")
+                client = self.kubectl_client(region, cluster_name).BatchV1Api()
                 clean_jobs(self, client, cluster_name)
+
+                self.log_info(f"Cleaning namespaces in EKS cluster {cluster_name} in region {region}")
+                client = self.kubectl_client(region, cluster_name).CoreV1Api()
+                clean_helm_namespaces(self, client, cluster_name)

ocw/lib/gke.py

@@ -3,7 +3,7 @@
 import googleapiclient.discovery
 from google.oauth2 import service_account
 from ocw.lib.gce import GCE
-from ocw.lib.k8s import clean_jobs
+from ocw.lib.k8s import clean_jobs, clean_helm_namespaces
 
 
 class GKE(GCE):
@@ -45,7 +45,7 @@ def kubectl_client(self, zone, cluster):
             raise FileNotFoundError(f"{kubeconfig} doesn't exists")
 
         kubernetes.config.load_kube_config(config_file=kubeconfig)
-        self.__kubectl_client[zone] = kubernetes.client.BatchV1Api()
+        self.__kubectl_client[zone] = kubernetes.client
         return self.__kubectl_client[zone]
 
     def get_clusters(self, zone):
@@ -56,12 +56,16 @@ def get_clusters(self, zone):
 
         return []
 
-    def cleanup_k8s_jobs(self):
-        self.log_info("Cleanup k8s jobs in GKE clusters")
+    def cleanup_k8s(self):
+        self.log_info("Cleanup GKE clusters")
         for region in self.list_regions():
             for zone in self.list_zones(region):
                 for cluster in self.get_clusters(zone):
                     cluster_name = cluster["name"]
-                    self.log_info(f"Cleanup k8s jobs in GKE cluster {cluster_name} in zone {zone}")
-                    client = self.kubectl_client(zone, cluster)
+                    self.log_info(f"Cleaning jobs in GKE cluster {cluster_name} in zone {zone}")
+                    client = self.kubectl_client(zone, cluster).BatchV1Api()
                     clean_jobs(self, client, cluster_name)
+
+                    self.log_info(f"Cleaning namespaces in GKE cluster {cluster_name} in zone {zone}")
+                    client = self.kubectl_client(zone, cluster).CoreV1Api()
+                    clean_helm_namespaces(self, client, cluster_name)

ocw/lib/k8s.py

@@ -1,5 +1,5 @@
 from datetime import datetime, timezone
-from kubernetes.client import BatchV1Api
+from kubernetes.client import BatchV1Api, CoreV1Api
 from ocw.lib.provider import Provider
 
 
@@ -16,3 +16,21 @@ def clean_jobs(provider: Provider, client: BatchV1Api, cluster_name: str):
             else:
                 provider.log_info(f"Skip deleting from {cluster_name} the job {job.metadata.name} " +
                                   f"with age {age} (days)")
+
+
+def clean_helm_namespaces(provider: Provider, client: CoreV1Api, cluster_name: str):
+    now = datetime.now(timezone.utc)
+    # Retrieve the list of all namespaces
+    namespaces = client.list_namespace(watch=False)
+
+    for ns in namespaces.items:
+        age = (now - ns.metadata.creation_timestamp).days
+        if (ns.metadata.name.startswith('helm-test') and age > 7):
+            # Delete the namespace
+            if not provider.dry_run:
+                provider.log_info(f"Deleting namespace {ns.metadata.name} created {ns.metadata.creation_timestamp}")
+                client.delete_namespace(ns.metadata.name)
+            else:
+                provider.log_info(f"Skip deleting namespace {ns.metadata.name} created {ns.metadata.creation_timestamp}")
+        else:
+            provider.log_info(f"Namespace {ns.metadata.name} will be kept.")

tests/test_aks.py

@@ -34,17 +34,17 @@ def test_kubectl_client(aks_patch, monkeypatch):
     assert aks_patch.kubectl_client("group", "cluster") == mocked_client1
 
 
-def test_cleanup_k8s_jobs(aks_patch, monkeypatch):
+def test_cleanup_k8s(aks_patch, monkeypatch):
     mocked_kubernetes = MockedKubernetesClient([MockedKubernetesJob("job1", 1), MockedKubernetesJob("job2", 0)])
     monkeypatch.setattr(AKS, "kubectl_client", lambda *args, **kwargs: mocked_kubernetes)
     monkeypatch.setattr(PCWConfig, "get_k8s_clusters_for_provider", lambda *args, **kwargs: [
         {'resource_group': 'group', 'cluster_name': 'cluster'}])
-    aks_patch.cleanup_k8s_jobs()
+    aks_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 1
     assert mocked_kubernetes.deleted_jobs[0] == "job1"
 
     # test dry_run
     aks_patch.dry_run = True
     mocked_kubernetes.deleted_jobs = []
-    aks_patch.cleanup_k8s_jobs()
+    aks_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 0

tests/test_eks.py

@@ -171,7 +171,7 @@ def test_delete_all_clusters(eks_patch, monkeypatch):
     assert mocked_eks.deleted_services == []
 
 
-def test_cleanup_k8s_jobs(eks_patch, monkeypatch):
+def test_cleanup_k8s(eks_patch, monkeypatch):
     mocked_eks = MockedEKSClient()
     mocked_eks.clusters_list = {'clusters': ['cluster1']}
     monkeypatch.setattr(EKS, 'eks_client', lambda self, region: mocked_eks)
@@ -180,12 +180,12 @@ def test_cleanup_k8s_jobs(eks_patch, monkeypatch):
     monkeypatch.setattr(kubernetes, 'config', MockedKubernetesConfig())
     mocked_kubernetes = MockedKubernetesClient([MockedKubernetesJob("job1", 1), MockedKubernetesJob("job2", 0)])
     monkeypatch.setattr(EKS, "kubectl_client", lambda *args, **kwargs: mocked_kubernetes)
-    eks_patch.cleanup_k8s_jobs()
+    eks_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 1
     assert mocked_kubernetes.deleted_jobs[0] == "job1"
 
     # test dry_run
     eks_patch.dry_run = True
     mocked_kubernetes.deleted_jobs = []
-    eks_patch.cleanup_k8s_jobs()
+    eks_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 0

tests/test_gke.py

@@ -17,16 +17,16 @@ def k8s_patch(monkeypatch):
     return GKE('fake')
 
 
-def test_cleanup_k8s_jobs(k8s_patch, monkeypatch):
+def test_cleanup_k8s(k8s_patch, monkeypatch):
     monkeypatch.setattr(kubernetes, 'config', MockedKubernetesConfig())
     mocked_kubernetes = MockedKubernetesClient([MockedKubernetesJob("job1", 1), MockedKubernetesJob("job2", 0)])
     monkeypatch.setattr(kubernetes, 'client', mocked_kubernetes)
     monkeypatch.setattr(GKE, 'kubectl_client', lambda *args, **kwargs: mocked_kubernetes)
-    k8s_patch.cleanup_k8s_jobs()
+    k8s_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 1
     assert mocked_kubernetes.deleted_jobs[0] == "job1"
 
     k8s_patch.dry_run = True
     mocked_kubernetes.deleted_jobs = []
-    k8s_patch.cleanup_k8s_jobs()
+    k8s_patch.cleanup_k8s()
     assert len(mocked_kubernetes.deleted_jobs) == 0