storage update

observatory_platform/airflow/tasks.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2023 Curtin University
+# Copyright 2020-2024 Curtin University
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -24,7 +24,6 @@
 from airflow.hooks.base import BaseHook
 from airflow.models import Variable
 
-from observatory_platform.google.gcp import gcp_delete_disk, gcp_create_disk
 from observatory_platform.google.gke import gke_create_volume, gke_delete_volume
 
 
@@ -81,9 +80,7 @@ def check_connections(*connections):
 
 
 @task
-def gke_create_storage(
-    project_id: str, zone: str, volume_name: str, volume_size: int, kubernetes_conn_id: str, **context
-):
+def gke_create_storage(volume_name: str, volume_size: int, kubernetes_conn_id: str, **context):
     """Create storage on a GKE cluster.
 
     :param project_id: the Google Cloud project ID.
@@ -95,12 +92,11 @@ def gke_create_storage(
     :return: None.
     """
 
-    gcp_create_disk(project_id=project_id, zone=zone, disk_name=volume_name, disk_size_gb=volume_size)
     gke_create_volume(kubernetes_conn_id=kubernetes_conn_id, volume_name=volume_name, size_gi=volume_size)
 
 
 @task
-def gke_delete_storage(project_id: str, zone: str, volume_name: str, kubernetes_conn_id: str, **context):
+def gke_delete_storage(volume_name: str, kubernetes_conn_id: str, **context):
     """Delete storage on a GKE cluster.
 
     :param project_id: the Google Cloud project ID.
@@ -112,4 +108,3 @@ def gke_delete_storage(project_id: str, zone: str, volume_name: str, kubernetes_
     """
 
     gke_delete_volume(kubernetes_conn_id=kubernetes_conn_id, volume_name=volume_name)
-    gcp_delete_disk(project_id=project_id, zone=zone, disk_name=volume_name)

observatory_platform/google/gke.py

@@ -19,13 +19,12 @@
 from kubernetes import client
 
 
-def gke_create_volume(*, kubernetes_conn_id: str, volume_name: str, size_gi: int):
-    """
+def gke_create_volume(*, kubernetes_conn_id: str, volume_name: str, size_gi: int) -> None:
+    """Creates a GKE volume
 
     :param kubernetes_conn_id:
     :param volume_name:
     :param size_gi:
-    :return: None.
     """
 
     # Make Kubernetes API Client from Airflow Connection
@@ -35,26 +34,10 @@ def gke_create_volume(*, kubernetes_conn_id: str, volume_name: str, size_gi: int
 
     # Create the PersistentVolume
     capacity = {"storage": f"{size_gi}Gi"}
-    pv = client.V1PersistentVolume(
-        api_version="v1",
-        kind="PersistentVolume",
-        metadata=client.V1ObjectMeta(
-            name=volume_name,
-            # TODO: supposed to use this user for the persistent volume but doesn't seem to do anything
-            # annotations={"pv.beta.kubernetes.io/uid": f"{uid}", "pv.beta.kubernetes.io/gid": f"{uid}"}
-        ),
-        spec=client.V1PersistentVolumeSpec(
-            capacity=capacity,
-            access_modes=["ReadWriteOnce"],
-            persistent_volume_reclaim_policy="Retain",
-            storage_class_name="standard",
-            gce_persistent_disk=client.V1GCEPersistentDiskVolumeSource(pd_name=volume_name),
-        ),
-    )
-    v1.create_persistent_volume(body=pv)
 
     # Create PersistentVolumeClaim
     namespace = hook.get_namespace()
+    namespace = "coki-astro"  # TODO: Figure out how to fix this
     pvc = client.V1PersistentVolumeClaim(
         api_version="v1",
         kind="PersistentVolumeClaim",
@@ -68,13 +51,12 @@ def gke_create_volume(*, kubernetes_conn_id: str, volume_name: str, size_gi: int
     v1.create_namespaced_persistent_volume_claim(namespace=namespace, body=pvc)
 
 
-def gke_delete_volume(*, kubernetes_conn_id: str, volume_name: str):
-    """
+def gke_delete_volume(*, kubernetes_conn_id: str, volume_name: str) -> None:
+    """Deletes a GKE volume
 
     :param kubernetes_conn_id:
     :param namespace:
     :param volume_name:
-    :return: None.
     """
 
     # Make Kubernetes API Client from Airflow Connection
@@ -84,6 +66,7 @@ def gke_delete_volume(*, kubernetes_conn_id: str, volume_name: str):
 
     # Delete VolumeClaim and Volume
     namespace = hook.get_namespace()
+    namespace = "coki-astro"  # TODO: Figure out how to fix this
     try:
         v1.delete_namespaced_persistent_volume_claim(name=volume_name, namespace=namespace)
     except kubernetes.client.exceptions.ApiException as e:
@@ -93,11 +76,3 @@ def gke_delete_volume(*, kubernetes_conn_id: str, volume_name: str):
             )
         else:
             raise e
-
-    try:
-        v1.delete_persistent_volume(name=volume_name)
-    except kubernetes.client.exceptions.ApiException as e:
-        if e.status == 404:
-            logging.info(f"gke_delete_volume: PersistentVolume with name={volume_name} does not exist")
-        else:
-            raise e