Merge pull request #378 from ricardobranco777/gce_cleanup_blobs

gce: Cleanup buckets

ocw/lib/gce.py

@@ -6,7 +6,7 @@
 import googleapiclient.discovery
 from googleapiclient.errors import HttpError
 from google.oauth2 import service_account
-from webui.PCWConfig import ConfigFile
+from webui.PCWConfig import ConfigFile, PCWConfig
 from .provider import Provider
 
 
@@ -21,9 +21,14 @@ def __new__(cls, namespace):
     def __init__(self, namespace):
         super().__init__(namespace)
 
+        try:
+            self.__bucket = PCWConfig.get_feature_property('cleanup', 'gce-bucket', namespace)
+        except LookupError:
+            self.__bucket = None
         self.__skip_networks = frozenset(ConfigFile().getList('cleanup/gce-skip-networks', ["default"]))
 
         self.__compute_client = None
+        self.__storage_client = None
         self.private_key_data = self.get_data()
         self.project = self.private_key_data["project_id"]
 
@@ -41,6 +46,7 @@ def _paginated(self, api_call, **kwargs) -> list:
 
     def _delete_resource(self, api_call, resource_name, *_, **kwargs) -> None:
         resource_type = {
+            self.storage_client().objects: "blob",
             self.compute_client().disks: "disk",
             self.compute_client().firewalls: "firewall",
             self.compute_client().forwardingRules: "forwardingRule",
@@ -87,6 +93,14 @@ def compute_client(self):
             )
         return self.__compute_client
 
+    def storage_client(self):
+        if self.__storage_client is None:
+            credentials = service_account.Credentials.from_service_account_info(self.private_key_data)
+            self.__storage_client = googleapiclient.discovery.build(
+                "storage", "v1", credentials=credentials, cache_discovery=False
+            )
+        return self.__storage_client
+
     def list_instances(self, zone) -> list:
         """ List all instances by zone."""
         self.log_dbg(f"Call list_instances for {zone}")
@@ -134,6 +148,8 @@ def get_error_reason(error: "googleapiclient.errors.HttpError") -> str:
 
     def cleanup_all(self) -> None:
         self.log_info("Call cleanup_all")
+        if self.__bucket is not None:
+            self.cleanup_blobs()
         self.cleanup_disks()
         self.cleanup_images()
         self.cleanup_firewalls()
@@ -142,6 +158,16 @@ def cleanup_all(self) -> None:
         self.cleanup_subnetworks()
         self.cleanup_networks()
 
+    def cleanup_blobs(self) -> None:
+        self.log_dbg("Blobs cleanup")
+        blobs = self._paginated(self.storage_client().objects, bucket=self.__bucket)
+        self.log_dbg(f"{len(blobs)} blobs found")
+        for blob in blobs:
+            if self.is_outdated(parse(blob["timeCreated"]).astimezone(timezone.utc)):
+                self._delete_resource(
+                    self.storage_client().objects, blob["name"], bucket=self.__bucket, object=blob["name"]
+                )
+
     def cleanup_disks(self) -> None:
         self.log_dbg("Disks cleanup")
         for region in self.list_regions():

templates/pcw.ini

@@ -56,6 +56,8 @@ azure-storage-resourcegroup = openqa-upload
 azure-storage-account-name = openqa
 # When set to true EC2 VPC cleanup will be enabled
 vpc_cleanup = true
+# GCE bucket to be cleaned up
+gce_bucket = bucket
 
 [updaterun]
 # if openqa_ttl tag is not defined this TTL will be set to the instance

tests/test_gce.py

@@ -40,7 +40,7 @@ def list_next(self, *args, **kwargs):
         return self.responses.pop(0)
 
     def delete(self, *args, **kwargs):
-        for resource in ('image', 'disk', 'instance', 'firewall', 'forwardingRule', 'route', 'network', 'subnetwork'):
+        for resource in ('object', 'image', 'disk', 'instance', 'firewall', 'forwardingRule', 'route', 'network', 'subnetwork'):
             if resource in kwargs:
                 if self.error_reason:
                     return MockRequest(error_reason=self.error_reason)
@@ -58,6 +58,7 @@ def something():
 
 
 class MockClient:
+    def objects(self): pass
     def disks(self): pass
     def firewalls(self): pass
     def forwardingRules(self): pass
@@ -76,6 +77,7 @@ def gce():
         patch.object(GCE, 'read_auth_json', return_value={}),
     ):
         gce = GCE('fake')
+        gce.storage_client = MockClient
         gce.compute_client = MockClient
         gce.compute_client.instances = MockResource([MockRequest({'items': ['instance1', 'instance2']}), None])
         yield gce
@@ -98,14 +100,14 @@ def mocked_resource():
     return MockResource([
         MockRequest({   # on images().list()
             'items': [
-                {'name': 'keep', 'creationTimestamp': now_age, 'network': 'mynetwork'},
-                {'name': 'delete1', 'creationTimestamp': older_than_max_age, 'network': 'mynetwork'}
+                {'name': 'keep', 'creationTimestamp': now_age, 'timeCreated': now_age, 'network': 'mynetwork'},
+                {'name': 'delete1', 'creationTimestamp': older_than_max_age, 'timeCreated': older_than_max_age, 'network': 'mynetwork'}
             ], 'id': "id"}),
         MockRequest(),  # on images().delete()
         MockRequest({   # on images().list_next()
             'items': [
-                {'name': 'keep', 'creationTimestamp': now_age, 'network': 'mynetwork'},
-                {'name': 'delete2', 'creationTimestamp': older_than_max_age, 'network': 'mynetwork'}
+                {'name': 'keep', 'creationTimestamp': now_age, 'timeCreated': now_age, 'network': 'mynetwork'},
+                {'name': 'delete2', 'creationTimestamp': older_than_max_age, 'timeCreated': older_than_max_age, 'network': 'mynetwork'}
             ], 'id': "id"}),
         MockRequest({'error': {'errors': [{'message': 'err message'}]},
                     'warnings': [{'message': 'warning message'}]}),
@@ -150,13 +152,20 @@ def _test_cleanup(gce, resource_type, cleanup_call, resources):
         patch.object(gce, 'list_zones', return_value=['zone1']),
     ):
         setattr(gce.compute_client, resource_type, resources)
+        setattr(gce.storage_client, resource_type, resources)
         cleanup_call()
         if gce.dry_run:
             assert resources.deleted_resources == []
         else:
             assert resources.deleted_resources == ['delete1', 'delete2']
 
 
+@mark.parametrize("dry_run", [True, False])
+def test_cleanup_blobs(gce, mocked_resource, dry_run):
+    gce.dry_run = dry_run
+    _test_cleanup(gce, "objects", gce.cleanup_blobs, mocked_resource)
+
+
 @mark.parametrize("dry_run", [True, False])
 def test_cleanup_disks(gce, mocked_resource, dry_run):
     gce.dry_run = dry_run
@@ -214,6 +223,7 @@ def test_cleanup_networks(gce, mocked_resource, dry_run):
 
 
 def test_cleanup_all(gce):
+    gce.cleanup_blobs = MagicMock()
     gce.cleanup_disks = MagicMock()
     gce.cleanup_images = MagicMock()
     gce.cleanup_firewalls = MagicMock()

webui/PCWConfig.py

@@ -55,11 +55,12 @@ class PCWConfig():
     @staticmethod
     def get_feature_property(feature: str, feature_property: str, namespace: str | None = None) -> str | int:
         default_values: dict[str, dict[str, int | type[int] | str | type[str] | type[str] | None]] = {
-            'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
             'cleanup/azure-gallery-name': {'default': 'test_image_gallery', 'return_type': str},
             'cleanup/azure-storage-resourcegroup': {'default': 'openqa-upload', 'return_type': str},
             'cleanup/azure-storage-account-name': {'default': 'openqa', 'return_type': str},
             'cleanup/ec2-max-age-days': {'default': -1, 'return_type': int},
+            'cleanup/gce-bucket': {'default': None, 'return_type': str},
+            'cleanup/max-age-hours': {'default': 24 * 7, 'return_type': int},
             'cleanup/openstack-image-max-age-days': {'default': 3, 'return_type': int},
             'cleanup/openstack-vm-max-age-days': {'default': 1, 'return_type': int},
             'cleanup/openstack-key-max-days': {'default': 1, 'return_type': int},