Merge pull request #397 from SUSE/gce_dump

Add dump stat for GCE

ocw/lib/dump_state.py

@@ -4,6 +4,7 @@
 from webui.PCWConfig import PCWConfig
 from ocw.lib.azure import Azure
 from ocw.lib.ec2 import EC2
+from ocw.lib.gce import GCE
 from ocw.enums import ProviderChoice
 from ocw.lib.influx import Influx
 
@@ -65,6 +66,43 @@ def dump_state():
                     namespace,
                     EC2(namespace).count_all_volumes
                 )
+                Influx().dump_resource(
+                    ProviderChoice.EC2.value,
+                    Influx.VPC_QUANTITY,
+                    namespace,
+                    EC2(namespace).count_all_vpc
+                )
+            if ProviderChoice.GCE in providers:
+                Influx().dump_resource(
+                    ProviderChoice.GCE.value,
+                    Influx.VMS_QUANTITY,
+                    namespace,
+                    GCE(namespace).count_all_instances
+                )
+                Influx().dump_resource(
+                    ProviderChoice.GCE.value,
+                    Influx.IMAGES_QUANTITY,
+                    namespace,
+                    GCE(namespace).count_all_images
+                )
+                Influx().dump_resource(
+                    ProviderChoice.GCE.value,
+                    Influx.DISK_QUANTITY,
+                    namespace,
+                    GCE(namespace).count_all_disks
+                )
+                Influx().dump_resource(
+                    ProviderChoice.GCE.value,
+                    Influx.BLOB_QUANTITY,
+                    namespace,
+                    GCE(namespace).count_all_blobs
+                )
+                Influx().dump_resource(
+                    ProviderChoice.GCE.value,
+                    Influx.NETWORK_QUANTITY,
+                    namespace,
+                    GCE(namespace).count_all_networks
+                )
         except Exception:
             logger.exception(
                 "[%s] Dump state failed!: \n %s", namespace, traceback.format_exc()

ocw/lib/ec2.py

@@ -345,6 +345,13 @@ def count_all_volumes(self) -> int:
             all_volumes_cnt += len(response['Volumes'])
         return all_volumes_cnt
 
+    def count_all_vpc(self) -> int:
+        all_vpcs = 0
+        for region in self.all_regions:
+            response = self.ec2_client(region).describe_vpcs(Filters=[{'Name': 'isDefault', 'Values': ['false']}])
+            all_vpcs += len(response['Vpcs'])
+        return all_vpcs
+
     def cleanup_images(self, valid_period_days: float) -> None:
         self.log_dbg('Call cleanup_images')
         for region in self.all_regions:

ocw/lib/gce.py

@@ -264,3 +264,22 @@ def cleanup_networks(self) -> None:
                 self._delete_resource(
                     self.compute_client().networks, network["name"], project=self.project, network=network["name"]
                 )
+
+    def count_all_instances(self) -> int:
+        return len(self.list_all_instances())
+
+    def count_all_images(self) -> int:
+        return len(self._paginated(self.compute_client().images, project=self.project))
+
+    def count_all_disks(self) -> int:
+        all_disks = 0
+        for region in self.list_regions():
+            for zone in self.list_zones(region):
+                all_disks += len(self._paginated(self.compute_client().disks, project=self.project, zone=zone))
+        return all_disks
+
+    def count_all_blobs(self) -> int:
+        return len(self._paginated(self.storage_client().objects, bucket=self.__bucket))
+
+    def count_all_networks(self) -> int:
+        return len(self._paginated(self.compute_client().networks, project=self.project))

ocw/lib/influx.py

@@ -17,8 +17,11 @@ class Influx:
     VMS_QUANTITY: str = "vms_quantity"
     IMAGES_QUANTITY: str = "images_quantity"
     DISK_QUANTITY: str = "disk_quantity"
+    BLOB_QUANTITY: str = "blob_quantity"
     VOLUMES_QUANTITY: str = "volumes_quanity"
     IMAGE_VERSION_QUANTITY: str = "img_version_quantity"
+    VPC_QUANTITY: str = "vpc_quantity"
+    NETWORK_QUANTITY: str = "network_quantity"
     NAMESPACE_TAG: str = "namespace"
 
     def __init__(self) -> None:

tests/test_ec2.py

@@ -532,3 +532,7 @@ def test_count_all_volumes(ec2_patch):
                      'Tags': [{'Key': 'pcw_ignore', 'Value': '1'}]}, ]
     }
     assert ec2_patch.count_all_volumes() == 3
+
+
+def test_count_all_vpcs(ec2_patch_for_vpc):
+    assert ec2_patch_for_vpc.count_all_vpc() == 1

tests/test_gce.py

@@ -253,3 +253,35 @@ def __init__(self, content) -> None:
     assert GCE.get_error_reason(MockHttpError({'error': {'errors': []}})) == "unknown"
     assert GCE.get_error_reason(MockHttpError({'error': {'errors': [{}]}})) == "unknown"
     assert GCE.get_error_reason(MockHttpError({'error': {'errors': [{'reason': 'aaa'}]}})) == "aaa"
+
+
+def test_count_all_instances(gce):
+    with (
+        patch.object(gce, 'list_regions', return_value=['region1']),
+        patch.object(gce, 'list_zones', return_value=['zone1']),
+    ):
+        assert gce.count_all_instances() == 2
+
+
+def test_count_all_images(gce):
+    with (patch.object(gce, '_paginated', return_value=[1, 2, 3, 4])):
+        assert gce.count_all_images() == 4
+
+
+def test_count_all_disks(gce):
+    with (
+        patch.object(gce, 'list_regions', return_value=['region1']),
+        patch.object(gce, 'list_zones', return_value=['zone1']),
+        patch.object(gce, '_paginated', return_value=[1, 2, 3, 4]),
+    ):
+        assert gce.count_all_disks() == 4
+
+
+def test_count_all_blobs(gce):
+    with (patch.object(gce, '_paginated', return_value=[1, 2, 3, 4])):
+        assert gce.count_all_blobs() == 4
+
+
+def test_count_all_networks(gce):
+    with (patch.object(gce, '_paginated', return_value=[1, 2, 3, 4])):
+        assert gce.count_all_networks() == 4