feat: add metric data query for all services

Signed-off-by: ImMin5 <mino@megazone.com>

.gitignore

@@ -12,3 +12,4 @@ local-conf.yml
 .venv/
 .venv.nosync/
 .DS_Store
+!src/spaceone/inventory/metrics/Disks/disk

Dockerfile

@@ -1,4 +1,4 @@
-FROM cloudforet/python-core:1
+FROM cloudforet/python-core:2
 
 ENV PYTHONUNBUFFERED 1
 ENV CLOUDONE_PORT 50051
@@ -20,4 +20,4 @@ RUN python3 setup.py install && \
 EXPOSE ${CLOUDONE_PORT}
 
 ENTRYPOINT ["spaceone"]
-CMD ["grpc", "spaceone.inventory"]
+CMD ["run", "grpc-server", "spaceone.inventory"]

pkg/pip_requirements.txt

@@ -1,6 +1,4 @@
 schematics
-adal
-msrestazure
 azure-identity
 azure-mgmt-resource
 azure-mgmt-compute
@@ -14,4 +12,5 @@ azure-keyvault-secrets
 azure-mgmt-rdbms
 azure-mgmt-cosmosdb
 azure-mgmt-containerinstance
-azure-mgmt-webpubsub
+azure-mgmt-webpubsub
+azure-mgmt-resource

src/setup.py

@@ -50,6 +50,11 @@
         "azure-mgmt-containerinstance",
         "azure-mgmt-webpubsub",
     ],
-    package_data={"spaceone": ["inventory/model/*/widget/*.yaml"]},
+    package_data={
+        "spaceone": [
+            "inventory/model/*/widget/*.yaml",
+            "inventory/metrics/**/**/*.yaml",
+        ]
+    },
     zip_safe=False,
 )

src/spaceone/inventory/conf/cloud_service_conf.py

@@ -1,29 +1,33 @@
 MAX_WORKER = 20
-SUPPORTED_FEATURES = ['garbage_collection']
-SUPPORTED_SCHEDULES = ['hours']
-SUPPORTED_RESOURCE_TYPE = ['inventory.CloudService', 'inventory.CloudServiceType', 'inventory.Region']
+SUPPORTED_FEATURES = ["garbage_collection"]
+SUPPORTED_SCHEDULES = ["hours"]
+SUPPORTED_RESOURCE_TYPE = [
+    "inventory.CloudService",
+    "inventory.CloudServiceType",
+    "inventory.Region",
+]
 FILTER_FORMAT = []
 
-ASSET_URL = 'https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/azure'
+ASSET_URL = "https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/azure"
 
 CLOUD_SERVICE_GROUP_MAP = {
-    'ApplicationGateways': 'ApplicationGatewaysManager',
-    'ContainerInstances': 'ContainerInstancesManager',
-    'CosmosDB': 'CosmosDBManager',
-    'Disks': 'DisksManager',
-    'KeyVaults': 'KeyVaultsManager',
-    'LoadBalancers': 'LoadBalancersManager',
-    'MySQLServers': 'MySQLServersManager',
-    'SQLServers': 'SQLServersManager',
-    'SQLDatabases': 'SQLDatabasesManager',
-    'NATGateways': 'NATGatewaysManager',
-    'NetworkSecurityGroups': 'NetworkSecurityGroupsManager',
-    'PostgreSQLServers': 'PostgreSQLServersManager',
-    'PublicIPAddresses': 'PublicIPAddressesManager',
-    'Snapshots': 'SnapshotsManager',
-    'StorageAccounts': 'StorageAccountsManager',
-    'VirtualMachines': 'VirtualMachinesManager',
-    'VirtualNetworks': 'VirtualNetworksManager',
-    'VMScaleSets': 'VmScaleSetsManager',
-    'WebPubSubService': 'WebPubSubServiceManager',
+    "VirtualMachines": "VirtualMachinesManager",
+    "ApplicationGateways": "ApplicationGatewaysManager",
+    "ContainerInstances": "ContainerInstancesManager",
+    "CosmosDB": "CosmosDBManager",
+    "Disks": "DisksManager",
+    "KeyVaults": "KeyVaultsManager",
+    "LoadBalancers": "LoadBalancersManager",
+    "MySQLServers": "MySQLServersManager",
+    "SQLServers": "SQLServersManager",
+    "SQLDatabases": "SQLDatabasesManager",
+    "NATGateways": "NATGatewaysManager",
+    "NetworkSecurityGroups": "NetworkSecurityGroupsManager",
+    "PostgreSQLServers": "PostgreSQLServersManager",
+    "PublicIPAddresses": "PublicIPAddressesManager",
+    "Snapshots": "SnapshotsManager",
+    "StorageAccounts": "StorageAccountsManager",
+    "VirtualNetworks": "VirtualNetworksManager",
+    "VMScaleSets": "VmScaleSetsManager",
+    "WebPubSubService": "WebPubSubServiceManager",
 }

src/spaceone/inventory/connector/__init__.py

@@ -5,9 +5,13 @@
 from spaceone.inventory.connector.load_balancers import LoadBalancersConnector
 from spaceone.inventory.connector.monitor import MonitorConnector
 from spaceone.inventory.connector.virtual_networks import VirtualNetworksConnector
-from spaceone.inventory.connector.application_gateways import ApplicationGatewaysConnector
+from spaceone.inventory.connector.application_gateways import (
+    ApplicationGatewaysConnector,
+)
 from spaceone.inventory.connector.public_ip_addresses import PublicIPAddressesConnector
-from spaceone.inventory.connector.network_security_groups import NetworkSecurityGroupsConnector
+from spaceone.inventory.connector.network_security_groups import (
+    NetworkSecurityGroupsConnector,
+)
 from spaceone.inventory.connector.nat_gateways import NATGatewaysConnector
 from spaceone.inventory.connector.storage_accounts import StorageAccountsConnector
 from spaceone.inventory.connector.key_vaults import KeyVaultsConnector
@@ -19,3 +23,4 @@
 from spaceone.inventory.connector.sql_databases import SQLDatabasesConnector
 from spaceone.inventory.connector.container_instances import ContainerInstancesConnector
 from spaceone.inventory.connector.web_pubsub_service import WebPubSubServiceConnector
+from spaceone.inventory.connector.resources import ResourcesConnector

src/spaceone/inventory/connector/key_vaults/connector.py

@@ -1,39 +1,50 @@
 import logging
 
+import azure.core.exceptions
+
 from spaceone.inventory.libs.connector import AzureConnector
 from spaceone.inventory.error.custom import *
 from azure.keyvault.secrets import SecretClient
 from azure.keyvault.certificates import CertificateClient
 from azure.identity import DefaultAzureCredential
 
-__all__ = ['KeyVaultsConnector']
+__all__ = ["KeyVaultsConnector"]
 _LOGGER = logging.getLogger(__name__)
 
 
 class KeyVaultsConnector(AzureConnector):
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.set_connect(kwargs.get('secret_data'))
-        self.key_vault_secret_client = None
-        self.key_vault_certificate_client = None
+        self.set_connect(kwargs.get("secret_data"))
+        # self.key_vault_secret_client = None
+        # self.key_vault_certificate_client = None
 
     def init_key_vault_secret_client(self, subscription_id, vault_uri):
         credential = DefaultAzureCredential()
-        key_vault_secret_client = SecretClient(credential=credential,  subscription_id=subscription_id, vault_url=vault_uri)
+        key_vault_secret_client = SecretClient(
+            credential=credential, subscription_id=subscription_id, vault_url=vault_uri
+        )
         return key_vault_secret_client
 
     def init_key_vault_certificate_client(self, subscription_id, vault_uri):
         credential = DefaultAzureCredential()
-        key_vault_certificate_client = CertificateClient(credential=credential,  subscription_id=subscription_id, vault_url=vault_uri)
+        key_vault_certificate_client = CertificateClient(
+            credential=credential, subscription_id=subscription_id, vault_url=vault_uri
+        )
         return key_vault_certificate_client
 
     def list_all_key_vaults(self):
         return self.key_vault_client.vaults.list_by_subscription()
 
+    def get_key_vaults(self):
+        return self.key_vault_client.vaults.get()
+
     def list_keys(self, resource_group_name, vault_name):
-        return self.key_vault_client.keys.list(resource_group_name=resource_group_name, vault_name=vault_name)
+        return self.key_vault_client.keys.list(
+            resource_group_name=resource_group_name, vault_name=vault_name
+        )
 
     def list_secrets(self):
-        return self.key_vault_secret_client.list_properties_of_secrets()
+        # return self.key_vault_secrets_client.list_properties_of_secrets()
+        return self.key_vault_client.secrets.list()

src/spaceone/inventory/connector/postgresql_servers/connector.py

@@ -2,28 +2,35 @@
 
 from spaceone.inventory.libs.connector import AzureConnector
 from spaceone.inventory.error.custom import *
-__all__ = ['PostgreSQLServersConnector']
+
+__all__ = ["PostgreSQLServersConnector"]
 _LOGGER = logging.getLogger(__name__)
 
 
 class PostgreSQLServersConnector(AzureConnector):
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.set_connect(kwargs.get('secret_data'))
+        self.set_connect(kwargs.get("secret_data"))
 
     def list_servers(self):
         return self.postgre_sql_client.servers.list()
 
     def list_firewall_rules_by_server(self, resource_group_name, server_name):
-        return self.postgre_sql_client.firewall_rules.list_by_server(resource_group_name=resource_group_name, server_name=server_name)
+        return self.postgre_sql_client.firewall_rules.list_by_server(
+            resource_group_name=resource_group_name, server_name=server_name
+        )
 
     def list_virtual_network_rules_by_server(self, resource_group_name, server_name):
-        return self.postgre_sql_client.virtual_network_rules.list_by_server(resource_group_name=resource_group_name,
-                                                                            server_name=server_name)
+        return self.postgre_sql_client.virtual_network_rules.list_by_server(
+            resource_group_name=resource_group_name, server_name=server_name
+        )
 
     def list_replicas_by_server(self, resource_group_name, server_name):
-        return self.postgre_sql_client.replicas.list_by_server(resource_group_name=resource_group_name, server_name=server_name)
+        return self.postgre_sql_client.replicas.list_by_server(
+            resource_group_name=resource_group_name, server_name=server_name
+        )
 
     def list_server_administrators(self, resource_group_name, server_name):
-        return self.postgre_sql_client.server_administrators.list(resource_group_name=resource_group_name, server_name=server_name)
+        return self.postgre_sql_client.server_administrators.list(
+            resource_group_name=resource_group_name, server_name=server_name
+        )

src/spaceone/inventory/connector/resources/__init__.py

@@ -0,0 +1 @@
+from spaceone.inventory.connector.resources.connector import ResourcesConnector

src/spaceone/inventory/connector/resources/connector.py

@@ -0,0 +1,16 @@
+import logging
+
+from spaceone.inventory.libs.connector import AzureConnector
+from spaceone.inventory.error import *
+
+__all__ = ["ResourcesConnector"]
+_LOGGER = logging.getLogger(__name__)
+
+
+class ResourcesConnector(AzureConnector):
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.set_connect(kwargs.get("secret_data"))
+
+    def list_resources(self) -> list:
+        return self.resource_client.resources.list()

src/spaceone/inventory/connector/storage_accounts/connector.py

@@ -1,21 +1,20 @@
 import logging
 
 from spaceone.inventory.libs.connector import AzureConnector
-from spaceone.inventory.error.custom import *
 
-__all__ = ['StorageAccountsConnector']
+__all__ = ["StorageAccountsConnector"]
 _LOGGER = logging.getLogger(__name__)
 
 
 class StorageAccountsConnector(AzureConnector):
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.set_connect(kwargs.get('secret_data'))
+        self.set_connect(kwargs.get("secret_data"))
 
     def list_storage_accounts(self):
         return self.storage_client.storage_accounts.list()
 
     def list_blob_containers(self, rg_name, account_name):
-        return self.storage_client.blob_containers.list(resource_group_name=rg_name, account_name=account_name)
-
+        return self.storage_client.blob_containers.list(
+            resource_group_name=rg_name, account_name=account_name
+        )

src/spaceone/inventory/connector/vm_scale_sets/connector.py

@@ -3,26 +3,42 @@
 from spaceone.inventory.libs.connector import AzureConnector
 from spaceone.inventory.error.custom import *
 
-__all__ = ['VmScaleSetsConnector']
+__all__ = ["VmScaleSetsConnector"]
 _LOGGER = logging.getLogger(__name__)
 
 
 class VmScaleSetsConnector(AzureConnector):
-
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
-        self.set_connect(kwargs.get('secret_data'))
+        self.set_connect(kwargs.get("secret_data"))
 
     def list_vm_scale_sets(self):
         return self.compute_client.virtual_machine_scale_sets.list_all()
 
     def list_vm_scale_set_vms(self, resource_group, vm_scale_set_name):
-        return self.compute_client.virtual_machine_scale_set_vms.list(resource_group, vm_scale_set_name)
-
-    def get_vm_scale_set_instance_view(self, resource_group, vm_scale_set_name, instance_id):
-        return self.compute_client.virtual_machine_scale_set_vms.get_instance_view(resource_group_name=resource_group,
-                                                                                   vm_scale_set_name=vm_scale_set_name,
-                                                                                   instance_id=instance_id)
+        return self.compute_client.virtual_machine_scale_set_vms.list(
+            resource_group, vm_scale_set_name
+        )
+
+    def get_vm_scale_set_instance_view(
+        self, resource_group, vm_scale_set_name, instance_id
+    ):
+        return self.compute_client.virtual_machine_scale_set_vms.get_instance_view(
+            resource_group_name=resource_group,
+            vm_scale_set_name=vm_scale_set_name,
+            instance_id="0",
+        )
+
+    def list_vm_scale_set_instance_view(
+        self, resource_group, vm_scale_set_name, instance_id
+    ):
+        return self.compute_client.virtual_machine_scale_set_vms.list(
+            resource_group_name=resource_group,
+            vm_scale_set_name=vm_scale_set_name,
+            instance_id=instance_id,
+        )
 
     def list_auto_scale_settings(self, resource_group):
-        return self.monitor_client.autoscale_settings.list_by_resource_group(resource_group_name=resource_group)
+        return self.monitor_client.autoscale_settings.list_by_resource_group(
+            resource_group_name=resource_group
+        )

src/spaceone/inventory/info/__init__.py

@@ -1,2 +1,3 @@
 from spaceone.inventory.info.collector_info import *
+from spaceone.inventory.info.job_info import *
 from spaceone.inventory.info.common_info import *

src/spaceone/inventory/info/job_info.py

@@ -0,0 +1,18 @@
+__all__ = ["TasksInfo", "TaskInfo"]
+
+import functools
+from spaceone.api.inventory.plugin import job_pb2
+from spaceone.core.pygrpc.message_type import *
+
+
+def TaskInfo(task_data):
+    info = {"task_options": change_struct_type(task_data["task_options"])}
+    return job_pb2.TaskInfo(**info)
+
+
+def TasksInfo(result, **kwargs):
+    tasks_data = result.get("tasks", [])
+
+    return job_pb2.TasksInfo(
+        tasks=list(map(functools.partial(TaskInfo, **kwargs), tasks_data)),
+    )

src/spaceone/inventory/interface/grpc/__init__.py

@@ -0,0 +1,9 @@
+from spaceone.core.pygrpc.server import GRPCServer
+from spaceone.inventory.interface.grpc.collector import Collector
+from spaceone.inventory.interface.grpc.job import Job
+
+_all_ = ["app"]
+
+app = GRPCServer()
+app.add_service(Collector)
+app.add_service(Job)

src/spaceone/inventory/api/plugin/collector.py → src/spaceone/inventory/interface/grpc/collector.py

@@ -15,25 +15,27 @@ class Collector(BaseAPI, collector_pb2_grpc.CollectorServicer):
     def init(self, request, context):
         params, metadata = self.parse_request(request, context)
 
-        with self.locator.get_service('CollectorService', metadata) as collector_svc:
+        with self.locator.get_service("CollectorService", metadata) as collector_svc:
             data = collector_svc.init(params)
-            return self.locator.get_info('PluginInfo', data)
+            return self.locator.get_info("PluginInfo", data)
 
     def verify(self, request, context):
         params, metadata = self.parse_request(request, context)
 
-        collector_svc: CollectorService = self.locator.get_service('CollectorService', metadata)
+        collector_svc: CollectorService = self.locator.get_service(
+            "CollectorService", metadata
+        )
 
         with collector_svc:
             collector_svc.verify(params)
-            return self.locator.get_info('EmptyInfo')
+            return self.locator.get_info("EmptyInfo")
 
     def collect(self, request, context):
         params, metadata = self.parse_request(request, context)
-        collector_svc: CollectorService = self.locator.get_service('CollectorService', metadata)
+        collector_svc: CollectorService = self.locator.get_service(
+            "CollectorService", metadata
+        )
 
         with collector_svc:
             for resource in collector_svc.collect(params):
-                yield self.locator.get_info('ResourceInfo', resource.to_primitive())
-
-
+                yield self.locator.get_info("ResourceInfo", resource)