Skip to content

Commit

Permalink
Merge pull request #88 from ImMin5/master
Browse files Browse the repository at this point in the history
Add metric data query for all services
  • Loading branch information
ImMin5 authored Jul 2, 2024
2 parents ce450ed + 9be3ebe commit 0cf9b77
Show file tree
Hide file tree
Showing 126 changed files with 9,223 additions and 4,085 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ local-conf.yml
.venv/
.venv.nosync/
.DS_Store
!src/spaceone/inventory/metrics/Disks/disk
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM cloudforet/python-core:1
FROM cloudforet/python-core:2

ENV PYTHONUNBUFFERED 1
ENV CLOUDONE_PORT 50051
Expand All @@ -20,4 +20,4 @@ RUN python3 setup.py install && \
EXPOSE ${CLOUDONE_PORT}

ENTRYPOINT ["spaceone"]
CMD ["grpc", "spaceone.inventory"]
CMD ["run", "grpc-server", "spaceone.inventory"]
5 changes: 2 additions & 3 deletions pkg/pip_requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
schematics
adal
msrestazure
azure-identity
azure-mgmt-resource
azure-mgmt-compute
Expand All @@ -14,4 +12,5 @@ azure-keyvault-secrets
azure-mgmt-rdbms
azure-mgmt-cosmosdb
azure-mgmt-containerinstance
azure-mgmt-webpubsub
azure-mgmt-webpubsub
azure-mgmt-resource
7 changes: 6 additions & 1 deletion src/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@
"azure-mgmt-containerinstance",
"azure-mgmt-webpubsub",
],
package_data={"spaceone": ["inventory/model/*/widget/*.yaml"]},
package_data={
"spaceone": [
"inventory/model/*/widget/*.yaml",
"inventory/metrics/**/**/*.yaml",
]
},
zip_safe=False,
)
Empty file.
50 changes: 27 additions & 23 deletions src/spaceone/inventory/conf/cloud_service_conf.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,33 @@
MAX_WORKER = 20
SUPPORTED_FEATURES = ['garbage_collection']
SUPPORTED_SCHEDULES = ['hours']
SUPPORTED_RESOURCE_TYPE = ['inventory.CloudService', 'inventory.CloudServiceType', 'inventory.Region']
SUPPORTED_FEATURES = ["garbage_collection"]
SUPPORTED_SCHEDULES = ["hours"]
SUPPORTED_RESOURCE_TYPE = [
"inventory.CloudService",
"inventory.CloudServiceType",
"inventory.Region",
]
FILTER_FORMAT = []

ASSET_URL = 'https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/azure'
ASSET_URL = "https://spaceone-custom-assets.s3.ap-northeast-2.amazonaws.com/console-assets/icons/cloud-services/azure"

CLOUD_SERVICE_GROUP_MAP = {
'ApplicationGateways': 'ApplicationGatewaysManager',
'ContainerInstances': 'ContainerInstancesManager',
'CosmosDB': 'CosmosDBManager',
'Disks': 'DisksManager',
'KeyVaults': 'KeyVaultsManager',
'LoadBalancers': 'LoadBalancersManager',
'MySQLServers': 'MySQLServersManager',
'SQLServers': 'SQLServersManager',
'SQLDatabases': 'SQLDatabasesManager',
'NATGateways': 'NATGatewaysManager',
'NetworkSecurityGroups': 'NetworkSecurityGroupsManager',
'PostgreSQLServers': 'PostgreSQLServersManager',
'PublicIPAddresses': 'PublicIPAddressesManager',
'Snapshots': 'SnapshotsManager',
'StorageAccounts': 'StorageAccountsManager',
'VirtualMachines': 'VirtualMachinesManager',
'VirtualNetworks': 'VirtualNetworksManager',
'VMScaleSets': 'VmScaleSetsManager',
'WebPubSubService': 'WebPubSubServiceManager',
"VirtualMachines": "VirtualMachinesManager",
"ApplicationGateways": "ApplicationGatewaysManager",
"ContainerInstances": "ContainerInstancesManager",
"CosmosDB": "CosmosDBManager",
"Disks": "DisksManager",
"KeyVaults": "KeyVaultsManager",
"LoadBalancers": "LoadBalancersManager",
"MySQLServers": "MySQLServersManager",
"SQLServers": "SQLServersManager",
"SQLDatabases": "SQLDatabasesManager",
"NATGateways": "NATGatewaysManager",
"NetworkSecurityGroups": "NetworkSecurityGroupsManager",
"PostgreSQLServers": "PostgreSQLServersManager",
"PublicIPAddresses": "PublicIPAddressesManager",
"Snapshots": "SnapshotsManager",
"StorageAccounts": "StorageAccountsManager",
"VirtualNetworks": "VirtualNetworksManager",
"VMScaleSets": "VmScaleSetsManager",
"WebPubSubService": "WebPubSubServiceManager",
}
9 changes: 7 additions & 2 deletions src/spaceone/inventory/connector/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
from spaceone.inventory.connector.load_balancers import LoadBalancersConnector
from spaceone.inventory.connector.monitor import MonitorConnector
from spaceone.inventory.connector.virtual_networks import VirtualNetworksConnector
from spaceone.inventory.connector.application_gateways import ApplicationGatewaysConnector
from spaceone.inventory.connector.application_gateways import (
ApplicationGatewaysConnector,
)
from spaceone.inventory.connector.public_ip_addresses import PublicIPAddressesConnector
from spaceone.inventory.connector.network_security_groups import NetworkSecurityGroupsConnector
from spaceone.inventory.connector.network_security_groups import (
NetworkSecurityGroupsConnector,
)
from spaceone.inventory.connector.nat_gateways import NATGatewaysConnector
from spaceone.inventory.connector.storage_accounts import StorageAccountsConnector
from spaceone.inventory.connector.key_vaults import KeyVaultsConnector
Expand All @@ -19,3 +23,4 @@
from spaceone.inventory.connector.sql_databases import SQLDatabasesConnector
from spaceone.inventory.connector.container_instances import ContainerInstancesConnector
from spaceone.inventory.connector.web_pubsub_service import WebPubSubServiceConnector
from spaceone.inventory.connector.resources import ResourcesConnector
29 changes: 20 additions & 9 deletions src/spaceone/inventory/connector/key_vaults/connector.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,50 @@
import logging

import azure.core.exceptions

from spaceone.inventory.libs.connector import AzureConnector
from spaceone.inventory.error.custom import *
from azure.keyvault.secrets import SecretClient
from azure.keyvault.certificates import CertificateClient
from azure.identity import DefaultAzureCredential

__all__ = ['KeyVaultsConnector']
__all__ = ["KeyVaultsConnector"]
_LOGGER = logging.getLogger(__name__)


class KeyVaultsConnector(AzureConnector):

def __init__(self, **kwargs):
super().__init__(**kwargs)

self.set_connect(kwargs.get('secret_data'))
self.key_vault_secret_client = None
self.key_vault_certificate_client = None
self.set_connect(kwargs.get("secret_data"))
# self.key_vault_secret_client = None
# self.key_vault_certificate_client = None

def init_key_vault_secret_client(self, subscription_id, vault_uri):
credential = DefaultAzureCredential()
key_vault_secret_client = SecretClient(credential=credential, subscription_id=subscription_id, vault_url=vault_uri)
key_vault_secret_client = SecretClient(
credential=credential, subscription_id=subscription_id, vault_url=vault_uri
)
return key_vault_secret_client

def init_key_vault_certificate_client(self, subscription_id, vault_uri):
credential = DefaultAzureCredential()
key_vault_certificate_client = CertificateClient(credential=credential, subscription_id=subscription_id, vault_url=vault_uri)
key_vault_certificate_client = CertificateClient(
credential=credential, subscription_id=subscription_id, vault_url=vault_uri
)
return key_vault_certificate_client

def list_all_key_vaults(self):
return self.key_vault_client.vaults.list_by_subscription()

def get_key_vaults(self):
return self.key_vault_client.vaults.get()

def list_keys(self, resource_group_name, vault_name):
return self.key_vault_client.keys.list(resource_group_name=resource_group_name, vault_name=vault_name)
return self.key_vault_client.keys.list(
resource_group_name=resource_group_name, vault_name=vault_name
)

def list_secrets(self):
return self.key_vault_secret_client.list_properties_of_secrets()
# return self.key_vault_secrets_client.list_properties_of_secrets()
return self.key_vault_client.secrets.list()
23 changes: 15 additions & 8 deletions src/spaceone/inventory/connector/postgresql_servers/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,35 @@

from spaceone.inventory.libs.connector import AzureConnector
from spaceone.inventory.error.custom import *
__all__ = ['PostgreSQLServersConnector']

__all__ = ["PostgreSQLServersConnector"]
_LOGGER = logging.getLogger(__name__)


class PostgreSQLServersConnector(AzureConnector):

def __init__(self, **kwargs):
super().__init__(**kwargs)
self.set_connect(kwargs.get('secret_data'))
self.set_connect(kwargs.get("secret_data"))

def list_servers(self):
return self.postgre_sql_client.servers.list()

def list_firewall_rules_by_server(self, resource_group_name, server_name):
return self.postgre_sql_client.firewall_rules.list_by_server(resource_group_name=resource_group_name, server_name=server_name)
return self.postgre_sql_client.firewall_rules.list_by_server(
resource_group_name=resource_group_name, server_name=server_name
)

def list_virtual_network_rules_by_server(self, resource_group_name, server_name):
return self.postgre_sql_client.virtual_network_rules.list_by_server(resource_group_name=resource_group_name,
server_name=server_name)
return self.postgre_sql_client.virtual_network_rules.list_by_server(
resource_group_name=resource_group_name, server_name=server_name
)

def list_replicas_by_server(self, resource_group_name, server_name):
return self.postgre_sql_client.replicas.list_by_server(resource_group_name=resource_group_name, server_name=server_name)
return self.postgre_sql_client.replicas.list_by_server(
resource_group_name=resource_group_name, server_name=server_name
)

def list_server_administrators(self, resource_group_name, server_name):
return self.postgre_sql_client.server_administrators.list(resource_group_name=resource_group_name, server_name=server_name)
return self.postgre_sql_client.server_administrators.list(
resource_group_name=resource_group_name, server_name=server_name
)
1 change: 1 addition & 0 deletions src/spaceone/inventory/connector/resources/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from spaceone.inventory.connector.resources.connector import ResourcesConnector
16 changes: 16 additions & 0 deletions src/spaceone/inventory/connector/resources/connector.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import logging

from spaceone.inventory.libs.connector import AzureConnector
from spaceone.inventory.error import *

__all__ = ["ResourcesConnector"]
_LOGGER = logging.getLogger(__name__)


class ResourcesConnector(AzureConnector):
def __init__(self, **kwargs):
super().__init__()
self.set_connect(kwargs.get("secret_data"))

def list_resources(self) -> list:
return self.resource_client.resources.list()
11 changes: 5 additions & 6 deletions src/spaceone/inventory/connector/storage_accounts/connector.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
import logging

from spaceone.inventory.libs.connector import AzureConnector
from spaceone.inventory.error.custom import *

__all__ = ['StorageAccountsConnector']
__all__ = ["StorageAccountsConnector"]
_LOGGER = logging.getLogger(__name__)


class StorageAccountsConnector(AzureConnector):

def __init__(self, **kwargs):
super().__init__(**kwargs)
self.set_connect(kwargs.get('secret_data'))
self.set_connect(kwargs.get("secret_data"))

def list_storage_accounts(self):
return self.storage_client.storage_accounts.list()

def list_blob_containers(self, rg_name, account_name):
return self.storage_client.blob_containers.list(resource_group_name=rg_name, account_name=account_name)

return self.storage_client.blob_containers.list(
resource_group_name=rg_name, account_name=account_name
)
36 changes: 26 additions & 10 deletions src/spaceone/inventory/connector/vm_scale_sets/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,42 @@
from spaceone.inventory.libs.connector import AzureConnector
from spaceone.inventory.error.custom import *

__all__ = ['VmScaleSetsConnector']
__all__ = ["VmScaleSetsConnector"]
_LOGGER = logging.getLogger(__name__)


class VmScaleSetsConnector(AzureConnector):

def __init__(self, **kwargs):
super().__init__(**kwargs)
self.set_connect(kwargs.get('secret_data'))
self.set_connect(kwargs.get("secret_data"))

def list_vm_scale_sets(self):
return self.compute_client.virtual_machine_scale_sets.list_all()

def list_vm_scale_set_vms(self, resource_group, vm_scale_set_name):
return self.compute_client.virtual_machine_scale_set_vms.list(resource_group, vm_scale_set_name)

def get_vm_scale_set_instance_view(self, resource_group, vm_scale_set_name, instance_id):
return self.compute_client.virtual_machine_scale_set_vms.get_instance_view(resource_group_name=resource_group,
vm_scale_set_name=vm_scale_set_name,
instance_id=instance_id)
return self.compute_client.virtual_machine_scale_set_vms.list(
resource_group, vm_scale_set_name
)

def get_vm_scale_set_instance_view(
self, resource_group, vm_scale_set_name, instance_id
):
return self.compute_client.virtual_machine_scale_set_vms.get_instance_view(
resource_group_name=resource_group,
vm_scale_set_name=vm_scale_set_name,
instance_id="0",
)

def list_vm_scale_set_instance_view(
self, resource_group, vm_scale_set_name, instance_id
):
return self.compute_client.virtual_machine_scale_set_vms.list(
resource_group_name=resource_group,
vm_scale_set_name=vm_scale_set_name,
instance_id=instance_id,
)

def list_auto_scale_settings(self, resource_group):
return self.monitor_client.autoscale_settings.list_by_resource_group(resource_group_name=resource_group)
return self.monitor_client.autoscale_settings.list_by_resource_group(
resource_group_name=resource_group
)
1 change: 1 addition & 0 deletions src/spaceone/inventory/info/__init__.py
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
from spaceone.inventory.info.collector_info import *
from spaceone.inventory.info.job_info import *
from spaceone.inventory.info.common_info import *
18 changes: 18 additions & 0 deletions src/spaceone/inventory/info/job_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
__all__ = ["TasksInfo", "TaskInfo"]

import functools
from spaceone.api.inventory.plugin import job_pb2
from spaceone.core.pygrpc.message_type import *


def TaskInfo(task_data):
info = {"task_options": change_struct_type(task_data["task_options"])}
return job_pb2.TaskInfo(**info)


def TasksInfo(result, **kwargs):
tasks_data = result.get("tasks", [])

return job_pb2.TasksInfo(
tasks=list(map(functools.partial(TaskInfo, **kwargs), tasks_data)),
)
File renamed without changes.
9 changes: 9 additions & 0 deletions src/spaceone/inventory/interface/grpc/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from spaceone.core.pygrpc.server import GRPCServer
from spaceone.inventory.interface.grpc.collector import Collector
from spaceone.inventory.interface.grpc.job import Job

_all_ = ["app"]

app = GRPCServer()
app.add_service(Collector)
app.add_service(Job)
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,27 @@ class Collector(BaseAPI, collector_pb2_grpc.CollectorServicer):
def init(self, request, context):
params, metadata = self.parse_request(request, context)

with self.locator.get_service('CollectorService', metadata) as collector_svc:
with self.locator.get_service("CollectorService", metadata) as collector_svc:
data = collector_svc.init(params)
return self.locator.get_info('PluginInfo', data)
return self.locator.get_info("PluginInfo", data)

def verify(self, request, context):
params, metadata = self.parse_request(request, context)

collector_svc: CollectorService = self.locator.get_service('CollectorService', metadata)
collector_svc: CollectorService = self.locator.get_service(
"CollectorService", metadata
)

with collector_svc:
collector_svc.verify(params)
return self.locator.get_info('EmptyInfo')
return self.locator.get_info("EmptyInfo")

def collect(self, request, context):
params, metadata = self.parse_request(request, context)
collector_svc: CollectorService = self.locator.get_service('CollectorService', metadata)
collector_svc: CollectorService = self.locator.get_service(
"CollectorService", metadata
)

with collector_svc:
for resource in collector_svc.collect(params):
yield self.locator.get_info('ResourceInfo', resource.to_primitive())


yield self.locator.get_info("ResourceInfo", resource)
Loading

0 comments on commit 0cf9b77

Please sign in to comment.