From 92ae82bca8c202032ceeb7dabde959e984547e43 Mon Sep 17 00:00:00 2001 From: Gwangil Kang Date: Mon, 17 Jun 2024 06:00:49 +0900 Subject: [PATCH] etc: support latest plugin framework Signed-off-by: Gwangil Kang --- pkg/pip_requirements.txt | 3 + src/cloudforet/__init__.py | 1 - src/cloudforet/cost_analysis/__init__.py | 1 - src/cloudforet/cost_analysis/conf/__init__.py | 0 .../cost_analysis/conf/cost_conf.py | 3 - .../cost_analysis/conf/global_conf.py | 30 ---- .../cost_analysis/conf/proto_conf.py | 5 - .../cost_analysis/connector/__init__.py | 2 - .../connector/bigquery_connector.py | 52 ------ .../connector/cloud_billing_connector.py | 36 ----- .../cost_analysis/error/__init__.py | 1 - src/cloudforet/cost_analysis/error/cost.py | 25 --- src/cloudforet/cost_analysis/info/__init__.py | 4 - .../cost_analysis/info/common_info.py | 7 - .../cost_analysis/info/cost_info.py | 38 ----- .../cost_analysis/info/data_source_info.py | 12 -- src/cloudforet/cost_analysis/info/job_info.py | 35 ---- .../cost_analysis/interface/__init__.py | 0 .../cost_analysis/interface/grpc/__init__.py | 0 .../interface/grpc/plugin/__init__.py | 0 .../interface/grpc/plugin/cost.py | 17 -- .../interface/grpc/plugin/data_source.py | 22 --- .../interface/grpc/plugin/job.py | 15 -- .../cost_analysis/manager/__init__.py | 3 - .../cost_analysis/manager/cost_manager.py | 149 ------------------ .../manager/data_source_manager.py | 23 --- .../cost_analysis/manager/job_manager.py | 137 ---------------- .../cost_analysis/model/__init__.py | 2 - .../cost_analysis/model/cost_model.py | 17 -- .../cost_analysis/model/data_source_model.py | 55 ------- .../cost_analysis/model/job_model.py | 27 ---- .../cost_analysis/service/__init__.py | 3 - .../cost_analysis/service/cost_service.py | 42 ----- .../service/data_source_service.py | 58 ------- .../cost_analysis/service/job_service.py | 46 ------ 35 files changed, 3 insertions(+), 868 deletions(-) delete mode 100644 src/cloudforet/__init__.py delete mode 100644 src/cloudforet/cost_analysis/__init__.py delete mode 100644 src/cloudforet/cost_analysis/conf/__init__.py delete mode 100644 src/cloudforet/cost_analysis/conf/cost_conf.py delete mode 100644 src/cloudforet/cost_analysis/conf/global_conf.py delete mode 100644 src/cloudforet/cost_analysis/conf/proto_conf.py delete mode 100644 src/cloudforet/cost_analysis/connector/__init__.py delete mode 100644 src/cloudforet/cost_analysis/connector/bigquery_connector.py delete mode 100644 src/cloudforet/cost_analysis/connector/cloud_billing_connector.py delete mode 100644 src/cloudforet/cost_analysis/error/__init__.py delete mode 100644 src/cloudforet/cost_analysis/error/cost.py delete mode 100644 src/cloudforet/cost_analysis/info/__init__.py delete mode 100644 src/cloudforet/cost_analysis/info/common_info.py delete mode 100644 src/cloudforet/cost_analysis/info/cost_info.py delete mode 100644 src/cloudforet/cost_analysis/info/data_source_info.py delete mode 100644 src/cloudforet/cost_analysis/info/job_info.py delete mode 100644 src/cloudforet/cost_analysis/interface/__init__.py delete mode 100644 src/cloudforet/cost_analysis/interface/grpc/__init__.py delete mode 100644 src/cloudforet/cost_analysis/interface/grpc/plugin/__init__.py delete mode 100644 src/cloudforet/cost_analysis/interface/grpc/plugin/cost.py delete mode 100644 src/cloudforet/cost_analysis/interface/grpc/plugin/data_source.py delete mode 100644 src/cloudforet/cost_analysis/interface/grpc/plugin/job.py delete mode 100644 src/cloudforet/cost_analysis/manager/__init__.py delete mode 100755 src/cloudforet/cost_analysis/manager/cost_manager.py delete mode 100644 src/cloudforet/cost_analysis/manager/data_source_manager.py delete mode 100644 src/cloudforet/cost_analysis/manager/job_manager.py delete mode 100644 src/cloudforet/cost_analysis/model/__init__.py delete mode 100644 src/cloudforet/cost_analysis/model/cost_model.py delete mode 100644 src/cloudforet/cost_analysis/model/data_source_model.py delete mode 100644 src/cloudforet/cost_analysis/model/job_model.py delete mode 100644 src/cloudforet/cost_analysis/service/__init__.py delete mode 100644 src/cloudforet/cost_analysis/service/cost_service.py delete mode 100644 src/cloudforet/cost_analysis/service/data_source_service.py delete mode 100644 src/cloudforet/cost_analysis/service/job_service.py diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index 7b81b2b..c7d0647 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -1,3 +1,6 @@ +spaceone-core +spaceone-api +spaceone-cost-analysis schematics google-api-python-client pandas-gbq diff --git a/src/cloudforet/__init__.py b/src/cloudforet/__init__.py deleted file mode 100644 index 0260537..0000000 --- a/src/cloudforet/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/src/cloudforet/cost_analysis/__init__.py b/src/cloudforet/cost_analysis/__init__.py deleted file mode 100644 index 9baac51..0000000 --- a/src/cloudforet/cost_analysis/__init__.py +++ /dev/null @@ -1 +0,0 @@ -name = 'cost_analysis' diff --git a/src/cloudforet/cost_analysis/conf/__init__.py b/src/cloudforet/cost_analysis/conf/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cloudforet/cost_analysis/conf/cost_conf.py b/src/cloudforet/cost_analysis/conf/cost_conf.py deleted file mode 100644 index d20ba7f..0000000 --- a/src/cloudforet/cost_analysis/conf/cost_conf.py +++ /dev/null @@ -1,3 +0,0 @@ -DEFAULT_BILLING_DATASET = 'spaceone_billing_data' -BIGQUERY_TABLE_PREFIX = 'gcp_billing_export_v1' -SECRET_TYPE_DEFAULT = 'MANUAL' \ No newline at end of file diff --git a/src/cloudforet/cost_analysis/conf/global_conf.py b/src/cloudforet/cost_analysis/conf/global_conf.py deleted file mode 100644 index 2ae2988..0000000 --- a/src/cloudforet/cost_analysis/conf/global_conf.py +++ /dev/null @@ -1,30 +0,0 @@ -LOG = { - 'filters': { - 'masking': { - 'rules': { - 'DataSource.verify': [ - 'secret_data' - ], - 'Job.get_tasks': [ - 'secret_data' - ], - 'Cost.get_data': [ - 'secret_data' - ] - } - } - } -} - -DEFAULT_LOGGER = 'cloudforet' - -CONNECTORS = { - 'SpaceConnector': { - 'backend': 'cloudforet.core.connector.space_connector.SpaceConnector', - 'endpoints': { - 'identity': 'grpc://identity:50051', - } - }, -} - -SECRET_TYPE_DEFAULT = 'USE_SERVICE_ACCOUNT_SECRET' diff --git a/src/cloudforet/cost_analysis/conf/proto_conf.py b/src/cloudforet/cost_analysis/conf/proto_conf.py deleted file mode 100644 index ae53429..0000000 --- a/src/cloudforet/cost_analysis/conf/proto_conf.py +++ /dev/null @@ -1,5 +0,0 @@ -PROTO = { - 'cloudforet.cost_analysis.interface.grpc.plugin.data_source': ['DataSource'], - 'cloudforet.cost_analysis.interface.grpc.plugin.job': ['Job'], - 'cloudforet.cost_analysis.interface.grpc.plugin.cost': ['Cost'], -} diff --git a/src/cloudforet/cost_analysis/connector/__init__.py b/src/cloudforet/cost_analysis/connector/__init__.py deleted file mode 100644 index d55e674..0000000 --- a/src/cloudforet/cost_analysis/connector/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from cloudforet.cost_analysis.connector.bigquery_connector import BigqueryConnector -from cloudforet.cost_analysis.connector.cloud_billing_connector import CloudBillingConnector diff --git a/src/cloudforet/cost_analysis/connector/bigquery_connector.py b/src/cloudforet/cost_analysis/connector/bigquery_connector.py deleted file mode 100644 index 321253a..0000000 --- a/src/cloudforet/cost_analysis/connector/bigquery_connector.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -import google.oauth2.service_account -import pandas_gbq -from googleapiclient.discovery import build - -from spaceone.core.connector import BaseConnector -from cloudforet.cost_analysis.error import * - -_LOGGER = logging.getLogger(__name__) - -REQUIRED_SECRET_KEYS = ["project_id", "private_key", "token_uri", "client_email"] - - -class BigqueryConnector(BaseConnector): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.project_id = None - self.credentials = None - self.google_client = None - - def create_session(self, options: dict, secret_data: dict, schema: str): - self._check_secret_data(secret_data) - self.project_id = secret_data['project_id'] - - self.credentials = google.oauth2.service_account.Credentials.from_service_account_info(secret_data) - self.google_client = build('bigquery', 'v2', credentials=self.credentials) - - def list_tables(self, dataset_id, **query): - table_list = [] - - query.update({'projectId': self.project_id, - 'datasetId': dataset_id}) - - request = self.google_client.tables().list(**query) - while request is not None: - response = request.execute() - for table in response.get('tables', []): - table_list.append(table) - request = self.google_client.tables().list_next(previous_request=request, previous_response=response) - - return table_list - - def read_df_from_bigquery(self, query): - return pandas_gbq.read_gbq(query, project_id=self.project_id, credentials=self.credentials) - - @staticmethod - def _check_secret_data(secret_data): - missing_keys = [key for key in REQUIRED_SECRET_KEYS if key not in secret_data] - if missing_keys: - for key in missing_keys: - raise ERROR_REQUIRED_PARAMETER(key=f"secret_data.{key}") diff --git a/src/cloudforet/cost_analysis/connector/cloud_billing_connector.py b/src/cloudforet/cost_analysis/connector/cloud_billing_connector.py deleted file mode 100644 index d7b2405..0000000 --- a/src/cloudforet/cost_analysis/connector/cloud_billing_connector.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging -import google.oauth2.service_account -from googleapiclient.discovery import build - -from spaceone.core.connector import BaseConnector -from cloudforet.cost_analysis.error import * - -_LOGGER = logging.getLogger(__name__) - -REQUIRED_SECRET_KEYS = ["project_id", "private_key", "token_uri", "client_email"] - - -class CloudBillingConnector(BaseConnector): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.project_id = None - self.google_client = None - - def create_session(self, options: dict, secret_data: dict, schema: str): - self._check_secret_data(secret_data) - self.project_id = secret_data['project_id'] - - credentials = google.oauth2.service_account.Credentials.from_service_account_info(secret_data) - self.google_client = build('cloudbilling', 'v1', credentials=credentials) - - def get_billing_info(self): - billing_info = self.google_client.projects().getBillingInfo(name=f'projects/{self.project_id}').execute() - return billing_info - - @staticmethod - def _check_secret_data(secret_data): - missing_keys = [key for key in REQUIRED_SECRET_KEYS if key not in secret_data] - if missing_keys: - for key in missing_keys: - raise ERROR_REQUIRED_PARAMETER(key=f"secret_data.{key}") diff --git a/src/cloudforet/cost_analysis/error/__init__.py b/src/cloudforet/cost_analysis/error/__init__.py deleted file mode 100644 index c469e59..0000000 --- a/src/cloudforet/cost_analysis/error/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from cloudforet.cost_analysis.error.cost import * \ No newline at end of file diff --git a/src/cloudforet/cost_analysis/error/cost.py b/src/cloudforet/cost_analysis/error/cost.py deleted file mode 100644 index 15d2ba0..0000000 --- a/src/cloudforet/cost_analysis/error/cost.py +++ /dev/null @@ -1,25 +0,0 @@ -from spaceone.core.error import * - - -class ERROR_INVALID_SECRET_TYPE(ERROR_INVALID_ARGUMENT): - _message = 'Invalid secret type: {secret_type}' - - -class ERROR_TOO_MANY_CSV_FILES(ERROR_UNKNOWN): - _message = 'Too many csv files: {target_dir}' - - -class ERROR_EXCHANGE_RATE_DATA_NOT_FOUND(ERROR_UNKNOWN): - _message = 'Exchange rate data not found' - - -class ERROR_NOT_FOUND_EXCHANGE_RATE(ERROR_UNKNOWN): - _message = 'Invalid exchange rate: {year}-{month}' - - -class ERROR_NOT_FOUND_TABLE(ERROR_UNKNOWN): - _message = 'Not found table: {table} / dataset: {dataset}' - - -class ERROR_NOT_EXIST_TARGET_PROJECT_ID(ERROR_INVALID_ARGUMENT): - _message = 'Not exist target_project_id: {target_project_id}' diff --git a/src/cloudforet/cost_analysis/info/__init__.py b/src/cloudforet/cost_analysis/info/__init__.py deleted file mode 100644 index d497b1f..0000000 --- a/src/cloudforet/cost_analysis/info/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from cloudforet.cost_analysis.info.common_info import * -from cloudforet.cost_analysis.info.data_source_info import * -from cloudforet.cost_analysis.info.job_info import * -from cloudforet.cost_analysis.info.cost_info import * diff --git a/src/cloudforet/cost_analysis/info/common_info.py b/src/cloudforet/cost_analysis/info/common_info.py deleted file mode 100644 index 10b98f1..0000000 --- a/src/cloudforet/cost_analysis/info/common_info.py +++ /dev/null @@ -1,7 +0,0 @@ -from google.protobuf.empty_pb2 import Empty - -__all__ = ['EmptyInfo'] - - -def EmptyInfo(): - return Empty() diff --git a/src/cloudforet/cost_analysis/info/cost_info.py b/src/cloudforet/cost_analysis/info/cost_info.py deleted file mode 100644 index 52fc123..0000000 --- a/src/cloudforet/cost_analysis/info/cost_info.py +++ /dev/null @@ -1,38 +0,0 @@ -import functools -import logging -from spaceone.api.cost_analysis.plugin import cost_pb2 -from spaceone.core.pygrpc.message_type import * -from spaceone.core import utils - -__all__ = ['CostInfo', 'CostsInfo'] - -_LOGGER = logging.getLogger(__name__) - - -def CostInfo(cost_data): - try: - info = { - 'cost': cost_data['cost'], - 'usage_quantity': cost_data.get('usage_quantity'), - 'usage_type': cost_data.get('usage_type'), - 'usage_unit': cost_data.get('usage_unit'), - 'provider': cost_data.get('provider'), - 'region_code': cost_data.get('region_code'), - 'product': cost_data.get('product'), - 'resource': cost_data.get('resource'), - 'tags': change_struct_type(cost_data['tags']) if 'tags' in cost_data else None, - 'additional_info': change_struct_type( - cost_data['additional_info']) if 'additional_info' in cost_data else None, - 'billed_date': cost_data['billed_date'] - } - - return cost_pb2.CostInfo(**info) - - except Exception as e: - _LOGGER.debug(f'[CostInfo] cost data: {cost_data}') - _LOGGER.debug(f'[CostInfo] error reason: {e}', exc_info=True) - raise e - - -def CostsInfo(costs_data, **kwargs): - return cost_pb2.CostsInfo(results=list(map(functools.partial(CostInfo, **kwargs), costs_data))) diff --git a/src/cloudforet/cost_analysis/info/data_source_info.py b/src/cloudforet/cost_analysis/info/data_source_info.py deleted file mode 100644 index f0af69f..0000000 --- a/src/cloudforet/cost_analysis/info/data_source_info.py +++ /dev/null @@ -1,12 +0,0 @@ -from spaceone.api.cost_analysis.plugin import data_source_pb2 -from spaceone.core.pygrpc.message_type import * - -__all__ = ['PluginInfo'] - - -def PluginInfo(plugin_data): - info = { - 'metadata': change_struct_type(plugin_data['metadata']), - } - - return data_source_pb2.PluginInfo(**info) diff --git a/src/cloudforet/cost_analysis/info/job_info.py b/src/cloudforet/cost_analysis/info/job_info.py deleted file mode 100644 index 5eff786..0000000 --- a/src/cloudforet/cost_analysis/info/job_info.py +++ /dev/null @@ -1,35 +0,0 @@ -import functools -from spaceone.api.cost_analysis.plugin import job_pb2 -from spaceone.core.pygrpc.message_type import * - -__all__ = ['TaskInfo', 'TasksInfo'] - - -def TaskInfo(task_data): - info = { - 'task_options': change_struct_type(task_data['task_options']) - } - - return job_pb2.TaskInfo(**info) - - -def ChangedInfo(changed_data): - info = { - 'start': changed_data['start'] - } - - if 'end' in changed_data: - info['end'] = changed_data['end'] - - if 'filter' in changed_data: - info['filter'] = change_struct_type(changed_data['filter']) - - return job_pb2.ChangedInfo(**info) - - -def TasksInfo(result, **kwargs): - tasks_data = result.get('tasks', []) - changed_data = result.get('changed', []) - - return job_pb2.TasksInfo(tasks=list(map(functools.partial(TaskInfo, **kwargs), tasks_data)), - changed=list(map(functools.partial(ChangedInfo, **kwargs), changed_data))) diff --git a/src/cloudforet/cost_analysis/interface/__init__.py b/src/cloudforet/cost_analysis/interface/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cloudforet/cost_analysis/interface/grpc/__init__.py b/src/cloudforet/cost_analysis/interface/grpc/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cloudforet/cost_analysis/interface/grpc/plugin/__init__.py b/src/cloudforet/cost_analysis/interface/grpc/plugin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/cloudforet/cost_analysis/interface/grpc/plugin/cost.py b/src/cloudforet/cost_analysis/interface/grpc/plugin/cost.py deleted file mode 100644 index 0965e66..0000000 --- a/src/cloudforet/cost_analysis/interface/grpc/plugin/cost.py +++ /dev/null @@ -1,17 +0,0 @@ -from spaceone.api.cost_analysis.plugin import cost_pb2, cost_pb2_grpc -from spaceone.core.pygrpc import BaseAPI -from cloudforet.cost_analysis.service import CostService -from cloudforet.cost_analysis.info import CostsInfo - - -class Cost(BaseAPI, cost_pb2_grpc.CostServicer): - pb2 = cost_pb2 - pb2_grpc = cost_pb2_grpc - - def get_data(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service(CostService, metadata) as cost_service: - response_stream = cost_service.get_data(params) - for costs_data in response_stream: - yield self.locator.get_info(CostsInfo, costs_data) diff --git a/src/cloudforet/cost_analysis/interface/grpc/plugin/data_source.py b/src/cloudforet/cost_analysis/interface/grpc/plugin/data_source.py deleted file mode 100644 index 95cf48d..0000000 --- a/src/cloudforet/cost_analysis/interface/grpc/plugin/data_source.py +++ /dev/null @@ -1,22 +0,0 @@ -from spaceone.api.cost_analysis.plugin import data_source_pb2, data_source_pb2_grpc -from spaceone.core.pygrpc import BaseAPI -from cloudforet.cost_analysis.service import DataSourceService -from cloudforet.cost_analysis.info import PluginInfo, EmptyInfo - - -class DataSource(BaseAPI, data_source_pb2_grpc.DataSourceServicer): - pb2 = data_source_pb2 - pb2_grpc = data_source_pb2_grpc - - def init(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service(DataSourceService, metadata) as data_source_service: - return self.locator.get_info(PluginInfo, data_source_service.init(params)) - - def verify(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service(DataSourceService, metadata) as data_source_service: - data_source_service.verify(params) - return self.locator.get_info(EmptyInfo) diff --git a/src/cloudforet/cost_analysis/interface/grpc/plugin/job.py b/src/cloudforet/cost_analysis/interface/grpc/plugin/job.py deleted file mode 100644 index ff58474..0000000 --- a/src/cloudforet/cost_analysis/interface/grpc/plugin/job.py +++ /dev/null @@ -1,15 +0,0 @@ -from spaceone.api.cost_analysis.plugin import job_pb2, job_pb2_grpc -from spaceone.core.pygrpc import BaseAPI -from cloudforet.cost_analysis.service import JobService -from cloudforet.cost_analysis.info import TasksInfo - - -class Job(BaseAPI, job_pb2_grpc.JobServicer): - pb2 = job_pb2 - pb2_grpc = job_pb2_grpc - - def get_tasks(self, request, context): - params, metadata = self.parse_request(request, context) - - with self.locator.get_service(JobService, metadata) as job_service: - return self.locator.get_info(TasksInfo, job_service.get_tasks(params)) diff --git a/src/cloudforet/cost_analysis/manager/__init__.py b/src/cloudforet/cost_analysis/manager/__init__.py deleted file mode 100644 index 8f5eb37..0000000 --- a/src/cloudforet/cost_analysis/manager/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cloudforet.cost_analysis.manager.data_source_manager import DataSourceManager -from cloudforet.cost_analysis.manager.job_manager import JobManager -from cloudforet.cost_analysis.manager.cost_manager import CostManager diff --git a/src/cloudforet/cost_analysis/manager/cost_manager.py b/src/cloudforet/cost_analysis/manager/cost_manager.py deleted file mode 100755 index 24821d5..0000000 --- a/src/cloudforet/cost_analysis/manager/cost_manager.py +++ /dev/null @@ -1,149 +0,0 @@ -import logging - -from spaceone.core.manager import BaseManager -from cloudforet.cost_analysis.error import * -from cloudforet.cost_analysis.conf.cost_conf import BIGQUERY_TABLE_PREFIX -from cloudforet.cost_analysis.connector import BigqueryConnector - -_LOGGER = logging.getLogger(__name__) - -REQUIRED_TASK_OPTIONS = ["start", "billing_dataset", "billing_account_id", "project_id"] -EXCLUSIVE_PRODUCT = ['Invoice'] - - -class CostManager(BaseManager): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.bigquery_connector: BigqueryConnector = self.locator.get_connector(BigqueryConnector) - self.billing_project_id = None - self.billing_dataset = None - self.billing_table = None - self.target_project_id = None - - def get_data(self, options, secret_data, schema, task_options): - self.bigquery_connector.create_session(options, secret_data, schema) - self._check_task_options(task_options) - - start = task_options['start'] - self.billing_project_id = secret_data['project_id'] - self.billing_dataset = task_options['billing_dataset'] - billing_account_id = task_options['billing_account_id'] - self.target_project_id = task_options['project_id'] - - self.billing_table = f'{BIGQUERY_TABLE_PREFIX}_{billing_account_id.replace("-", "_")}' - self._validate_table_exists() - - _LOGGER.debug(f'[get_data] task_options: {task_options} / start: {start})') - - query = self._create_google_sql(start) - response_stream = self.bigquery_connector.read_df_from_bigquery(query) - for index, row in response_stream.iterrows(): - yield self._make_cost_data(row) - - yield [] - - @staticmethod - def _check_task_options(task_options): - missing_keys = [key for key in REQUIRED_TASK_OPTIONS if key not in task_options] - if missing_keys: - for key in missing_keys: - raise ERROR_REQUIRED_PARAMETER(key=f"task_options.{key}") - - def _validate_table_exists(self): - bigquery_tables_info = self.bigquery_connector.list_tables(self.billing_dataset) - bigquery_table_names = [table_info["tableReference"]["tableId"] for table_info in bigquery_tables_info] - - if self.billing_table not in bigquery_table_names: - raise ERROR_NOT_FOUND_TABLE(table=self.billing_table, dataset=self.billing_dataset) - - def _make_cost_data(self, row): - """ Source Data Model (DataFrame) - class CostSummaryItem(DataFrame): - billed_at: str - billing_account_id: str - sku_description: str - id: str - name: str - region_code: str - currency_conversion_rate: float - pricing_unit: str - month: str - cost_type: str - labels: str(list of dict) - cost: float - usage_quantity: float - """ - costs_data = [] - try: - if row.product not in EXCLUSIVE_PRODUCT: - data = { - 'cost': row.cost, - 'usage_quantity': row.usage_quantity, - 'provider': 'google_cloud', - 'product': row.description, - 'region_code': row.region_code, - 'usage_type': row.sku_description, - 'usage_unit': row.pricing_unit, - 'billed_date': self._change_datetime_to_string(row.billed_at), - 'additional_info': { - 'Project ID': row.id, - 'Project Name': row.project_name, - 'Billing Account ID': row.billing_account_id, - 'Cost Type': row.cost_type, - 'Invoice Month': row.month, - }, - 'tags': {} - } - - if labels := eval(row.labels): - for label_object in labels: - data['tags'][label_object['key']] = label_object['value'] - - costs_data.append(data) - - except Exception as e: - _LOGGER.error(f'[_make_cost_data] make data error: {e}', exc_info=True) - raise e - - return costs_data - - def _create_google_sql(self, start): - where_condition = f""" - WHERE usage_start_time >= TIMESTAMP('{start}-01') - """ - if self.target_project_id != '*': - where_condition += f" AND project.id = '{self.target_project_id}'" - - query = f""" - SELECT - timestamp_trunc(usage_start_time, DAY) as billed_at, - billing_account_id, - service.description, - sku.description as sku_description, - project.id, - project.name as project_name, - IFNULL((location.region), 'global') as region_code, - currency_conversion_rate, - usage.pricing_unit, - invoice.month, - cost_type, - TO_JSON_STRING(labels) as labels, - - SUM(cost) - + SUM(IFNULL((SELECT SUM(c.amount) - FROM UNNEST(credits) c), 0)) - AS cost, - - SUM(usage.amount_in_pricing_units) as usage_quantity, - FROM `{self.billing_project_id}.{self.billing_dataset}.{self.billing_table}` - {where_condition} - GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 - ORDER BY billed_at desc - ; - """ - return query - - @staticmethod - def _change_datetime_to_string(date_time): - return str(date_time.strftime("%Y-%m-%d")) diff --git a/src/cloudforet/cost_analysis/manager/data_source_manager.py b/src/cloudforet/cost_analysis/manager/data_source_manager.py deleted file mode 100644 index d9f238e..0000000 --- a/src/cloudforet/cost_analysis/manager/data_source_manager.py +++ /dev/null @@ -1,23 +0,0 @@ -import logging - -from spaceone.core.manager import BaseManager -from cloudforet.cost_analysis.connector import BigqueryConnector -from cloudforet.cost_analysis.model import PluginMetadata - -_LOGGER = logging.getLogger(__name__) - - -class DataSourceManager(BaseManager): - - @staticmethod - def init_response(options): - plugin_metadata = PluginMetadata() - plugin_metadata.validate() - - return { - 'metadata': plugin_metadata.to_primitive() - } - - def verify_plugin(self, options, secret_data, schema): - bigquery_connector: BigqueryConnector = self.locator.get_connector(BigqueryConnector) - bigquery_connector.create_session(options, secret_data, schema) diff --git a/src/cloudforet/cost_analysis/manager/job_manager.py b/src/cloudforet/cost_analysis/manager/job_manager.py deleted file mode 100644 index a40bbbd..0000000 --- a/src/cloudforet/cost_analysis/manager/job_manager.py +++ /dev/null @@ -1,137 +0,0 @@ -import logging -import re -from datetime import datetime, timedelta - -from spaceone.core.manager import BaseManager -from cloudforet.cost_analysis.error import * -from cloudforet.cost_analysis.conf.cost_conf import * -from cloudforet.cost_analysis.connector import BigqueryConnector, CloudBillingConnector -from cloudforet.cost_analysis.model import Tasks - -_LOGGER = logging.getLogger(__name__) - - -class JobManager(BaseManager): - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.bigquery_connector: BigqueryConnector = self.locator.get_connector(BigqueryConnector) - self.cloud_billing_connector: CloudBillingConnector = self.locator.get_connector(CloudBillingConnector) - self.billing_account_id = None - - def get_tasks(self, options, secret_data, schema, start, last_synchronized_at, domain_id): - - tasks = [] - changed = [] - - start_month = self._get_start_month(start, last_synchronized_at) - - self.bigquery_connector.create_session(options, secret_data, schema) - self.cloud_billing_connector.create_session(options, secret_data, schema) - self._check_target_project_id_in_secret_data(secret_data) - - billing_dataset = self._get_billing_dataset_from_secret_data(secret_data) - - if secret_data.get('target_billing_account_id'): - self.billing_account_id = secret_data['target_billing_account_id'] - else: - billing_info = self.cloud_billing_connector.get_billing_info() - prefix, self.billing_account_id = billing_info['billingAccountName'].split('/') - - target_project_ids = self._get_target_project_ids(secret_data['target_project_id']) - - secret_type = options.get('secret_type', SECRET_TYPE_DEFAULT) - - if secret_type == 'USE_SERVICE_ACCOUNT_SECRET': - # NOT IMPLEMENTED - pass - elif secret_type == 'MANUAL': - - if target_project_ids: - for target_project_id in target_project_ids: - task, change_info = self._generate_task_and_change_info( - start_month, - billing_dataset, - target_project_id - ) - tasks.append(task) - changed.append(change_info) - else: - target_project_id = '*' - task, change_info = self._generate_task_and_change_info( - start_month, - billing_dataset, - target_project_id - ) - tasks.append(task) - changed.append(change_info) - else: - raise ERROR_INVALID_SECRET_TYPE(secret_type=options.get('secret_type')) - - tasks = Tasks({'tasks': tasks, 'changed': changed}) - tasks.validate() - return tasks.to_primitive() - - def _get_start_month(self, start, last_synchronized_at=None): - if start: - start_time: datetime = self._parse_start_time(start) - elif last_synchronized_at: - start_time: datetime = last_synchronized_at - timedelta(days=7) - start_time = start_time.replace(day=1) - else: - start_time: datetime = datetime.utcnow() - timedelta(days=365) - start_time = start_time.replace(day=1) - - start_time = start_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) - - return start_time.strftime('%Y-%m') - - @staticmethod - def _parse_start_time(start_str): - date_format = '%Y-%m' - - try: - return datetime.strptime(start_str, date_format) - except Exception as e: - raise ERROR_INVALID_PARAMETER_TYPE(key='start', type=date_format) - - @staticmethod - def _check_target_project_id_in_secret_data(secret_data): - if not secret_data.get('target_project_id'): - raise ERROR_REQUIRED_PARAMETER(key='secret_data.target_project_id') - - @staticmethod - def _get_billing_dataset_from_secret_data(secret_data): - if not secret_data.get('billing_dataset'): - _LOGGER.debug( - f'[get_tasks] Not exist billing_dataset in secret_data. Use default: {DEFAULT_BILLING_DATASET}') - billing_dataset = DEFAULT_BILLING_DATASET - else: - _LOGGER.debug(f'[get_tasks] Use billing_dataset in secret_data: {secret_data["billing_dataset"]}') - billing_dataset = secret_data['billing_dataset'] - return billing_dataset - - def _get_target_project_ids(self, target_project_id: list): - if not target_project_id: - _LOGGER.info(f'[get_tasks] Not exist target_project_id: {self.billing_account_id}') - raise ERROR_NOT_EXIST_TARGET_PROJECT_ID(target_project_id=target_project_id) - - elif '*' in target_project_id: - _LOGGER.info(f'[get_tasks] Use all projects in billing account: {self.billing_account_id}') - return [] - else: - return target_project_id - - def _generate_task_and_change_info(self, start_month, billing_dataset, target_project_id): - task = { - 'task_options': { - 'start': start_month, - 'billing_dataset': billing_dataset, - 'billing_account_id': self.billing_account_id, - 'project_id': target_project_id - } - } - - changed = {'start': start_month} - - return task, changed diff --git a/src/cloudforet/cost_analysis/model/__init__.py b/src/cloudforet/cost_analysis/model/__init__.py deleted file mode 100644 index a6cd295..0000000 --- a/src/cloudforet/cost_analysis/model/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from cloudforet.cost_analysis.model.data_source_model import * -from cloudforet.cost_analysis.model.job_model import * \ No newline at end of file diff --git a/src/cloudforet/cost_analysis/model/cost_model.py b/src/cloudforet/cost_analysis/model/cost_model.py deleted file mode 100644 index 594e22c..0000000 --- a/src/cloudforet/cost_analysis/model/cost_model.py +++ /dev/null @@ -1,17 +0,0 @@ -from schematics.models import Model -from schematics.types import DictType, StringType, FloatType, DateTimeType - -__all__ = ['Cost'] - - -class Cost(Model): - cost = FloatType(required=True) - provider = StringType(required=True) - region_code = StringType() - product = StringType() - usage_type = StringType() - usage_unit = StringType(default=None) - usage_quantity = FloatType(required=True) - billed_date = StringType(required=True) - additional_info = DictType(StringType, default={}) - tags = DictType(StringType, default={}) diff --git a/src/cloudforet/cost_analysis/model/data_source_model.py b/src/cloudforet/cost_analysis/model/data_source_model.py deleted file mode 100644 index 07199ff..0000000 --- a/src/cloudforet/cost_analysis/model/data_source_model.py +++ /dev/null @@ -1,55 +0,0 @@ -from schematics.models import Model -from schematics.types import ListType, DictType, StringType, IntType, BooleanType -from schematics.types.compound import ModelType - -__all__ = ['PluginMetadata'] - - -_DEFAULT_DATA_SOURCE_RULES = [ - { - 'name': 'match_service_account', - 'conditions_policy': 'ALWAYS', - 'actions': { - 'match_service_account': { - 'source': 'additional_info.Project ID', - 'target': 'data.project_id' - } - }, - 'options': { - 'stop_processing': True - } - } -] - - -class MatchServiceAccount(Model): - source = StringType(required=True) - target = StringType(required=True) - - -class Actions(Model): - match_service_account = ModelType(MatchServiceAccount) - - -class Options(Model): - stop_processing = BooleanType(default=False) - - -class Condition(Model): - key = StringType(required=True) - value = StringType(required=True) - operator = StringType(required=True, choices=['eq', 'contain', 'not', 'not_contain']) - - -class DataSourceRule(Model): - name = StringType(required=True) - conditions = ListType(ModelType(Condition), default=[]) - conditions_policy = StringType(required=True, choices=['ALL', 'ANY', 'ALWAYS']) - actions = ModelType(Actions, required=True) - options = ModelType(Options, default={}) - tags = DictType(StringType, default={}) - - -class PluginMetadata(Model): - data_source_rules = ListType(ModelType(DataSourceRule), default=_DEFAULT_DATA_SOURCE_RULES) - currency = StringType(default='KRW') diff --git a/src/cloudforet/cost_analysis/model/job_model.py b/src/cloudforet/cost_analysis/model/job_model.py deleted file mode 100644 index 942790e..0000000 --- a/src/cloudforet/cost_analysis/model/job_model.py +++ /dev/null @@ -1,27 +0,0 @@ -from schematics.models import Model -from schematics.types import ListType, DateTimeType, StringType, DictType -from schematics.types.compound import ModelType - -__all__ = ['Tasks'] - - -class TaskOptions(Model): - start = StringType(required=True, max_length=7) - billing_dataset = StringType() - billing_account_id = StringType(default=None) - project_id = StringType() - - -class Task(Model): - task_options = ModelType(TaskOptions, required=True) - - -class Changed(Model): - start = StringType(required=True, max_length=7) - end = StringType(default=None, max_length=7) - filter = DictType(StringType, default={}) - - -class Tasks(Model): - tasks = ListType(ModelType(Task), required=True) - changed = ListType(ModelType(Changed), default=[]) diff --git a/src/cloudforet/cost_analysis/service/__init__.py b/src/cloudforet/cost_analysis/service/__init__.py deleted file mode 100644 index 1ac202f..0000000 --- a/src/cloudforet/cost_analysis/service/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from cloudforet.cost_analysis.service.data_source_service import DataSourceService -from cloudforet.cost_analysis.service.job_service import JobService -from cloudforet.cost_analysis.service.cost_service import CostService diff --git a/src/cloudforet/cost_analysis/service/cost_service.py b/src/cloudforet/cost_analysis/service/cost_service.py deleted file mode 100644 index 63a9f5a..0000000 --- a/src/cloudforet/cost_analysis/service/cost_service.py +++ /dev/null @@ -1,42 +0,0 @@ -import logging - -from spaceone.core.service import * -from cloudforet.cost_analysis.manager.cost_manager import CostManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class CostService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.cost_mgr: CostManager = self.locator.get_manager('CostManager') - - @transaction - @check_required(['options', 'secret_data', 'task_options']) - def get_data(self, params): - """Get Cost Data - - Args: - params (dict): { - 'options': 'dict', - 'secret_data': 'dict', - 'schema': 'str', - 'task_options': 'dict', - 'domain_id': 'str' - } - - Returns: - list of cost_data - - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - task_options = params['task_options'] - - return self.cost_mgr.get_data(options, secret_data, schema, task_options) diff --git a/src/cloudforet/cost_analysis/service/data_source_service.py b/src/cloudforet/cost_analysis/service/data_source_service.py deleted file mode 100644 index eae7e06..0000000 --- a/src/cloudforet/cost_analysis/service/data_source_service.py +++ /dev/null @@ -1,58 +0,0 @@ -import logging - -from spaceone.core.service import * -from cloudforet.cost_analysis.manager.data_source_manager import DataSourceManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class DataSourceService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.data_source_mgr: DataSourceManager = self.locator.get_manager(DataSourceManager) - - @transaction - @check_required(['options']) - def init(self, params): - """ init plugin by options - - Args: - params (dict): { - 'options': 'dict', - 'domain_id': 'str' - } - - Returns: - None - """ - - options = params.get('options', {}) - - return self.data_source_mgr.init_response(options) - - @transaction - @check_required(['options', 'secret_data']) - def verify(self, params): - """ Verifying data source plugin - - Args: - params (dict): { - 'options': 'dict', - 'schema': 'str', - 'secret_data': 'dict', - 'domain_id': 'str' - } - - Returns: - None - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - - return self.data_source_mgr.verify_plugin(options, secret_data, schema) diff --git a/src/cloudforet/cost_analysis/service/job_service.py b/src/cloudforet/cost_analysis/service/job_service.py deleted file mode 100644 index f8d1c2c..0000000 --- a/src/cloudforet/cost_analysis/service/job_service.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging - -from spaceone.core.service import * -from cloudforet.cost_analysis.manager.job_manager import JobManager - -_LOGGER = logging.getLogger(__name__) - - -@authentication_handler -@authorization_handler -@event_handler -class JobService(BaseService): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.job_mgr: JobManager = self.locator.get_manager('JobManager') - - @transaction - @check_required(['options', 'secret_data']) - @change_timestamp_value(['last_synchronized_at'], timestamp_format='iso8601') - def get_tasks(self, params): - """Get Job Tasks - - Args: - params (dict): { - 'options': 'dict', - 'secret_data': 'dict', - 'schema': 'str', - 'start': 'datetime', - 'last_synchronized_at': 'datetime', - 'domain_id': 'str' - } - - Returns: - list of task_data - - """ - - options = params['options'] - secret_data = params['secret_data'] - schema = params.get('schema') - start = params.get('start') - last_synchronized_at = params.get('last_synchronized_at') - domain_id = params['domain_id'] - - return self.job_mgr.get_tasks(options, secret_data, schema, start, last_synchronized_at, domain_id)