From 977a84fc57e28cd9300737bdfc38cc186a96624d Mon Sep 17 00:00:00 2001 From: hippo-an Date: Fri, 18 Oct 2024 17:27:04 +0900 Subject: [PATCH 1/8] add update cost estimation forecast api --- api/docs.go | 231 ++++++++++++++++++++++++ api/swagger.json | 231 ++++++++++++++++++++++++ api/swagger.yaml | 156 ++++++++++++++++ internal/app/cost_estimation_handler.go | 87 +++++++++ internal/app/cost_estimation_req.go | 14 ++ internal/app/router.go | 7 +- internal/config/config.go | 9 + internal/core/cost/dtos.go | 61 +++++++ internal/core/cost/models.go | 2 + internal/core/cost/price_collector.go | 149 +++++++++++++++ internal/core/cost/repository.go | 32 +++- internal/core/cost/service.go | 141 ++++++++++++++- internal/utils/log.go | 18 +- 13 files changed, 1132 insertions(+), 6 deletions(-) diff --git a/api/docs.go b/api/docs.go index 0c885eb..a8c982f 100644 --- a/api/docs.go +++ b/api/docs.go @@ -16,6 +16,53 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/api/v1/cost/forecast": { + "post": { + "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimation]" + ], + "summary": "Estimate Forecast Cost", + "operationId": "EstimateForecastCost", + "parameters": [ + { + "description": "Request body containing estimation parameters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/app.EstimateForecastCostReq" + } + } + ], + "responses": { + "200": { + "description": "Successfully estimated forecast cost", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_EstimateForecastCostResult" + } + }, + "400": { + "description": "Invalid request parameters", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Failed to estimate forecast cost", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + } + }, "/api/v1/cost/info": { "get": { "description": "Retrieve cost information for specified parameters within a defined date range. The date range must be within a 6-month period. Optionally, you can specify cost aggregation type and date order for the results.", @@ -1125,6 +1172,23 @@ const docTemplate = `{ } } }, + "app.AntResponse-cost_EstimateForecastCostResult": { + "type": "object", + "properties": { + "code": { + "type": "integer" + }, + "errorMessage": { + "type": "string" + }, + "result": { + "$ref": "#/definitions/cost.EstimateForecastCostResult" + }, + "successMessage": { + "type": "string" + } + } + }, "app.AntResponse-cost_UpdateCostInfoResult": { "type": "object", "properties": { @@ -1340,6 +1404,49 @@ const docTemplate = `{ } } }, + "app.EstimateForecastCostReq": { + "type": "object", + "required": [ + "recommendSpecs", + "recommendSpecsWithFormat" + ], + "properties": { + "recommendSpecs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + } + } + } + }, + "recommendSpecsWithFormat": { + "type": "array", + "items": { + "type": "object", + "properties": { + "commonImage": { + "type": "string" + }, + "commonSpec": { + "type": "string" + } + } + } + } + } + }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1526,6 +1633,37 @@ const docTemplate = `{ "Remote" ] }, + "constant.PriceCurrency": { + "type": "string", + "enum": [ + "USD", + "KRW" + ], + "x-enum-varnames": [ + "USD", + "KRW" + ] + }, + "constant.PricePolicy": { + "type": "string", + "enum": [ + "OnDemand" + ], + "x-enum-varnames": [ + "OnDemand" + ] + }, + "constant.PriceUnit": { + "type": "string", + "enum": [ + "PerHour", + "PerYear" + ], + "x-enum-varnames": [ + "PerHour", + "PerYear" + ] + }, "constant.ResourceType": { "type": "string", "enum": [ @@ -1541,6 +1679,99 @@ const docTemplate = `{ "Etc" ] }, + "cost.EsimateForecastCostSpecResult": { + "type": "object", + "properties": { + "estimateForecastCostSpecDetailResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EstimateForecastCostSpecDetailResult" + } + }, + "imageName": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + }, + "totalMaxMonthlyPrice": { + "type": "number" + }, + "totalMinMonthlyPrice": { + "type": "number" + } + } + }, + "cost.EstimateForecastCostResult": { + "type": "object", + "properties": { + "esimateForecastCostSpecResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EsimateForecastCostSpecResult" + } + }, + "totalMaxMonthlyPrice": { + "type": "number" + }, + "totalMinMonthlyPrice": { + "type": "number" + } + } + }, + "cost.EstimateForecastCostSpecDetailResult": { + "type": "object", + "properties": { + "calculatedMonthlyPrice": { + "type": "number" + }, + "currency": { + "$ref": "#/definitions/constant.PriceCurrency" + }, + "id": { + "type": "integer" + }, + "lastUpdatedAt": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "originalPricePolicy": { + "type": "string" + }, + "osType": { + "type": "string" + }, + "price": { + "type": "string" + }, + "priceDescription": { + "type": "string" + }, + "pricePolicy": { + "$ref": "#/definitions/constant.PricePolicy" + }, + "productDescription": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "unit": { + "$ref": "#/definitions/constant.PriceUnit" + }, + "vCpu": { + "type": "string" + } + } + }, "cost.GetCostInfoResult": { "type": "object", "properties": { diff --git a/api/swagger.json b/api/swagger.json index 42848e0..10db7bd 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -8,6 +8,53 @@ }, "basePath": "/ant", "paths": { + "/api/v1/cost/forecast": { + "post": { + "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimation]" + ], + "summary": "Estimate Forecast Cost", + "operationId": "EstimateForecastCost", + "parameters": [ + { + "description": "Request body containing estimation parameters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/app.EstimateForecastCostReq" + } + } + ], + "responses": { + "200": { + "description": "Successfully estimated forecast cost", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_EstimateForecastCostResult" + } + }, + "400": { + "description": "Invalid request parameters", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Failed to estimate forecast cost", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + } + }, "/api/v1/cost/info": { "get": { "description": "Retrieve cost information for specified parameters within a defined date range. The date range must be within a 6-month period. Optionally, you can specify cost aggregation type and date order for the results.", @@ -1117,6 +1164,23 @@ } } }, + "app.AntResponse-cost_EstimateForecastCostResult": { + "type": "object", + "properties": { + "code": { + "type": "integer" + }, + "errorMessage": { + "type": "string" + }, + "result": { + "$ref": "#/definitions/cost.EstimateForecastCostResult" + }, + "successMessage": { + "type": "string" + } + } + }, "app.AntResponse-cost_UpdateCostInfoResult": { "type": "object", "properties": { @@ -1332,6 +1396,49 @@ } } }, + "app.EstimateForecastCostReq": { + "type": "object", + "required": [ + "recommendSpecs", + "recommendSpecsWithFormat" + ], + "properties": { + "recommendSpecs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + } + } + } + }, + "recommendSpecsWithFormat": { + "type": "array", + "items": { + "type": "object", + "properties": { + "commonImage": { + "type": "string" + }, + "commonSpec": { + "type": "string" + } + } + } + } + } + }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1518,6 +1625,37 @@ "Remote" ] }, + "constant.PriceCurrency": { + "type": "string", + "enum": [ + "USD", + "KRW" + ], + "x-enum-varnames": [ + "USD", + "KRW" + ] + }, + "constant.PricePolicy": { + "type": "string", + "enum": [ + "OnDemand" + ], + "x-enum-varnames": [ + "OnDemand" + ] + }, + "constant.PriceUnit": { + "type": "string", + "enum": [ + "PerHour", + "PerYear" + ], + "x-enum-varnames": [ + "PerHour", + "PerYear" + ] + }, "constant.ResourceType": { "type": "string", "enum": [ @@ -1533,6 +1671,99 @@ "Etc" ] }, + "cost.EsimateForecastCostSpecResult": { + "type": "object", + "properties": { + "estimateForecastCostSpecDetailResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EstimateForecastCostSpecDetailResult" + } + }, + "imageName": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + }, + "totalMaxMonthlyPrice": { + "type": "number" + }, + "totalMinMonthlyPrice": { + "type": "number" + } + } + }, + "cost.EstimateForecastCostResult": { + "type": "object", + "properties": { + "esimateForecastCostSpecResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EsimateForecastCostSpecResult" + } + }, + "totalMaxMonthlyPrice": { + "type": "number" + }, + "totalMinMonthlyPrice": { + "type": "number" + } + } + }, + "cost.EstimateForecastCostSpecDetailResult": { + "type": "object", + "properties": { + "calculatedMonthlyPrice": { + "type": "number" + }, + "currency": { + "$ref": "#/definitions/constant.PriceCurrency" + }, + "id": { + "type": "integer" + }, + "lastUpdatedAt": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "originalPricePolicy": { + "type": "string" + }, + "osType": { + "type": "string" + }, + "price": { + "type": "string" + }, + "priceDescription": { + "type": "string" + }, + "pricePolicy": { + "$ref": "#/definitions/constant.PricePolicy" + }, + "productDescription": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "unit": { + "$ref": "#/definitions/constant.PriceUnit" + }, + "vCpu": { + "type": "string" + } + } + }, "cost.GetCostInfoResult": { "type": "object", "properties": { diff --git a/api/swagger.yaml b/api/swagger.yaml index a835ed8..e31ff2a 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -52,6 +52,17 @@ definitions: successMessage: type: string type: object + app.AntResponse-cost_EstimateForecastCostResult: + properties: + code: + type: integer + errorMessage: + type: string + result: + $ref: '#/definitions/cost.EstimateForecastCostResult' + successMessage: + type: string + type: object app.AntResponse-cost_UpdateCostInfoResult: properties: code: @@ -191,6 +202,34 @@ definitions: resourceType: $ref: '#/definitions/constant.ResourceType' type: object + app.EstimateForecastCostReq: + properties: + recommendSpecs: + items: + properties: + image: + type: string + instanceType: + type: string + providerName: + type: string + regionName: + type: string + type: object + type: array + recommendSpecsWithFormat: + items: + properties: + commonImage: + type: string + commonSpec: + type: string + type: object + type: array + required: + - recommendSpecs + - recommendSpecsWithFormat + type: object app.InstallLoadGeneratorReq: properties: installLocation: @@ -321,6 +360,28 @@ definitions: x-enum-varnames: - Local - Remote + constant.PriceCurrency: + enum: + - USD + - KRW + type: string + x-enum-varnames: + - USD + - KRW + constant.PricePolicy: + enum: + - OnDemand + type: string + x-enum-varnames: + - OnDemand + constant.PriceUnit: + enum: + - PerHour + - PerYear + type: string + x-enum-varnames: + - PerHour + - PerYear constant.ResourceType: enum: - VM @@ -333,6 +394,67 @@ definitions: - VNet - DataDisk - Etc + cost.EsimateForecastCostSpecResult: + properties: + estimateForecastCostSpecDetailResults: + items: + $ref: '#/definitions/cost.EstimateForecastCostSpecDetailResult' + type: array + imageName: + type: string + instanceType: + type: string + providerName: + type: string + regionName: + type: string + totalMaxMonthlyPrice: + type: number + totalMinMonthlyPrice: + type: number + type: object + cost.EstimateForecastCostResult: + properties: + esimateForecastCostSpecResults: + items: + $ref: '#/definitions/cost.EsimateForecastCostSpecResult' + type: array + totalMaxMonthlyPrice: + type: number + totalMinMonthlyPrice: + type: number + type: object + cost.EstimateForecastCostSpecDetailResult: + properties: + calculatedMonthlyPrice: + type: number + currency: + $ref: '#/definitions/constant.PriceCurrency' + id: + type: integer + lastUpdatedAt: + type: string + memory: + type: string + originalPricePolicy: + type: string + osType: + type: string + price: + type: string + priceDescription: + type: string + pricePolicy: + $ref: '#/definitions/constant.PricePolicy' + productDescription: + type: string + storage: + type: string + unit: + $ref: '#/definitions/constant.PriceUnit' + vCpu: + type: string + type: object cost.GetCostInfoResult: properties: category: @@ -660,6 +782,40 @@ info: title: CM-ANT REST API version: 0.2.2 paths: + /api/v1/cost/forecast: + post: + consumes: + - application/json + description: Estimate the forecast cost for cloud resources based on recommended + specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat + in the request body. Returns an error if the required properties are missing + or if the request is invalid. + operationId: EstimateForecastCost + parameters: + - description: Request body containing estimation parameters + in: body + name: body + required: true + schema: + $ref: '#/definitions/app.EstimateForecastCostReq' + produces: + - application/json + responses: + "200": + description: Successfully estimated forecast cost + schema: + $ref: '#/definitions/app.AntResponse-cost_EstimateForecastCostResult' + "400": + description: Invalid request parameters + schema: + $ref: '#/definitions/app.AntResponse-string' + "500": + description: Failed to estimate forecast cost + schema: + $ref: '#/definitions/app.AntResponse-string' + summary: Estimate Forecast Cost + tags: + - '[Cost Estimation]' /api/v1/cost/info: get: consumes: diff --git a/internal/app/cost_estimation_handler.go b/internal/app/cost_estimation_handler.go index 75f4788..5af8495 100644 --- a/internal/app/cost_estimation_handler.go +++ b/internal/app/cost_estimation_handler.go @@ -6,11 +6,98 @@ import ( "strings" "time" + "github.com/cloud-barista/cm-ant/internal/config" "github.com/cloud-barista/cm-ant/internal/core/common/constant" "github.com/cloud-barista/cm-ant/internal/core/cost" + "github.com/cloud-barista/cm-ant/internal/utils" "github.com/labstack/echo/v4" ) +// @Id EstimateForecastCost +// @Summary Estimate Forecast Cost +// @Description Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid. +// @Tags [Cost Estimation] +// @Accept json +// @Produce json +// @Param body body EstimateForecastCostReq true "Request body containing estimation parameters" +// @Success 200 {object} app.AntResponse[cost.EstimateForecastCostResult] "Successfully estimated forecast cost" +// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" +// @Failure 500 {object} app.AntResponse[string] "Failed to estimate forecast cost" +// @Router /api/v1/cost/forecast [post] +func (a *AntServer) estimateForecastCost(c echo.Context) error { + var req EstimateForecastCostReq + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, err.Error()) + } + + if len(req.RecommendSpecs) == 0 && len(req.RecommendSpecsWithFormat) == 0 { + return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") + } + + pastTime := time.Now().Add(-config.AppConfig.Cost.Estimation.Forcast.PriceUpdateInterval) + + recommendSpecs := make([]cost.RecommendSpecParam, 0) + + if len(req.RecommendSpecs) > 0 { + for _, v := range req.RecommendSpecs { + param := cost.RecommendSpecParam{ + ProviderName: strings.TrimSpace(strings.ToLower(v.ProviderName)), + RegionName: strings.TrimSpace(v.RegionName), + InstanceType: strings.TrimSpace(v.InstanceType), + Image: strings.TrimSpace(v.Image), + } + recommendSpecs = append(recommendSpecs, param) + } + } + + if len(req.RecommendSpecsWithFormat) > 0 { + for _, v := range req.RecommendSpecsWithFormat { + + ci := strings.TrimSpace(v.CommonImage) + cs := strings.TrimSpace(v.CommonSpec) + + splitedCommonImage := strings.Split(ci, "+") + splitedCommonSpec := strings.Split(cs, "+") + + if len(splitedCommonImage) != 3 || len(splitedCommonSpec) != 3 { + utils.LogErrorf("common image and spec format is not correct; image: %s; spec: %s", ci, cs) + continue + } + + if splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1] { + utils.LogErrorf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs) + continue + } + + param := cost.RecommendSpecParam{ + ProviderName: strings.TrimSpace(strings.ToLower(splitedCommonImage[0])), + RegionName: strings.TrimSpace(splitedCommonImage[1]), + InstanceType: strings.TrimSpace(splitedCommonSpec[2]), + Image: strings.TrimSpace(splitedCommonImage[2]), + } + recommendSpecs = append(recommendSpecs, param) + } + } + + arg := cost.EstimateForecastCostParam{ + RecommendSpecs: recommendSpecs, + TimeStandard: time.Date(pastTime.Year(), pastTime.Month(), pastTime.Day(), 0, 0, 0, 0, pastTime.Location()), + PricePolicy: constant.OnDemand, + } + + res, err := a.services.costService.EstimateForecastCost(arg) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, err.Error()) + } + + return successResponseJson( + c, + "retrieved pricing information", + res, + ) +} + // @Id UpdatePriceInfos // @Summar Update Price Information // @Description Retrieve pricing information for cloud resources based on specified parameters. If saved data is more than 7 days, fetch new data and insert new price data even if same price as before. diff --git a/internal/app/cost_estimation_req.go b/internal/app/cost_estimation_req.go index f857555..34a6448 100644 --- a/internal/app/cost_estimation_req.go +++ b/internal/app/cost_estimation_req.go @@ -2,6 +2,20 @@ package app import "github.com/cloud-barista/cm-ant/internal/core/common/constant" +type EstimateForecastCostReq struct { + RecommendSpecs []struct { + ProviderName string `json:"providerName"` + RegionName string `json:"regionName"` + InstanceType string `json:"instanceType"` + Image string `json:"image"` + } `json:"recommendSpecs" validate:"required"` + + RecommendSpecsWithFormat []struct { + CommonImage string `json:"commonImage"` + CommonSpec string `json:"commonSpec"` + } `json:"recommendSpecsWithFormat" validate:"required"` +} + type UpdatePriceInfosReq struct { ProviderName string `json:"providerName" validate:"required"` RegionName string `json:"regionName" validate:"required"` diff --git a/internal/app/router.go b/internal/app/router.go index 4a05fdb..1cfaa68 100644 --- a/internal/app/router.go +++ b/internal/app/router.go @@ -56,8 +56,13 @@ func (server *AntServer) InitRouter() error { } } } - + + { + costEstimationHandler := versionRouter.Group("/cost-estimation") + + costEstimationHandler.POST("/forecast", server.estimateForecastCost) + priceRouter := versionRouter.Group("/price") { priceRouter.POST("/info", server.updatePriceInfos) diff --git a/internal/config/config.go b/internal/config/config.go index 9364f12..f92162b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "strings" + "time" "github.com/cloud-barista/cm-ant/internal/utils" "github.com/spf13/viper" @@ -31,6 +32,14 @@ type AntConfig struct { Username string `yaml:"username"` Password string `yaml:"password"` } `yaml:"tumblebug"` + + Cost struct { + Estimation struct { + Forcast struct { + PriceUpdateInterval time.Duration `yaml:"priceUpdateInterval"` + } `yaml:"forecast"` + } `yaml:"estimation"` + } `yaml:"cost"` Load struct { Retry int `yaml:"retry"` JMeter struct { diff --git a/internal/core/cost/dtos.go b/internal/core/cost/dtos.go index ebead09..cae46f9 100644 --- a/internal/core/cost/dtos.go +++ b/internal/core/cost/dtos.go @@ -1,11 +1,72 @@ package cost import ( + "crypto/sha256" + "encoding/hex" "time" "github.com/cloud-barista/cm-ant/internal/core/common/constant" ) +type EstimateForecastCostParam struct { + RecommendSpecs []RecommendSpecParam `json:"recommendSpecs"` + + TimeStandard time.Time `json:"timeStandard"` + PricePolicy constant.PricePolicy `json:"pricePolicy"` +} + +type RecommendSpecParam struct { + ProviderName string `json:"providerName"` + RegionName string `json:"regionName"` + InstanceType string `json:"instanceType"` + Image string `json:"image"` +} + +func (r RecommendSpecParam) Hash() string { + h := sha256.New() + + h.Write([]byte(r.ProviderName)) + h.Write([]byte(r.RegionName)) + h.Write([]byte(r.InstanceType)) + h.Write([]byte(r.Image)) + + hashBytes := h.Sum(nil) + return hex.EncodeToString(hashBytes) +} + +type EstimateForecastCostResult struct { + TotalMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` + TotalMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` + EsimateForecastCostSpecResults []EsimateForecastCostSpecResult `json:"esimateForecastCostSpecResults"` +} + +type EsimateForecastCostSpecResult struct { + ProviderName string `json:"providerName"` + RegionName string `json:"regionName"` + InstanceType string `json:"instanceType"` + ImageName string `json:"imageName"` + SpecMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` + SpecMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` + EstimateForecastCostSpecDetailResults []EstimateForecastCostSpecDetailResult `json:"estimateForecastCostSpecDetailResults"` +} + +type EstimateForecastCostSpecDetailResult struct { + ID uint `json:"id"` + VCpu string `json:"vCpu,omitempty"` + Memory string `json:"memory,omitempty"` + Storage string `json:"storage,omitempty"` + OsType string `json:"osType,omitempty"` + ProductDescription string `json:"productDescription,omitempty"` + OriginalPricePolicy string `json:"originalPricePolicy,omitempty"` + PricePolicy constant.PricePolicy `json:"pricePolicy,omitempty"` + Unit constant.PriceUnit `json:"unit,omitempty"` + Currency constant.PriceCurrency `json:"currency,omitempty"` + Price string `json:"price,omitempty"` + CalculatedMonthlyPrice float64 `json:"calculatedMonthlyPrice,omitempty"` + PriceDescription string `json:"priceDescription,omitempty"` + LastUpdatedAt time.Time `json:"lastUpdatedAt,omitempty"` +} + type UpdatePriceInfosParam struct { MigrationId string ProviderName string diff --git a/internal/core/cost/models.go b/internal/core/cost/models.go index b779fc7..4eb2c1e 100644 --- a/internal/core/cost/models.go +++ b/internal/core/cost/models.go @@ -31,6 +31,8 @@ type PriceInfo struct { OriginalCurrency string CalculatedMonthlyPrice float64 `gorm:"index"` PriceDescription string + LastUpdatedAt time.Time + ImageName string `gorm:"index"` } type CostInfos []CostInfo diff --git a/internal/core/cost/price_collector.go b/internal/core/cost/price_collector.go index 9403315..f68e362 100644 --- a/internal/core/cost/price_collector.go +++ b/internal/core/cost/price_collector.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "time" "unicode" "github.com/cloud-barista/cm-ant/internal/core/common/constant" @@ -17,6 +18,7 @@ import ( type PriceCollector interface { Readyz(context.Context) error GetPriceInfos(context.Context, UpdatePriceInfosParam) (PriceInfos, error) + FetchPriceInfos(context.Context, RecommendSpecParam) (PriceInfos, error) } var ( @@ -191,6 +193,153 @@ func (s *SpiderPriceCollector) GetPriceInfos(ctx context.Context, param UpdatePr return createdPriceInfo, nil } +func (s *SpiderPriceCollector) FetchPriceInfos(ctx context.Context, param RecommendSpecParam) (PriceInfos, error) { + connectionName := fmt.Sprintf("%s-%s", strings.ToLower(param.ProviderName), strings.ToLower(param.RegionName)) + + req := spider.PriceInfoReq{ + ConnectionName: connectionName, + FilterList: s.generateFilter(param), + } + + result, err := s.sc.GetPriceInfoWithContext(ctx, param.RegionName, req) + + if err != nil { + + if strings.Contains(err.Error(), "you don't have any permission") { + return nil, fmt.Errorf("you don't have permission to query the price for %s", param.ProviderName) + } + return nil, err + } + + createdPriceInfo := make([]*PriceInfo, 0) + if result.CloudPriceList != nil { + for i := range result.CloudPriceList { + p := result.CloudPriceList[i] + + if p.PriceList != nil { + for j := range p.PriceList { + + pl := p.PriceList[j] + + productInfo := pl.ProductInfo + vCpu := s.naChecker(productInfo.Vcpu) + originalMemory := s.naChecker(productInfo.Memory) + + if vCpu == "" || originalMemory == "" { + continue + } + + memory, memoryUnit := s.splitMemory(originalMemory) + zoneName := s.naChecker(productInfo.ZoneName) + osType := s.naChecker(productInfo.OperatingSystem) + storage := s.naChecker(productInfo.Storage) + productDescription := s.naChecker(productInfo.Description) + + var price, originalCurrency, originalUnit, priceDescription string + var unit constant.PriceUnit + var currency constant.PriceCurrency + + priceInfo := pl.PriceInfo + + if priceInfo.PricingPolicies != nil { + for k := range priceInfo.PricingPolicies { + policy := priceInfo.PricingPolicies[k] + originalPricePolicy := s.naChecker(policy.PricingPolicy) + priceDescription = s.naChecker(policy.Description) + originalCurrency = s.naChecker(policy.Currency) + originalUnit = s.naChecker(policy.Unit) + unit = s.parseUnit(originalUnit) + currency = s.parseCurrency(policy.Currency) + convertedPrice, err := strconv.ParseFloat(policy.Price, 64) + if err != nil { + utils.LogWarnf("not allowed for error; %s", err) + continue + } + + if convertedPrice == float64(0) { + utils.LogWarn("not allowed for empty price") + continue + } + price = s.naChecker(policy.Price) + + if price == "" { + utils.LogWarn("not allowed for empty price") + continue + } + + if strings.Contains(strings.ToLower(priceDescription), "dedicated") { + utils.LogWarnf("not allowed for dedicated instance hour; %s", priceDescription) + continue + } + + pi := PriceInfo{ + ProviderName: param.ProviderName, + RegionName: productInfo.RegionName, + InstanceType: productInfo.InstanceType, + ZoneName: zoneName, + VCpu: vCpu, + OriginalMemory: originalMemory, + Memory: memory, + MemoryUnit: memoryUnit, + Storage: storage, + OsType: osType, + ProductDescription: productDescription, + OriginalPricePolicy: originalPricePolicy, + PricePolicy: constant.OnDemand, + Price: price, + Currency: currency, + Unit: unit, + OriginalUnit: originalUnit, + OriginalCurrency: originalCurrency, + PriceDescription: priceDescription, + CalculatedMonthlyPrice: s.calculatePrice(price, unit), + LastUpdatedAt: time.Now(), + ImageName: param.Image, + } + + if !priceValidator[param.ProviderName](&pi) { + continue + } + + createdPriceInfo = append(createdPriceInfo, &pi) + } + } + + } + } + } + } + + sort.Slice(createdPriceInfo, func(i, j int) bool { + return createdPriceInfo[i].Price < createdPriceInfo[j].Price + }) + + return createdPriceInfo, nil +} + +func (s *SpiderPriceCollector) generateFilter(param RecommendSpecParam) []spider.FilterReq { + + providerName := strings.ToLower(param.ProviderName) + param.ProviderName = providerName + + ret := []spider.FilterReq{ + { + Key: "pricingPolicy", + Value: onDemandPricingPolicyMap[providerName], + }, + { + Key: "regionName", + Value: param.RegionName, + }, + { + Key: "instanceType", + Value: param.InstanceType, + }, + } + + return ret +} + func (s *SpiderPriceCollector) generateFilterList(param UpdatePriceInfosParam) []spider.FilterReq { providerName := strings.ToLower(param.ProviderName) diff --git a/internal/core/cost/repository.go b/internal/core/cost/repository.go index e0c9b4f..d7ebbd8 100644 --- a/internal/core/cost/repository.go +++ b/internal/core/cost/repository.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "github.com/cloud-barista/cm-ant/internal/core/common/constant" "gorm.io/gorm" @@ -82,6 +83,35 @@ func (r *CostRepository) GetAllMatchingPriceInfoList(ctx context.Context, param return priceInfoList, err } +func (r *CostRepository) GetMatchingForecastCost(ctx context.Context, param RecommendSpecParam, timeStandard time.Time, pricePolicy constant.PricePolicy) (PriceInfos, error) { + var priceInfos []*PriceInfo + + err := r.execInTransaction(ctx, func(d *gorm.DB) error { + q := d.Model(&PriceInfo{}). + Where( + "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND instance_type = ? AND image_name = ? AND price_policy = ? AND last_updated_at >= ?", + strings.ToLower(param.ProviderName), + strings.ToLower(param.RegionName), + strings.ToLower(param.InstanceType), + strings.ToLower(param.Image), + pricePolicy, + timeStandard, + ) + + if err := q.Find(&priceInfos).Error; err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + return priceInfos, nil +} + func (r *CostRepository) CountMatchingPriceInfoList(ctx context.Context, param UpdatePriceInfosParam) (int64, error) { var totalCount int64 @@ -103,7 +133,7 @@ func (r *CostRepository) CountMatchingPriceInfoList(ctx context.Context, param U } -func (r *CostRepository) BatchInsertAllResult(ctx context.Context, param UpdatePriceInfosParam, created PriceInfos) error { +func (r *CostRepository) BatchInsertAllForecastCostResult(ctx context.Context, created PriceInfos) error { batchSize := 100 err := r.execInTransaction(ctx, func(d *gorm.DB) error { diff --git a/internal/core/cost/service.go b/internal/core/cost/service.go index 3976ec2..0522322 100644 --- a/internal/core/cost/service.go +++ b/internal/core/cost/service.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "math" "strings" + "sync" "time" "github.com/cloud-barista/cm-ant/internal/core/common/constant" @@ -52,6 +54,143 @@ func (c *CostService) Readyz() error { return nil } +var forecastUpdateLockMap sync.Map + +func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (EstimateForecastCostResult, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + var wg sync.WaitGroup + var mu sync.Mutex + var results []EsimateForecastCostSpecResult + var errList []error + var esimateForecastCostSpecResult EstimateForecastCostResult + + utils.LogInfof("Fetching price information for spec: %+v", param) + + for _, v := range param.RecommendSpecs { + wg.Add(1) + go func(p RecommendSpecParam) { + defer wg.Done() + + // memory lock + rl, _ := forecastUpdateLockMap.LoadOrStore(p.Hash(), &sync.Mutex{}) + lock := rl.(*sync.Mutex) + + lock.Lock() + defer lock.Unlock() + + priceInfos, err := c.costRepo.GetMatchingForecastCost(ctx, v, param.TimeStandard, param.PricePolicy) + if err != nil { + mu.Lock() + errList = append(errList, err) + mu.Unlock() + utils.LogErrorf("Error fetching price info for spec %+v: %v", v, err) + + return + + } + + if len(priceInfos) == 0 { + utils.LogInfof("No matching forecast cost found for spec: %+v, fetching from price collector", v) + + resList, err := c.priceCollector.FetchPriceInfos(ctx, v) + if err != nil { + mu.Lock() + errList = append(errList, fmt.Errorf("error retrieving prices for %+v: %w", v, err)) + mu.Unlock() + return + } + + if len(resList) > 0 { + utils.LogInfof("Inserting fetched price results for spec: %+v", v) + + err = c.costRepo.BatchInsertAllForecastCostResult(ctx, resList) + if err != nil { + mu.Lock() + errList = append(errList, fmt.Errorf("error batch inserting results for %+v: %w", v, err)) + mu.Unlock() + return + } + } + priceInfos = resList + } + + if len(priceInfos) > 0 { + + minPrice := float64(math.MaxFloat64) + maxPrice := float64(math.SmallestNonzeroFloat64) + + res := EsimateForecastCostSpecResult{ + ProviderName: v.ProviderName, + RegionName: v.RegionName, + InstanceType: v.InstanceType, + ImageName: v.Image, + EstimateForecastCostSpecDetailResults: make([]EstimateForecastCostSpecDetailResult, 0), + } + + for _, v := range priceInfos { + + calculatedPrice := v.CalculatedMonthlyPrice + utils.LogInfof("Price calculated for spec %+v: %f", v, calculatedPrice) + + if calculatedPrice < minPrice { + minPrice = calculatedPrice + } + if calculatedPrice > maxPrice { + maxPrice = calculatedPrice + } + + specDetail := EstimateForecastCostSpecDetailResult{ + ID: v.ID, + VCpu: v.VCpu, + Memory: fmt.Sprintf("%s %s", v.Memory, v.MemoryUnit), + Storage: v.Storage, + OsType: v.OsType, + ProductDescription: v.ProductDescription, + OriginalPricePolicy: v.OriginalPricePolicy, + PricePolicy: v.PricePolicy, + Unit: v.Unit, + Currency: v.Currency, + Price: v.Price, + CalculatedMonthlyPrice: calculatedPrice, + PriceDescription: v.PriceDescription, + LastUpdatedAt: v.LastUpdatedAt, + } + + res.SpecMinMonthlyPrice = minPrice + res.SpecMaxMonthlyPrice = maxPrice + res.EstimateForecastCostSpecDetailResults = append(res.EstimateForecastCostSpecDetailResults, specDetail) + } + + mu.Lock() + results = append(results, res) + mu.Unlock() + utils.LogInfof("Successfully calculated forecast cost for spec: %+v", param) + } + + }(v) + } + wg.Wait() + + if len(errList) > 0 { + return esimateForecastCostSpecResult, fmt.Errorf("errors occurred during processing: %v", errList) + } + + if len(results) > 0 { + esimateForecastCostSpecResult.EsimateForecastCostSpecResults = results + + for _, v := range results { + esimateForecastCostSpecResult.TotalMinMonthlyPrice += v.SpecMinMonthlyPrice + esimateForecastCostSpecResult.TotalMaxMonthlyPrice += v.SpecMaxMonthlyPrice + } + utils.LogInfof("Total min monthly price: %f, Total max monthly price: %f", esimateForecastCostSpecResult.TotalMinMonthlyPrice, esimateForecastCostSpecResult.TotalMaxMonthlyPrice) + + } + + return esimateForecastCostSpecResult, nil +} + func (c *CostService) UpdatePriceInfos(param UpdatePriceInfosParam) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() @@ -75,7 +214,7 @@ func (c *CostService) UpdatePriceInfos(param UpdatePriceInfosParam) error { } if len(resList) > 0 { - err := c.costRepo.BatchInsertAllResult(ctx, param, resList) + err := c.costRepo.BatchInsertAllForecastCostResult(ctx, resList) if err != nil { return err } diff --git a/internal/utils/log.go b/internal/utils/log.go index 11c3e23..ce685bb 100644 --- a/internal/utils/log.go +++ b/internal/utils/log.go @@ -6,9 +6,10 @@ import ( ) const ( - colorReset = "\033[0m" - colorRed = "\033[31m" - colorGreen = "\033[32m" + colorReset = "\033[0m" + colorRed = "\033[31m" + colorGreen = "\033[32m" + colorYellow = "\033[33m" ) // LogLevel type to represent different log levels @@ -16,6 +17,7 @@ type LogLevel string const ( Info LogLevel = "INFO" + Warn LogLevel = "WARN" Error LogLevel = "ERROR" ) @@ -25,6 +27,8 @@ func Log(level LogLevel, v ...interface{}) { switch level { case Info: color = colorGreen + case Warn: + color = colorYellow case Error: color = colorRed default: @@ -56,6 +60,14 @@ func LogInfof(format string, v ...interface{}) { Logf(Info, format, v...) } +func LogWarn(v ...interface{}) { + Log(Warn, v...) +} + +func LogWarnf(format string, v ...interface{}) { + Logf(Warn, format, v...) +} + func LogError(v ...interface{}) { Log(Error, v...) } From ab9546bdbd52bffe599272992891757b8f19a204 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Fri, 18 Oct 2024 17:38:07 +0900 Subject: [PATCH 2/8] update endpoint for cost estimation forecast --- api/docs.go | 2 +- api/swagger.json | 2 +- api/swagger.yaml | 2 +- internal/app/cost_estimation_handler.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/docs.go b/api/docs.go index a8c982f..d959c15 100644 --- a/api/docs.go +++ b/api/docs.go @@ -16,7 +16,7 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/api/v1/cost/forecast": { + "/api/v1/cost-estimation/forecast": { "post": { "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", "consumes": [ diff --git a/api/swagger.json b/api/swagger.json index 10db7bd..bb509c6 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -8,7 +8,7 @@ }, "basePath": "/ant", "paths": { - "/api/v1/cost/forecast": { + "/api/v1/cost-estimation/forecast": { "post": { "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", "consumes": [ diff --git a/api/swagger.yaml b/api/swagger.yaml index e31ff2a..351a80c 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -782,7 +782,7 @@ info: title: CM-ANT REST API version: 0.2.2 paths: - /api/v1/cost/forecast: + /api/v1/cost-estimation/forecast: post: consumes: - application/json diff --git a/internal/app/cost_estimation_handler.go b/internal/app/cost_estimation_handler.go index 5af8495..b8c5b2a 100644 --- a/internal/app/cost_estimation_handler.go +++ b/internal/app/cost_estimation_handler.go @@ -23,7 +23,7 @@ import ( // @Success 200 {object} app.AntResponse[cost.EstimateForecastCostResult] "Successfully estimated forecast cost" // @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" // @Failure 500 {object} app.AntResponse[string] "Failed to estimate forecast cost" -// @Router /api/v1/cost/forecast [post] +// @Router /api/v1/cost-estimation/forecast [post] func (a *AntServer) estimateForecastCost(c echo.Context) error { var req EstimateForecastCostReq if err := c.Bind(&req); err != nil { From 72fce61f9a9222e262871a4534780a44d27db6b1 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Fri, 18 Oct 2024 17:41:30 +0900 Subject: [PATCH 3/8] update invalid request return for formated recommend spec --- internal/app/cost_estimation_handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cost_estimation_handler.go b/internal/app/cost_estimation_handler.go index b8c5b2a..517934a 100644 --- a/internal/app/cost_estimation_handler.go +++ b/internal/app/cost_estimation_handler.go @@ -61,12 +61,12 @@ func (a *AntServer) estimateForecastCost(c echo.Context) error { if len(splitedCommonImage) != 3 || len(splitedCommonSpec) != 3 { utils.LogErrorf("common image and spec format is not correct; image: %s; spec: %s", ci, cs) - continue + return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec format is not correct; image: %s; spec: %s", ci, cs)) } if splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1] { utils.LogErrorf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs) - continue + return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs)) } param := cost.RecommendSpecParam{ From 10cd57baeb807c058d23951bf16ade998ecb9415 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Mon, 21 Oct 2024 09:31:08 +0900 Subject: [PATCH 4/8] add config and update properties for request --- api/docs.go | 16 ++++++++--- api/swagger.json | 16 ++++++++--- api/swagger.yaml | 14 +++++++--- config.yaml | 8 +++--- internal/app/cost_estimation_handler.go | 36 ++++++++++++++----------- internal/app/cost_estimation_req.go | 16 +++++------ 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/api/docs.go b/api/docs.go index d959c15..7015d1f 100644 --- a/api/docs.go +++ b/api/docs.go @@ -1407,14 +1407,19 @@ const docTemplate = `{ "app.EstimateForecastCostReq": { "type": "object", "required": [ - "recommendSpecs", - "recommendSpecsWithFormat" + "specs", + "specsWithFormat" ], "properties": { - "recommendSpecs": { + "specs": { "type": "array", "items": { "type": "object", + "required": [ + "instanceType", + "providerName", + "regionName" + ], "properties": { "image": { "type": "string" @@ -1431,10 +1436,13 @@ const docTemplate = `{ } } }, - "recommendSpecsWithFormat": { + "specsWithFormat": { "type": "array", "items": { "type": "object", + "required": [ + "commonSpec" + ], "properties": { "commonImage": { "type": "string" diff --git a/api/swagger.json b/api/swagger.json index bb509c6..c2ad570 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1399,14 +1399,19 @@ "app.EstimateForecastCostReq": { "type": "object", "required": [ - "recommendSpecs", - "recommendSpecsWithFormat" + "specs", + "specsWithFormat" ], "properties": { - "recommendSpecs": { + "specs": { "type": "array", "items": { "type": "object", + "required": [ + "instanceType", + "providerName", + "regionName" + ], "properties": { "image": { "type": "string" @@ -1423,10 +1428,13 @@ } } }, - "recommendSpecsWithFormat": { + "specsWithFormat": { "type": "array", "items": { "type": "object", + "required": [ + "commonSpec" + ], "properties": { "commonImage": { "type": "string" diff --git a/api/swagger.yaml b/api/swagger.yaml index 351a80c..e1b4544 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -204,7 +204,7 @@ definitions: type: object app.EstimateForecastCostReq: properties: - recommendSpecs: + specs: items: properties: image: @@ -215,20 +215,26 @@ definitions: type: string regionName: type: string + required: + - instanceType + - providerName + - regionName type: object type: array - recommendSpecsWithFormat: + specsWithFormat: items: properties: commonImage: type: string commonSpec: type: string + required: + - commonSpec type: object type: array required: - - recommendSpecs - - recommendSpecsWithFormat + - specs + - specsWithFormat type: object app.InstallLoadGeneratorReq: properties: diff --git a/config.yaml b/config.yaml index 8585a21..0a9f4db 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,3 @@ -root: - path: - server: port: 8880 @@ -16,6 +13,11 @@ tumblebug: username: default password: default +cost: + estimation: + forecast: + priceUpdateInterval: 7d + load: retry: 2 jmeter: diff --git a/internal/app/cost_estimation_handler.go b/internal/app/cost_estimation_handler.go index 517934a..da94e1e 100644 --- a/internal/app/cost_estimation_handler.go +++ b/internal/app/cost_estimation_handler.go @@ -30,7 +30,7 @@ func (a *AntServer) estimateForecastCost(c echo.Context) error { return errorResponseJson(http.StatusBadRequest, err.Error()) } - if len(req.RecommendSpecs) == 0 && len(req.RecommendSpecsWithFormat) == 0 { + if len(req.Specs) == 0 && len(req.SpecsWithFormat) == 0 { return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") } @@ -38,8 +38,8 @@ func (a *AntServer) estimateForecastCost(c echo.Context) error { recommendSpecs := make([]cost.RecommendSpecParam, 0) - if len(req.RecommendSpecs) > 0 { - for _, v := range req.RecommendSpecs { + if len(req.Specs) > 0 { + for _, v := range req.Specs { param := cost.RecommendSpecParam{ ProviderName: strings.TrimSpace(strings.ToLower(v.ProviderName)), RegionName: strings.TrimSpace(v.RegionName), @@ -50,31 +50,37 @@ func (a *AntServer) estimateForecastCost(c echo.Context) error { } } - if len(req.RecommendSpecsWithFormat) > 0 { - for _, v := range req.RecommendSpecsWithFormat { + if len(req.SpecsWithFormat) > 0 { + delim := "+" + + for _, v := range req.SpecsWithFormat { - ci := strings.TrimSpace(v.CommonImage) cs := strings.TrimSpace(v.CommonSpec) + ci := strings.TrimSpace(v.CommonImage) - splitedCommonImage := strings.Split(ci, "+") - splitedCommonSpec := strings.Split(cs, "+") + splitedCommonSpec := strings.Split(cs, delim) + splitedCommonImage := strings.Split(ci, delim) - if len(splitedCommonImage) != 3 || len(splitedCommonSpec) != 3 { - utils.LogErrorf("common image and spec format is not correct; image: %s; spec: %s", ci, cs) - return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec format is not correct; image: %s; spec: %s", ci, cs)) + if len(splitedCommonSpec) != 3 { + utils.LogErrorf("common spec format is not correct; image: %s; spec: %s", ci, cs) + return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common spec format is not correct; image: %s; spec: %s", ci, cs)) } - if splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1] { + if len(splitedCommonImage) == 3 && (splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1]) { utils.LogErrorf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs) return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs)) } param := cost.RecommendSpecParam{ - ProviderName: strings.TrimSpace(strings.ToLower(splitedCommonImage[0])), - RegionName: strings.TrimSpace(splitedCommonImage[1]), + ProviderName: strings.TrimSpace(strings.ToLower(splitedCommonSpec[0])), + RegionName: strings.TrimSpace(splitedCommonSpec[1]), InstanceType: strings.TrimSpace(splitedCommonSpec[2]), - Image: strings.TrimSpace(splitedCommonImage[2]), } + + if len(splitedCommonImage) == 3 { + param.Image = strings.TrimSpace(splitedCommonImage[2]) + } + recommendSpecs = append(recommendSpecs, param) } } diff --git a/internal/app/cost_estimation_req.go b/internal/app/cost_estimation_req.go index 34a6448..f951277 100644 --- a/internal/app/cost_estimation_req.go +++ b/internal/app/cost_estimation_req.go @@ -3,17 +3,17 @@ package app import "github.com/cloud-barista/cm-ant/internal/core/common/constant" type EstimateForecastCostReq struct { - RecommendSpecs []struct { - ProviderName string `json:"providerName"` - RegionName string `json:"regionName"` - InstanceType string `json:"instanceType"` + Specs []struct { + ProviderName string `json:"providerName" validate:"required"` + RegionName string `json:"regionName" validate:"required"` + InstanceType string `json:"instanceType" validate:"required"` Image string `json:"image"` - } `json:"recommendSpecs" validate:"required"` + } `json:"specs" validate:"required"` - RecommendSpecsWithFormat []struct { + SpecsWithFormat []struct { + CommonSpec string `json:"commonSpec" validate:"required"` CommonImage string `json:"commonImage"` - CommonSpec string `json:"commonSpec"` - } `json:"recommendSpecsWithFormat" validate:"required"` + } `json:"specsWithFormat" validate:"required"` } type UpdatePriceInfosReq struct { From 059093543c6f41920dcf3b5c5a2b7f9ebdcf8370 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Wed, 30 Oct 2024 15:01:32 +0900 Subject: [PATCH 5/8] update cost estimate endpoint and handlers --- api/docs.go | 610 +++++++++--------- api/swagger.json | 610 +++++++++--------- api/swagger.yaml | 476 +++++++------- cmd/cm-ant/main.go | 74 ++- config.yaml | 5 +- docker-compose.yaml | 5 +- internal/app/cost_estimation_handler.go | 326 ---------- internal/app/estimate_cost_handler.go | 363 +++++++++++ ...estimation_req.go => estimate_cost_req.go} | 48 +- internal/app/middlewares.go | 168 +++-- internal/app/router.go | 21 +- internal/app/server.go | 2 +- internal/config/config.go | 17 +- internal/core/cost/cost_collector.go | 113 +++- internal/core/cost/dtos.go | 102 +-- internal/core/cost/models.go | 17 +- internal/core/cost/price_collector.go | 167 +---- internal/core/cost/repository.go | 79 +-- internal/core/cost/service.go | 180 +++--- internal/infra/db/db.go | 4 +- 20 files changed, 1721 insertions(+), 1666 deletions(-) delete mode 100644 internal/app/cost_estimation_handler.go create mode 100644 internal/app/estimate_cost_handler.go rename internal/app/{cost_estimation_req.go => estimate_cost_req.go} (71%) diff --git a/api/docs.go b/api/docs.go index 7015d1f..7f8aff9 100644 --- a/api/docs.go +++ b/api/docs.go @@ -16,9 +16,93 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/api/v1/cost-estimation/forecast": { + "/api/v1/cost/estimate": { + "get": { + "description": "Fetch estimated cost details based on provider, region, instance type, and resource specifications. Pagination support is provided through ` + "`" + `Page` + "`" + ` and ` + "`" + `Size` + "`" + ` parameters.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimate]" + ], + "summary": "Retrieve Estimated Cost Information", + "operationId": "GetEstimateCost", + "parameters": [ + { + "type": "string", + "description": "Cloud provider name to filter estimated costs", + "name": "providerName", + "in": "query" + }, + { + "type": "string", + "description": "Region name to filter estimated costs", + "name": "regionName", + "in": "query" + }, + { + "type": "string", + "description": "Instance type to filter estimated costs", + "name": "instanceType", + "in": "query" + }, + { + "type": "string", + "description": "Number of vCPUs to filter estimated costs", + "name": "vCpu", + "in": "query" + }, + { + "type": "string", + "description": "Memory size to filter estimated costs", + "name": "memory", + "in": "query" + }, + { + "type": "string", + "description": "Operating system type to filter estimated costs", + "name": "osType", + "in": "query" + }, + { + "type": "integer", + "description": "Page number for pagination (default: 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Number of records per page (default: 100, max: 100)", + "name": "size", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved estimated cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_EstimateCostInfoResults" + } + }, + "400": { + "description": "Invalid request parameters", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Failed to retrieve estimated cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + }, "post": { - "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", + "description": "Update the estimate cost based on provided specifications and retrieve the updated cost estimation. Required fields for each specification include ` + "`" + `ProviderName` + "`" + `, ` + "`" + `RegionName` + "`" + `, and ` + "`" + `InstanceType` + "`" + `. Specifications can also be provided in a formatted string using ` + "`" + `+` + "`" + ` delimiter.", "consumes": [ "application/json" ], @@ -26,36 +110,36 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Cost Estimation]" + "[Cost Estimate]" ], - "summary": "Estimate Forecast Cost", - "operationId": "EstimateForecastCost", + "summary": "Update and Retrieve Estimated Cost Information", + "operationId": "UpdateAndGetEstimateCost", "parameters": [ { - "description": "Request body containing estimation parameters", + "description": "Request body for updating and retrieving estimated cost information", "name": "body", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/app.EstimateForecastCostReq" + "$ref": "#/definitions/app.UpdateAndGetEstimateCostReq" } } ], "responses": { "200": { - "description": "Successfully estimated forecast cost", + "description": "Successfully updated and retrieved estimated cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-cost_EstimateForecastCostResult" + "$ref": "#/definitions/app.AntResponse-cost_EstimateCostResults" } }, "400": { - "description": "Invalid request parameters", + "description": "Invalid request parameters or format", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to estimate forecast cost", + "description": "Failed to update or retrieve estimated cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -63,9 +147,9 @@ const docTemplate = `{ } } }, - "/api/v1/cost/info": { + "/api/v1/cost/estimate/forecast": { "get": { - "description": "Retrieve cost information for specified parameters within a defined date range. The date range must be within a 6-month period. Optionally, you can specify cost aggregation type and date order for the results.", + "description": "Fetch estimated forecast cost data based on specified parameters, including a date range that must be within 6 months. Supports pagination and filtering by namespace IDs, migration configuration IDs, and resource types.", "consumes": [ "application/json" ], @@ -73,21 +157,21 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Cost Management]" + "[Cost Estimate]" ], - "summary": "Get Cost Information", - "operationId": "GetCostInfo", + "summary": "Retrieve Estimated Forecast Cost Information", + "operationId": "GetEstimateForecastCost", "parameters": [ { "type": "string", - "description": "Start date for the cost information retrieval in 'YYYY-MM-DD' format", + "description": "Start date for the forecast cost retrieval in 'YYYY-MM-DD' format", "name": "startDate", "in": "query", "required": true }, { "type": "string", - "description": "End date for the cost information retrieval in 'YYYY-MM-DD' format", + "description": "End date for the forecast cost retrieval in 'YYYY-MM-DD' format", "name": "endDate", "in": "query", "required": true @@ -98,8 +182,8 @@ const docTemplate = `{ "type": "string" }, "collectionFormat": "csv", - "description": "List of migration IDs to filter the cost information", - "name": "migrationIds", + "description": "List of namespace IDs to filter forecast cost information", + "name": "nsIds", "in": "query" }, { @@ -108,8 +192,8 @@ const docTemplate = `{ "type": "string" }, "collectionFormat": "csv", - "description": "List of cloud providers to filter the cost information", - "name": "provider", + "description": "List of migration configuration IDs to filter forecast cost information", + "name": "mciIds", "in": "query" }, { @@ -118,7 +202,17 @@ const docTemplate = `{ "type": "string" }, "collectionFormat": "csv", - "description": "List of resource types to filter the cost information", + "description": "List of cloud providers to filter forecast cost information", + "name": "providers", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "csv", + "description": "List of resource types to filter forecast cost information", "name": "resourceTypes", "in": "query" }, @@ -128,16 +222,15 @@ const docTemplate = `{ "type": "string" }, "collectionFormat": "csv", - "description": "List of resource IDs to filter the cost information", + "description": "List of resource IDs to filter forecast cost information", "name": "resourceIds", "in": "query" }, { "type": "string", - "description": "Type of cost aggregation for the results (e.g., 'daily', 'weekly', 'monthly')", + "description": "Type of cost aggregation (e.g., 'daily', 'weekly', 'monthly')", "name": "costAggregationType", - "in": "query", - "required": true + "in": "query" }, { "type": "string", @@ -150,23 +243,35 @@ const docTemplate = `{ "description": "Order of resource types in the result (e.g., 'asc', 'desc')", "name": "resourceTypeOrder", "in": "query" + }, + { + "type": "integer", + "description": "Page number for pagination (default: 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Number of records per page (default: 10000, max: 10000)", + "name": "size", + "in": "query" } ], "responses": { "200": { - "description": "Successfully retrieved cost information", + "description": "Successfully retrieved estimated forecast cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-array_cost_GetCostInfoResult" + "$ref": "#/definitions/app.AntResponse-cost_GetEstimateForecastCostInfoResults" } }, "400": { - "description": "Invalid request parameters", + "description": "Invalid request parameters or date format errors", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to retrieve cost information", + "description": "Failed to retrieve estimated forecast cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -174,7 +279,7 @@ const docTemplate = `{ } }, "post": { - "description": "Update cost information for specified resources, including details such as migration ID, cost resources, and additional AWS info if applicable. The request body must include a valid migration ID and a list of cost resources. If AWS-specific details are provided, ensure all required fields are populated.", + "description": "Update and retrieve forecasted cost estimates for a specified namespace and migration configuration ID over the past 14 days.", "consumes": [ "application/json" ], @@ -182,36 +287,36 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Cost Management]" + "[Cost Estimate]" ], - "summary": "Update Cost Information", - "operationId": "UpdateCostInfo", + "summary": "Update and Retrieve Estimated Forecast Cost", + "operationId": "UpdateEstimateForecastCost", "parameters": [ { - "description": "Request body containing cost update information", + "description": "Request body containing NsId (Namespace ID) and MciId (Migration Configuration ID) for cost estimation forecast", "name": "body", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/app.UpdateCostInfoReq" + "$ref": "#/definitions/app.UpdateEstimateForecastCostReq" } } ], "responses": { "200": { - "description": "Successfully updated cost information", + "description": "Successfully updated and retrieved estimated forecast cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-cost_UpdateCostInfoResult" + "$ref": "#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult" } }, "400": { - "description": "Invalid request parameters", + "description": "Request body binding error", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to update cost information", + "description": "Failed to update or retrieve forecast cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -934,126 +1039,6 @@ const docTemplate = `{ } } }, - "/api/v1/price/info": { - "get": { - "description": "Retrieve pricing information for cloud resources based on specified query parameters. Returns price data based on provider, region, instance type, vCPU, memory, and OS type. It offer instances with the lowest monthly prices what in the database.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Price Management" - ], - "summary": "Get Price Information", - "operationId": "GetPriceInfos", - "parameters": [ - { - "type": "string", - "description": "Cloud provider name - aws|alibaba|tencent|gcp|azure|ibm", - "name": "providerName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Region name", - "name": "regionName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Instance type", - "name": "instanceType", - "in": "query" - }, - { - "type": "string", - "description": "Number of vCPUs", - "name": "vCpu", - "in": "query" - }, - { - "type": "string", - "description": "Amount of memory", - "name": "memory", - "in": "query" - }, - { - "type": "string", - "description": "Operating system type", - "name": "osType", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "400": { - "description": "Invalid request parameters", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "500": { - "description": "Failed to retrieve pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - } - } - }, - "post": { - "description": "Retrieve pricing information for cloud resources based on specified parameters. If saved data is more than 7 days, fetch new data and insert new price data even if same price as before.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "[Price Management]" - ], - "operationId": "UpdatePriceInfos", - "parameters": [ - { - "description": "Request body containing get price information", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/app.UpdatePriceInfosReq" - } - } - ], - "responses": { - "200": { - "description": "Successfully retrieved pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "400": { - "description": "Invalid request parameters", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "500": { - "description": "Failed to retrieve pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - } - } - } - }, "/readyz": { "get": { "description": "This endpoint checks if the CB-Ant API server is ready by verifying the status of both the load service and the cost service. If either service is unavailable, it returns a 503 status indicating the server is not ready.", @@ -1092,7 +1077,7 @@ const docTemplate = `{ } }, "definitions": { - "app.AntResponse-array_cost_GetCostInfoResult": { + "app.AntResponse-array_load_LoadTestStatistics": { "type": "object", "properties": { "code": { @@ -1104,7 +1089,7 @@ const docTemplate = `{ "result": { "type": "array", "items": { - "$ref": "#/definitions/cost.GetCostInfoResult" + "$ref": "#/definitions/load.LoadTestStatistics" } }, "successMessage": { @@ -1112,7 +1097,7 @@ const docTemplate = `{ } } }, - "app.AntResponse-array_load_LoadTestStatistics": { + "app.AntResponse-array_load_MetricsSummary": { "type": "object", "properties": { "code": { @@ -1124,7 +1109,7 @@ const docTemplate = `{ "result": { "type": "array", "items": { - "$ref": "#/definitions/load.LoadTestStatistics" + "$ref": "#/definitions/load.MetricsSummary" } }, "successMessage": { @@ -1132,7 +1117,7 @@ const docTemplate = `{ } } }, - "app.AntResponse-array_load_MetricsSummary": { + "app.AntResponse-array_load_ResultSummary": { "type": "object", "properties": { "code": { @@ -1144,7 +1129,7 @@ const docTemplate = `{ "result": { "type": "array", "items": { - "$ref": "#/definitions/load.MetricsSummary" + "$ref": "#/definitions/load.ResultSummary" } }, "successMessage": { @@ -1152,7 +1137,7 @@ const docTemplate = `{ } } }, - "app.AntResponse-array_load_ResultSummary": { + "app.AntResponse-cost_EstimateCostInfoResults": { "type": "object", "properties": { "code": { @@ -1162,17 +1147,14 @@ const docTemplate = `{ "type": "string" }, "result": { - "type": "array", - "items": { - "$ref": "#/definitions/load.ResultSummary" - } + "$ref": "#/definitions/cost.EstimateCostInfoResults" }, "successMessage": { "type": "string" } } }, - "app.AntResponse-cost_EstimateForecastCostResult": { + "app.AntResponse-cost_EstimateCostResults": { "type": "object", "properties": { "code": { @@ -1182,14 +1164,14 @@ const docTemplate = `{ "type": "string" }, "result": { - "$ref": "#/definitions/cost.EstimateForecastCostResult" + "$ref": "#/definitions/cost.EstimateCostResults" }, "successMessage": { "type": "string" } } }, - "app.AntResponse-cost_UpdateCostInfoResult": { + "app.AntResponse-cost_GetEstimateForecastCostInfoResults": { "type": "object", "properties": { "code": { @@ -1199,7 +1181,24 @@ const docTemplate = `{ "type": "string" }, "result": { - "$ref": "#/definitions/cost.UpdateCostInfoResult" + "$ref": "#/definitions/cost.GetEstimateForecastCostInfoResults" + }, + "successMessage": { + "type": "string" + } + } + }, + "app.AntResponse-cost_UpdateEstimateForecastCostInfoResult": { + "type": "object", + "properties": { + "code": { + "type": "integer" + }, + "errorMessage": { + "type": "string" + }, + "result": { + "$ref": "#/definitions/cost.UpdateEstimateForecastCostInfoResult" }, "successMessage": { "type": "string" @@ -1376,85 +1375,6 @@ const docTemplate = `{ } } }, - "app.AwsAdditionalInfoReq": { - "type": "object", - "properties": { - "ownerId": { - "type": "string" - }, - "regions": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "app.CostResourceReq": { - "type": "object", - "properties": { - "resourceIds": { - "type": "array", - "items": { - "type": "string" - } - }, - "resourceType": { - "$ref": "#/definitions/constant.ResourceType" - } - } - }, - "app.EstimateForecastCostReq": { - "type": "object", - "required": [ - "specs", - "specsWithFormat" - ], - "properties": { - "specs": { - "type": "array", - "items": { - "type": "object", - "required": [ - "instanceType", - "providerName", - "regionName" - ], - "properties": { - "image": { - "type": "string" - }, - "instanceType": { - "type": "string" - }, - "providerName": { - "type": "string" - }, - "regionName": { - "type": "string" - } - } - } - }, - "specsWithFormat": { - "type": "array", - "items": { - "type": "object", - "required": [ - "commonSpec" - ], - "properties": { - "commonImage": { - "type": "string" - }, - "commonSpec": { - "type": "string" - } - } - } - } - } - }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1558,45 +1478,60 @@ const docTemplate = `{ } } }, - "app.UpdateCostInfoReq": { + "app.UpdateAndGetEstimateCostReq": { "type": "object", - "required": [ - "connectionName", - "costResources" - ], "properties": { - "awsAdditionalInfo": { - "$ref": "#/definitions/app.AwsAdditionalInfoReq" - }, - "connectionName": { - "type": "string" - }, - "costResources": { + "specs": { "type": "array", "items": { - "$ref": "#/definitions/app.CostResourceReq" + "type": "object", + "required": [ + "instanceType", + "providerName", + "regionName" + ], + "properties": { + "image": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + } + } } }, - "migrationId": { - "type": "string" + "specsWithFormat": { + "type": "array", + "items": { + "type": "object", + "required": [ + "commonSpec" + ], + "properties": { + "commonImage": { + "type": "string" + }, + "commonSpec": { + "type": "string" + } + } + } } } }, - "app.UpdatePriceInfosReq": { + "app.UpdateEstimateForecastCostReq": { "type": "object", - "required": [ - "instanceType", - "providerName", - "regionName" - ], "properties": { - "instanceType": { - "type": "string" - }, - "providerName": { + "mciId": { "type": "string" }, - "regionName": { + "nsId": { "type": "string" } } @@ -1672,28 +1607,13 @@ const docTemplate = `{ "PerYear" ] }, - "constant.ResourceType": { - "type": "string", - "enum": [ - "VM", - "VNet", - "DataDisk", - "Etc" - ], - "x-enum-varnames": [ - "VM", - "VNet", - "DataDisk", - "Etc" - ] - }, - "cost.EsimateForecastCostSpecResult": { + "cost.EsimateCostSpecResults": { "type": "object", "properties": { "estimateForecastCostSpecDetailResults": { "type": "array", "items": { - "$ref": "#/definitions/cost.EstimateForecastCostSpecDetailResult" + "$ref": "#/definitions/cost.EstimateCostSpecDetailResult" } }, "imageName": { @@ -1716,13 +1636,83 @@ const docTemplate = `{ } } }, - "cost.EstimateForecastCostResult": { + "cost.EstimateCostInfoResult": { + "type": "object", + "properties": { + "calculatedMonthlyPrice": { + "type": "number" + }, + "currency": { + "$ref": "#/definitions/constant.PriceCurrency" + }, + "id": { + "type": "integer" + }, + "instanceType": { + "type": "string" + }, + "lastUpdatedAt": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "originalPricePolicy": { + "type": "string" + }, + "osType": { + "type": "string" + }, + "price": { + "type": "string" + }, + "priceDescription": { + "type": "string" + }, + "pricePolicy": { + "$ref": "#/definitions/constant.PricePolicy" + }, + "productDescription": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "unit": { + "$ref": "#/definitions/constant.PriceUnit" + }, + "vCpu": { + "type": "string" + } + } + }, + "cost.EstimateCostInfoResults": { + "type": "object", + "properties": { + "estimateCostInfoResult": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EstimateCostInfoResult" + } + }, + "resultCount": { + "type": "integer" + } + } + }, + "cost.EstimateCostResults": { "type": "object", "properties": { "esimateForecastCostSpecResults": { "type": "array", "items": { - "$ref": "#/definitions/cost.EsimateForecastCostSpecResult" + "$ref": "#/definitions/cost.EsimateCostSpecResults" } }, "totalMaxMonthlyPrice": { @@ -1733,7 +1723,7 @@ const docTemplate = `{ } } }, - "cost.EstimateForecastCostSpecDetailResult": { + "cost.EstimateCostSpecDetailResult": { "type": "object", "properties": { "calculatedMonthlyPrice": { @@ -1780,7 +1770,7 @@ const docTemplate = `{ } } }, - "cost.GetCostInfoResult": { + "cost.GetEstimateForecastCostInfoResult": { "type": "object", "properties": { "category": { @@ -1806,7 +1796,21 @@ const docTemplate = `{ } } }, - "cost.UpdateCostInfoResult": { + "cost.GetEstimateForecastCostInfoResults": { + "type": "object", + "properties": { + "getEstimateForecastCostInfoResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.GetEstimateForecastCostInfoResult" + } + }, + "resultCount": { + "type": "integer" + } + } + }, + "cost.UpdateEstimateForecastCostInfoResult": { "type": "object", "properties": { "fetchedDataCount": { diff --git a/api/swagger.json b/api/swagger.json index c2ad570..b3fc3d6 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -8,9 +8,93 @@ }, "basePath": "/ant", "paths": { - "/api/v1/cost-estimation/forecast": { + "/api/v1/cost/estimate": { + "get": { + "description": "Fetch estimated cost details based on provider, region, instance type, and resource specifications. Pagination support is provided through `Page` and `Size` parameters.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimate]" + ], + "summary": "Retrieve Estimated Cost Information", + "operationId": "GetEstimateCost", + "parameters": [ + { + "type": "string", + "description": "Cloud provider name to filter estimated costs", + "name": "providerName", + "in": "query" + }, + { + "type": "string", + "description": "Region name to filter estimated costs", + "name": "regionName", + "in": "query" + }, + { + "type": "string", + "description": "Instance type to filter estimated costs", + "name": "instanceType", + "in": "query" + }, + { + "type": "string", + "description": "Number of vCPUs to filter estimated costs", + "name": "vCpu", + "in": "query" + }, + { + "type": "string", + "description": "Memory size to filter estimated costs", + "name": "memory", + "in": "query" + }, + { + "type": "string", + "description": "Operating system type to filter estimated costs", + "name": "osType", + "in": "query" + }, + { + "type": "integer", + "description": "Page number for pagination (default: 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Number of records per page (default: 100, max: 100)", + "name": "size", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successfully retrieved estimated cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_EstimateCostInfoResults" + } + }, + "400": { + "description": "Invalid request parameters", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Failed to retrieve estimated cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + }, "post": { - "description": "Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid.", + "description": "Update the estimate cost based on provided specifications and retrieve the updated cost estimation. Required fields for each specification include `ProviderName`, `RegionName`, and `InstanceType`. Specifications can also be provided in a formatted string using `+` delimiter.", "consumes": [ "application/json" ], @@ -18,36 +102,36 @@ "application/json" ], "tags": [ - "[Cost Estimation]" + "[Cost Estimate]" ], - "summary": "Estimate Forecast Cost", - "operationId": "EstimateForecastCost", + "summary": "Update and Retrieve Estimated Cost Information", + "operationId": "UpdateAndGetEstimateCost", "parameters": [ { - "description": "Request body containing estimation parameters", + "description": "Request body for updating and retrieving estimated cost information", "name": "body", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/app.EstimateForecastCostReq" + "$ref": "#/definitions/app.UpdateAndGetEstimateCostReq" } } ], "responses": { "200": { - "description": "Successfully estimated forecast cost", + "description": "Successfully updated and retrieved estimated cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-cost_EstimateForecastCostResult" + "$ref": "#/definitions/app.AntResponse-cost_EstimateCostResults" } }, "400": { - "description": "Invalid request parameters", + "description": "Invalid request parameters or format", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to estimate forecast cost", + "description": "Failed to update or retrieve estimated cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -55,9 +139,9 @@ } } }, - "/api/v1/cost/info": { + "/api/v1/cost/estimate/forecast": { "get": { - "description": "Retrieve cost information for specified parameters within a defined date range. The date range must be within a 6-month period. Optionally, you can specify cost aggregation type and date order for the results.", + "description": "Fetch estimated forecast cost data based on specified parameters, including a date range that must be within 6 months. Supports pagination and filtering by namespace IDs, migration configuration IDs, and resource types.", "consumes": [ "application/json" ], @@ -65,21 +149,21 @@ "application/json" ], "tags": [ - "[Cost Management]" + "[Cost Estimate]" ], - "summary": "Get Cost Information", - "operationId": "GetCostInfo", + "summary": "Retrieve Estimated Forecast Cost Information", + "operationId": "GetEstimateForecastCost", "parameters": [ { "type": "string", - "description": "Start date for the cost information retrieval in 'YYYY-MM-DD' format", + "description": "Start date for the forecast cost retrieval in 'YYYY-MM-DD' format", "name": "startDate", "in": "query", "required": true }, { "type": "string", - "description": "End date for the cost information retrieval in 'YYYY-MM-DD' format", + "description": "End date for the forecast cost retrieval in 'YYYY-MM-DD' format", "name": "endDate", "in": "query", "required": true @@ -90,8 +174,8 @@ "type": "string" }, "collectionFormat": "csv", - "description": "List of migration IDs to filter the cost information", - "name": "migrationIds", + "description": "List of namespace IDs to filter forecast cost information", + "name": "nsIds", "in": "query" }, { @@ -100,8 +184,8 @@ "type": "string" }, "collectionFormat": "csv", - "description": "List of cloud providers to filter the cost information", - "name": "provider", + "description": "List of migration configuration IDs to filter forecast cost information", + "name": "mciIds", "in": "query" }, { @@ -110,7 +194,17 @@ "type": "string" }, "collectionFormat": "csv", - "description": "List of resource types to filter the cost information", + "description": "List of cloud providers to filter forecast cost information", + "name": "providers", + "in": "query" + }, + { + "type": "array", + "items": { + "type": "string" + }, + "collectionFormat": "csv", + "description": "List of resource types to filter forecast cost information", "name": "resourceTypes", "in": "query" }, @@ -120,16 +214,15 @@ "type": "string" }, "collectionFormat": "csv", - "description": "List of resource IDs to filter the cost information", + "description": "List of resource IDs to filter forecast cost information", "name": "resourceIds", "in": "query" }, { "type": "string", - "description": "Type of cost aggregation for the results (e.g., 'daily', 'weekly', 'monthly')", + "description": "Type of cost aggregation (e.g., 'daily', 'weekly', 'monthly')", "name": "costAggregationType", - "in": "query", - "required": true + "in": "query" }, { "type": "string", @@ -142,23 +235,35 @@ "description": "Order of resource types in the result (e.g., 'asc', 'desc')", "name": "resourceTypeOrder", "in": "query" + }, + { + "type": "integer", + "description": "Page number for pagination (default: 1)", + "name": "page", + "in": "query" + }, + { + "type": "integer", + "description": "Number of records per page (default: 10000, max: 10000)", + "name": "size", + "in": "query" } ], "responses": { "200": { - "description": "Successfully retrieved cost information", + "description": "Successfully retrieved estimated forecast cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-array_cost_GetCostInfoResult" + "$ref": "#/definitions/app.AntResponse-cost_GetEstimateForecastCostInfoResults" } }, "400": { - "description": "Invalid request parameters", + "description": "Invalid request parameters or date format errors", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to retrieve cost information", + "description": "Failed to retrieve estimated forecast cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -166,7 +271,7 @@ } }, "post": { - "description": "Update cost information for specified resources, including details such as migration ID, cost resources, and additional AWS info if applicable. The request body must include a valid migration ID and a list of cost resources. If AWS-specific details are provided, ensure all required fields are populated.", + "description": "Update and retrieve forecasted cost estimates for a specified namespace and migration configuration ID over the past 14 days.", "consumes": [ "application/json" ], @@ -174,36 +279,36 @@ "application/json" ], "tags": [ - "[Cost Management]" + "[Cost Estimate]" ], - "summary": "Update Cost Information", - "operationId": "UpdateCostInfo", + "summary": "Update and Retrieve Estimated Forecast Cost", + "operationId": "UpdateEstimateForecastCost", "parameters": [ { - "description": "Request body containing cost update information", + "description": "Request body containing NsId (Namespace ID) and MciId (Migration Configuration ID) for cost estimation forecast", "name": "body", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/app.UpdateCostInfoReq" + "$ref": "#/definitions/app.UpdateEstimateForecastCostReq" } } ], "responses": { "200": { - "description": "Successfully updated cost information", + "description": "Successfully updated and retrieved estimated forecast cost information", "schema": { - "$ref": "#/definitions/app.AntResponse-cost_UpdateCostInfoResult" + "$ref": "#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult" } }, "400": { - "description": "Invalid request parameters", + "description": "Request body binding error", "schema": { "$ref": "#/definitions/app.AntResponse-string" } }, "500": { - "description": "Failed to update cost information", + "description": "Failed to update or retrieve forecast cost information", "schema": { "$ref": "#/definitions/app.AntResponse-string" } @@ -926,126 +1031,6 @@ } } }, - "/api/v1/price/info": { - "get": { - "description": "Retrieve pricing information for cloud resources based on specified query parameters. Returns price data based on provider, region, instance type, vCPU, memory, and OS type. It offer instances with the lowest monthly prices what in the database.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "Price Management" - ], - "summary": "Get Price Information", - "operationId": "GetPriceInfos", - "parameters": [ - { - "type": "string", - "description": "Cloud provider name - aws|alibaba|tencent|gcp|azure|ibm", - "name": "providerName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Region name", - "name": "regionName", - "in": "query", - "required": true - }, - { - "type": "string", - "description": "Instance type", - "name": "instanceType", - "in": "query" - }, - { - "type": "string", - "description": "Number of vCPUs", - "name": "vCpu", - "in": "query" - }, - { - "type": "string", - "description": "Amount of memory", - "name": "memory", - "in": "query" - }, - { - "type": "string", - "description": "Operating system type", - "name": "osType", - "in": "query" - } - ], - "responses": { - "200": { - "description": "Successfully retrieved pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "400": { - "description": "Invalid request parameters", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "500": { - "description": "Failed to retrieve pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - } - } - }, - "post": { - "description": "Retrieve pricing information for cloud resources based on specified parameters. If saved data is more than 7 days, fetch new data and insert new price data even if same price as before.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "[Price Management]" - ], - "operationId": "UpdatePriceInfos", - "parameters": [ - { - "description": "Request body containing get price information", - "name": "body", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/app.UpdatePriceInfosReq" - } - } - ], - "responses": { - "200": { - "description": "Successfully retrieved pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "400": { - "description": "Invalid request parameters", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - }, - "500": { - "description": "Failed to retrieve pricing information", - "schema": { - "$ref": "#/definitions/app.AntResponse-string" - } - } - } - } - }, "/readyz": { "get": { "description": "This endpoint checks if the CB-Ant API server is ready by verifying the status of both the load service and the cost service. If either service is unavailable, it returns a 503 status indicating the server is not ready.", @@ -1084,7 +1069,7 @@ } }, "definitions": { - "app.AntResponse-array_cost_GetCostInfoResult": { + "app.AntResponse-array_load_LoadTestStatistics": { "type": "object", "properties": { "code": { @@ -1096,7 +1081,7 @@ "result": { "type": "array", "items": { - "$ref": "#/definitions/cost.GetCostInfoResult" + "$ref": "#/definitions/load.LoadTestStatistics" } }, "successMessage": { @@ -1104,7 +1089,7 @@ } } }, - "app.AntResponse-array_load_LoadTestStatistics": { + "app.AntResponse-array_load_MetricsSummary": { "type": "object", "properties": { "code": { @@ -1116,7 +1101,7 @@ "result": { "type": "array", "items": { - "$ref": "#/definitions/load.LoadTestStatistics" + "$ref": "#/definitions/load.MetricsSummary" } }, "successMessage": { @@ -1124,7 +1109,7 @@ } } }, - "app.AntResponse-array_load_MetricsSummary": { + "app.AntResponse-array_load_ResultSummary": { "type": "object", "properties": { "code": { @@ -1136,7 +1121,7 @@ "result": { "type": "array", "items": { - "$ref": "#/definitions/load.MetricsSummary" + "$ref": "#/definitions/load.ResultSummary" } }, "successMessage": { @@ -1144,7 +1129,7 @@ } } }, - "app.AntResponse-array_load_ResultSummary": { + "app.AntResponse-cost_EstimateCostInfoResults": { "type": "object", "properties": { "code": { @@ -1154,17 +1139,14 @@ "type": "string" }, "result": { - "type": "array", - "items": { - "$ref": "#/definitions/load.ResultSummary" - } + "$ref": "#/definitions/cost.EstimateCostInfoResults" }, "successMessage": { "type": "string" } } }, - "app.AntResponse-cost_EstimateForecastCostResult": { + "app.AntResponse-cost_EstimateCostResults": { "type": "object", "properties": { "code": { @@ -1174,14 +1156,14 @@ "type": "string" }, "result": { - "$ref": "#/definitions/cost.EstimateForecastCostResult" + "$ref": "#/definitions/cost.EstimateCostResults" }, "successMessage": { "type": "string" } } }, - "app.AntResponse-cost_UpdateCostInfoResult": { + "app.AntResponse-cost_GetEstimateForecastCostInfoResults": { "type": "object", "properties": { "code": { @@ -1191,7 +1173,24 @@ "type": "string" }, "result": { - "$ref": "#/definitions/cost.UpdateCostInfoResult" + "$ref": "#/definitions/cost.GetEstimateForecastCostInfoResults" + }, + "successMessage": { + "type": "string" + } + } + }, + "app.AntResponse-cost_UpdateEstimateForecastCostInfoResult": { + "type": "object", + "properties": { + "code": { + "type": "integer" + }, + "errorMessage": { + "type": "string" + }, + "result": { + "$ref": "#/definitions/cost.UpdateEstimateForecastCostInfoResult" }, "successMessage": { "type": "string" @@ -1368,85 +1367,6 @@ } } }, - "app.AwsAdditionalInfoReq": { - "type": "object", - "properties": { - "ownerId": { - "type": "string" - }, - "regions": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "app.CostResourceReq": { - "type": "object", - "properties": { - "resourceIds": { - "type": "array", - "items": { - "type": "string" - } - }, - "resourceType": { - "$ref": "#/definitions/constant.ResourceType" - } - } - }, - "app.EstimateForecastCostReq": { - "type": "object", - "required": [ - "specs", - "specsWithFormat" - ], - "properties": { - "specs": { - "type": "array", - "items": { - "type": "object", - "required": [ - "instanceType", - "providerName", - "regionName" - ], - "properties": { - "image": { - "type": "string" - }, - "instanceType": { - "type": "string" - }, - "providerName": { - "type": "string" - }, - "regionName": { - "type": "string" - } - } - } - }, - "specsWithFormat": { - "type": "array", - "items": { - "type": "object", - "required": [ - "commonSpec" - ], - "properties": { - "commonImage": { - "type": "string" - }, - "commonSpec": { - "type": "string" - } - } - } - } - } - }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1550,45 +1470,60 @@ } } }, - "app.UpdateCostInfoReq": { + "app.UpdateAndGetEstimateCostReq": { "type": "object", - "required": [ - "connectionName", - "costResources" - ], "properties": { - "awsAdditionalInfo": { - "$ref": "#/definitions/app.AwsAdditionalInfoReq" - }, - "connectionName": { - "type": "string" - }, - "costResources": { + "specs": { "type": "array", "items": { - "$ref": "#/definitions/app.CostResourceReq" + "type": "object", + "required": [ + "instanceType", + "providerName", + "regionName" + ], + "properties": { + "image": { + "type": "string" + }, + "instanceType": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + } + } } }, - "migrationId": { - "type": "string" + "specsWithFormat": { + "type": "array", + "items": { + "type": "object", + "required": [ + "commonSpec" + ], + "properties": { + "commonImage": { + "type": "string" + }, + "commonSpec": { + "type": "string" + } + } + } } } }, - "app.UpdatePriceInfosReq": { + "app.UpdateEstimateForecastCostReq": { "type": "object", - "required": [ - "instanceType", - "providerName", - "regionName" - ], "properties": { - "instanceType": { - "type": "string" - }, - "providerName": { + "mciId": { "type": "string" }, - "regionName": { + "nsId": { "type": "string" } } @@ -1664,28 +1599,13 @@ "PerYear" ] }, - "constant.ResourceType": { - "type": "string", - "enum": [ - "VM", - "VNet", - "DataDisk", - "Etc" - ], - "x-enum-varnames": [ - "VM", - "VNet", - "DataDisk", - "Etc" - ] - }, - "cost.EsimateForecastCostSpecResult": { + "cost.EsimateCostSpecResults": { "type": "object", "properties": { "estimateForecastCostSpecDetailResults": { "type": "array", "items": { - "$ref": "#/definitions/cost.EstimateForecastCostSpecDetailResult" + "$ref": "#/definitions/cost.EstimateCostSpecDetailResult" } }, "imageName": { @@ -1708,13 +1628,83 @@ } } }, - "cost.EstimateForecastCostResult": { + "cost.EstimateCostInfoResult": { + "type": "object", + "properties": { + "calculatedMonthlyPrice": { + "type": "number" + }, + "currency": { + "$ref": "#/definitions/constant.PriceCurrency" + }, + "id": { + "type": "integer" + }, + "instanceType": { + "type": "string" + }, + "lastUpdatedAt": { + "type": "string" + }, + "memory": { + "type": "string" + }, + "originalPricePolicy": { + "type": "string" + }, + "osType": { + "type": "string" + }, + "price": { + "type": "string" + }, + "priceDescription": { + "type": "string" + }, + "pricePolicy": { + "$ref": "#/definitions/constant.PricePolicy" + }, + "productDescription": { + "type": "string" + }, + "providerName": { + "type": "string" + }, + "regionName": { + "type": "string" + }, + "storage": { + "type": "string" + }, + "unit": { + "$ref": "#/definitions/constant.PriceUnit" + }, + "vCpu": { + "type": "string" + } + } + }, + "cost.EstimateCostInfoResults": { + "type": "object", + "properties": { + "estimateCostInfoResult": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.EstimateCostInfoResult" + } + }, + "resultCount": { + "type": "integer" + } + } + }, + "cost.EstimateCostResults": { "type": "object", "properties": { "esimateForecastCostSpecResults": { "type": "array", "items": { - "$ref": "#/definitions/cost.EsimateForecastCostSpecResult" + "$ref": "#/definitions/cost.EsimateCostSpecResults" } }, "totalMaxMonthlyPrice": { @@ -1725,7 +1715,7 @@ } } }, - "cost.EstimateForecastCostSpecDetailResult": { + "cost.EstimateCostSpecDetailResult": { "type": "object", "properties": { "calculatedMonthlyPrice": { @@ -1772,7 +1762,7 @@ } } }, - "cost.GetCostInfoResult": { + "cost.GetEstimateForecastCostInfoResult": { "type": "object", "properties": { "category": { @@ -1798,7 +1788,21 @@ } } }, - "cost.UpdateCostInfoResult": { + "cost.GetEstimateForecastCostInfoResults": { + "type": "object", + "properties": { + "getEstimateForecastCostInfoResults": { + "type": "array", + "items": { + "$ref": "#/definitions/cost.GetEstimateForecastCostInfoResult" + } + }, + "resultCount": { + "type": "integer" + } + } + }, + "cost.UpdateEstimateForecastCostInfoResult": { "type": "object", "properties": { "fetchedDataCount": { diff --git a/api/swagger.yaml b/api/swagger.yaml index e1b4544..09c4003 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -1,6 +1,6 @@ basePath: /ant definitions: - app.AntResponse-array_cost_GetCostInfoResult: + app.AntResponse-array_load_LoadTestStatistics: properties: code: type: integer @@ -8,12 +8,12 @@ definitions: type: string result: items: - $ref: '#/definitions/cost.GetCostInfoResult' + $ref: '#/definitions/load.LoadTestStatistics' type: array successMessage: type: string type: object - app.AntResponse-array_load_LoadTestStatistics: + app.AntResponse-array_load_MetricsSummary: properties: code: type: integer @@ -21,12 +21,12 @@ definitions: type: string result: items: - $ref: '#/definitions/load.LoadTestStatistics' + $ref: '#/definitions/load.MetricsSummary' type: array successMessage: type: string type: object - app.AntResponse-array_load_MetricsSummary: + app.AntResponse-array_load_ResultSummary: properties: code: type: integer @@ -34,43 +34,52 @@ definitions: type: string result: items: - $ref: '#/definitions/load.MetricsSummary' + $ref: '#/definitions/load.ResultSummary' type: array successMessage: type: string type: object - app.AntResponse-array_load_ResultSummary: + app.AntResponse-cost_EstimateCostInfoResults: properties: code: type: integer errorMessage: type: string result: - items: - $ref: '#/definitions/load.ResultSummary' - type: array + $ref: '#/definitions/cost.EstimateCostInfoResults' successMessage: type: string type: object - app.AntResponse-cost_EstimateForecastCostResult: + app.AntResponse-cost_EstimateCostResults: properties: code: type: integer errorMessage: type: string result: - $ref: '#/definitions/cost.EstimateForecastCostResult' + $ref: '#/definitions/cost.EstimateCostResults' successMessage: type: string type: object - app.AntResponse-cost_UpdateCostInfoResult: + app.AntResponse-cost_GetEstimateForecastCostInfoResults: properties: code: type: integer errorMessage: type: string result: - $ref: '#/definitions/cost.UpdateCostInfoResult' + $ref: '#/definitions/cost.GetEstimateForecastCostInfoResults' + successMessage: + type: string + type: object + app.AntResponse-cost_UpdateEstimateForecastCostInfoResult: + properties: + code: + type: integer + errorMessage: + type: string + result: + $ref: '#/definitions/cost.UpdateEstimateForecastCostInfoResult' successMessage: type: string type: object @@ -184,58 +193,6 @@ definitions: successMessage: type: string type: object - app.AwsAdditionalInfoReq: - properties: - ownerId: - type: string - regions: - items: - type: string - type: array - type: object - app.CostResourceReq: - properties: - resourceIds: - items: - type: string - type: array - resourceType: - $ref: '#/definitions/constant.ResourceType' - type: object - app.EstimateForecastCostReq: - properties: - specs: - items: - properties: - image: - type: string - instanceType: - type: string - providerName: - type: string - regionName: - type: string - required: - - instanceType - - providerName - - regionName - type: object - type: array - specsWithFormat: - items: - properties: - commonImage: - type: string - commonSpec: - type: string - required: - - commonSpec - type: object - type: array - required: - - specs - - specsWithFormat - type: object app.InstallLoadGeneratorReq: properties: installLocation: @@ -303,34 +260,43 @@ definitions: loadTestKey: type: string type: object - app.UpdateCostInfoReq: + app.UpdateAndGetEstimateCostReq: properties: - awsAdditionalInfo: - $ref: '#/definitions/app.AwsAdditionalInfoReq' - connectionName: - type: string - costResources: + specs: items: - $ref: '#/definitions/app.CostResourceReq' + properties: + image: + type: string + instanceType: + type: string + providerName: + type: string + regionName: + type: string + required: + - instanceType + - providerName + - regionName + type: object + type: array + specsWithFormat: + items: + properties: + commonImage: + type: string + commonSpec: + type: string + required: + - commonSpec + type: object type: array - migrationId: - type: string - required: - - connectionName - - costResources type: object - app.UpdatePriceInfosReq: + app.UpdateEstimateForecastCostReq: properties: - instanceType: - type: string - providerName: + mciId: type: string - regionName: + nsId: type: string - required: - - instanceType - - providerName - - regionName type: object constant.ExecutionStatus: enum: @@ -388,23 +354,11 @@ definitions: x-enum-varnames: - PerHour - PerYear - constant.ResourceType: - enum: - - VM - - VNet - - DataDisk - - Etc - type: string - x-enum-varnames: - - VM - - VNet - - DataDisk - - Etc - cost.EsimateForecastCostSpecResult: + cost.EsimateCostSpecResults: properties: estimateForecastCostSpecDetailResults: items: - $ref: '#/definitions/cost.EstimateForecastCostSpecDetailResult' + $ref: '#/definitions/cost.EstimateCostSpecDetailResult' type: array imageName: type: string @@ -419,18 +373,64 @@ definitions: totalMinMonthlyPrice: type: number type: object - cost.EstimateForecastCostResult: + cost.EstimateCostInfoResult: + properties: + calculatedMonthlyPrice: + type: number + currency: + $ref: '#/definitions/constant.PriceCurrency' + id: + type: integer + instanceType: + type: string + lastUpdatedAt: + type: string + memory: + type: string + originalPricePolicy: + type: string + osType: + type: string + price: + type: string + priceDescription: + type: string + pricePolicy: + $ref: '#/definitions/constant.PricePolicy' + productDescription: + type: string + providerName: + type: string + regionName: + type: string + storage: + type: string + unit: + $ref: '#/definitions/constant.PriceUnit' + vCpu: + type: string + type: object + cost.EstimateCostInfoResults: + properties: + estimateCostInfoResult: + items: + $ref: '#/definitions/cost.EstimateCostInfoResult' + type: array + resultCount: + type: integer + type: object + cost.EstimateCostResults: properties: esimateForecastCostSpecResults: items: - $ref: '#/definitions/cost.EsimateForecastCostSpecResult' + $ref: '#/definitions/cost.EsimateCostSpecResults' type: array totalMaxMonthlyPrice: type: number totalMinMonthlyPrice: type: number type: object - cost.EstimateForecastCostSpecDetailResult: + cost.EstimateCostSpecDetailResult: properties: calculatedMonthlyPrice: type: number @@ -461,7 +461,7 @@ definitions: vCpu: type: string type: object - cost.GetCostInfoResult: + cost.GetEstimateForecastCostInfoResult: properties: category: type: string @@ -478,7 +478,16 @@ definitions: unit: type: string type: object - cost.UpdateCostInfoResult: + cost.GetEstimateForecastCostInfoResults: + properties: + getEstimateForecastCostInfoResults: + items: + $ref: '#/definitions/cost.GetEstimateForecastCostInfoResult' + type: array + resultCount: + type: integer + type: object + cost.UpdateEstimateForecastCostInfoResult: properties: fetchedDataCount: type: integer @@ -788,93 +797,155 @@ info: title: CM-ANT REST API version: 0.2.2 paths: - /api/v1/cost-estimation/forecast: + /api/v1/cost/estimate: + get: + consumes: + - application/json + description: Fetch estimated cost details based on provider, region, instance + type, and resource specifications. Pagination support is provided through + `Page` and `Size` parameters. + operationId: GetEstimateCost + parameters: + - description: Cloud provider name to filter estimated costs + in: query + name: providerName + type: string + - description: Region name to filter estimated costs + in: query + name: regionName + type: string + - description: Instance type to filter estimated costs + in: query + name: instanceType + type: string + - description: Number of vCPUs to filter estimated costs + in: query + name: vCpu + type: string + - description: Memory size to filter estimated costs + in: query + name: memory + type: string + - description: Operating system type to filter estimated costs + in: query + name: osType + type: string + - description: 'Page number for pagination (default: 1)' + in: query + name: page + type: integer + - description: 'Number of records per page (default: 100, max: 100)' + in: query + name: size + type: integer + produces: + - application/json + responses: + "200": + description: Successfully retrieved estimated cost information + schema: + $ref: '#/definitions/app.AntResponse-cost_EstimateCostInfoResults' + "400": + description: Invalid request parameters + schema: + $ref: '#/definitions/app.AntResponse-string' + "500": + description: Failed to retrieve estimated cost information + schema: + $ref: '#/definitions/app.AntResponse-string' + summary: Retrieve Estimated Cost Information + tags: + - '[Cost Estimate]' post: consumes: - application/json - description: Estimate the forecast cost for cloud resources based on recommended - specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat - in the request body. Returns an error if the required properties are missing - or if the request is invalid. - operationId: EstimateForecastCost + description: Update the estimate cost based on provided specifications and retrieve + the updated cost estimation. Required fields for each specification include + `ProviderName`, `RegionName`, and `InstanceType`. Specifications can also + be provided in a formatted string using `+` delimiter. + operationId: UpdateAndGetEstimateCost parameters: - - description: Request body containing estimation parameters + - description: Request body for updating and retrieving estimated cost information in: body name: body required: true schema: - $ref: '#/definitions/app.EstimateForecastCostReq' + $ref: '#/definitions/app.UpdateAndGetEstimateCostReq' produces: - application/json responses: "200": - description: Successfully estimated forecast cost + description: Successfully updated and retrieved estimated cost information schema: - $ref: '#/definitions/app.AntResponse-cost_EstimateForecastCostResult' + $ref: '#/definitions/app.AntResponse-cost_EstimateCostResults' "400": - description: Invalid request parameters + description: Invalid request parameters or format schema: $ref: '#/definitions/app.AntResponse-string' "500": - description: Failed to estimate forecast cost + description: Failed to update or retrieve estimated cost information schema: $ref: '#/definitions/app.AntResponse-string' - summary: Estimate Forecast Cost + summary: Update and Retrieve Estimated Cost Information tags: - - '[Cost Estimation]' - /api/v1/cost/info: + - '[Cost Estimate]' + /api/v1/cost/estimate/forecast: get: consumes: - application/json - description: Retrieve cost information for specified parameters within a defined - date range. The date range must be within a 6-month period. Optionally, you - can specify cost aggregation type and date order for the results. - operationId: GetCostInfo + description: Fetch estimated forecast cost data based on specified parameters, + including a date range that must be within 6 months. Supports pagination and + filtering by namespace IDs, migration configuration IDs, and resource types. + operationId: GetEstimateForecastCost parameters: - - description: Start date for the cost information retrieval in 'YYYY-MM-DD' - format + - description: Start date for the forecast cost retrieval in 'YYYY-MM-DD' format in: query name: startDate required: true type: string - - description: End date for the cost information retrieval in 'YYYY-MM-DD' format + - description: End date for the forecast cost retrieval in 'YYYY-MM-DD' format in: query name: endDate required: true type: string - collectionFormat: csv - description: List of migration IDs to filter the cost information + description: List of namespace IDs to filter forecast cost information in: query items: type: string - name: migrationIds + name: nsIds type: array - collectionFormat: csv - description: List of cloud providers to filter the cost information + description: List of migration configuration IDs to filter forecast cost information in: query items: type: string - name: provider + name: mciIds type: array - collectionFormat: csv - description: List of resource types to filter the cost information + description: List of cloud providers to filter forecast cost information + in: query + items: + type: string + name: providers + type: array + - collectionFormat: csv + description: List of resource types to filter forecast cost information in: query items: type: string name: resourceTypes type: array - collectionFormat: csv - description: List of resource IDs to filter the cost information + description: List of resource IDs to filter forecast cost information in: query items: type: string name: resourceIds type: array - - description: Type of cost aggregation for the results (e.g., 'daily', 'weekly', - 'monthly') + - description: Type of cost aggregation (e.g., 'daily', 'weekly', 'monthly') in: query name: costAggregationType - required: true type: string - description: Order of dates in the result (e.g., 'asc', 'desc') in: query @@ -884,57 +955,65 @@ paths: in: query name: resourceTypeOrder type: string + - description: 'Page number for pagination (default: 1)' + in: query + name: page + type: integer + - description: 'Number of records per page (default: 10000, max: 10000)' + in: query + name: size + type: integer produces: - application/json responses: "200": - description: Successfully retrieved cost information + description: Successfully retrieved estimated forecast cost information schema: - $ref: '#/definitions/app.AntResponse-array_cost_GetCostInfoResult' + $ref: '#/definitions/app.AntResponse-cost_GetEstimateForecastCostInfoResults' "400": - description: Invalid request parameters + description: Invalid request parameters or date format errors schema: $ref: '#/definitions/app.AntResponse-string' "500": - description: Failed to retrieve cost information + description: Failed to retrieve estimated forecast cost information schema: $ref: '#/definitions/app.AntResponse-string' - summary: Get Cost Information + summary: Retrieve Estimated Forecast Cost Information tags: - - '[Cost Management]' + - '[Cost Estimate]' post: consumes: - application/json - description: Update cost information for specified resources, including details - such as migration ID, cost resources, and additional AWS info if applicable. - The request body must include a valid migration ID and a list of cost resources. - If AWS-specific details are provided, ensure all required fields are populated. - operationId: UpdateCostInfo + description: Update and retrieve forecasted cost estimates for a specified namespace + and migration configuration ID over the past 14 days. + operationId: UpdateEstimateForecastCost parameters: - - description: Request body containing cost update information + - description: Request body containing NsId (Namespace ID) and MciId (Migration + Configuration ID) for cost estimation forecast in: body name: body required: true schema: - $ref: '#/definitions/app.UpdateCostInfoReq' + $ref: '#/definitions/app.UpdateEstimateForecastCostReq' produces: - application/json responses: "200": - description: Successfully updated cost information + description: Successfully updated and retrieved estimated forecast cost + information schema: - $ref: '#/definitions/app.AntResponse-cost_UpdateCostInfoResult' + $ref: '#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult' "400": - description: Invalid request parameters + description: Request body binding error schema: $ref: '#/definitions/app.AntResponse-string' "500": - description: Failed to update cost information + description: Failed to update or retrieve forecast cost information schema: $ref: '#/definitions/app.AntResponse-string' - summary: Update Cost Information + summary: Update and Retrieve Estimated Forecast Cost tags: - - '[Cost Management]' + - '[Cost Estimate]' /api/v1/load/generators: get: consumes: @@ -1412,91 +1491,6 @@ paths: summary: Stop Load Test tags: - '[Load Test Execution Management]' - /api/v1/price/info: - get: - consumes: - - application/json - description: Retrieve pricing information for cloud resources based on specified - query parameters. Returns price data based on provider, region, instance type, - vCPU, memory, and OS type. It offer instances with the lowest monthly prices - what in the database. - operationId: GetPriceInfos - parameters: - - description: Cloud provider name - aws|alibaba|tencent|gcp|azure|ibm - in: query - name: providerName - required: true - type: string - - description: Region name - in: query - name: regionName - required: true - type: string - - description: Instance type - in: query - name: instanceType - type: string - - description: Number of vCPUs - in: query - name: vCpu - type: string - - description: Amount of memory - in: query - name: memory - type: string - - description: Operating system type - in: query - name: osType - type: string - produces: - - application/json - responses: - "200": - description: Successfully retrieved pricing information - schema: - $ref: '#/definitions/app.AntResponse-string' - "400": - description: Invalid request parameters - schema: - $ref: '#/definitions/app.AntResponse-string' - "500": - description: Failed to retrieve pricing information - schema: - $ref: '#/definitions/app.AntResponse-string' - summary: Get Price Information - tags: - - Price Management - post: - consumes: - - application/json - description: Retrieve pricing information for cloud resources based on specified - parameters. If saved data is more than 7 days, fetch new data and insert new - price data even if same price as before. - operationId: UpdatePriceInfos - parameters: - - description: Request body containing get price information - in: body - name: body - required: true - schema: - $ref: '#/definitions/app.UpdatePriceInfosReq' - produces: - - application/json - responses: - "200": - description: Successfully retrieved pricing information - schema: - $ref: '#/definitions/app.AntResponse-string' - "400": - description: Invalid request parameters - schema: - $ref: '#/definitions/app.AntResponse-string' - "500": - description: Failed to retrieve pricing information - schema: - $ref: '#/definitions/app.AntResponse-string' - tags: - - '[Price Management]' /readyz: get: consumes: diff --git a/cmd/cm-ant/main.go b/cmd/cm-ant/main.go index b6b4fd2..9814d3b 100644 --- a/cmd/cm-ant/main.go +++ b/cmd/cm-ant/main.go @@ -1,14 +1,20 @@ package main import ( - "log" + "fmt" + "os" "os/signal" + "runtime" + "strings" "syscall" + "time" "github.com/cloud-barista/cm-ant/internal/app" "github.com/cloud-barista/cm-ant/internal/config" "github.com/cloud-barista/cm-ant/internal/utils" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) // InitRouter initializes the routing for CM-ANT API server. @@ -17,55 +23,99 @@ import ( // @version 0.2.2 // @description CM-ANT REST API swagger document. // @basePath /ant + +type CallerHook struct{} + +func (h CallerHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { + if pc, file, line, ok := runtime.Caller(3); ok { + shortFile := file[strings.LastIndex(file, "/")+1:] + e.Str("file", fmt.Sprintf("%s:%d", shortFile, line)) + funcName := strings.Replace(runtime.FuncForPC(pc).Name(), "github.com/cloud-barista/", "", 1) + e.Str("func", funcName) + } +} + func main() { + output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} + output.FormatLevel = func(i interface{}) string { + level := strings.ToUpper(fmt.Sprintf("%s", i)) + switch level { + case "DEBUG": + return fmt.Sprintf("\033[36m| %-6s|\033[0m", level) // Cyan + case "INFO": + return fmt.Sprintf("\033[32m| %-6s|\033[0m", level) // Green + case "WARN": + return fmt.Sprintf("\033[33m| %-6s|\033[0m", level) // Yellow + case "ERROR": + return fmt.Sprintf("\033[31m| %-6s|\033[0m", level) // Red + case "FATAL": + return fmt.Sprintf("\033[35m| %-6s|\033[0m", level) // Magenta + default: + return fmt.Sprintf("| %-6s|", level) // Default color + } + } + output.FormatMessage = func(i interface{}) string { + if i == nil { + return "" + } + return fmt.Sprintf("message: \033[1m%s\033[0m", i) + } + + output.FormatFieldName = func(i interface{}) string { + return fmt.Sprintf("%s:", i) + } + output.FormatFieldValue = func(i interface{}) string { + return fmt.Sprintf("\033[1m%s\033[0m", i) + } + + log.Logger = zerolog.New(output).With().Timestamp().Logger().Hook(CallerHook{}) err := utils.Script(utils.JoinRootPathWith("/script/install_required_utils.sh"), []string{}) if err != nil { - log.Fatal("required tool can not install") + log.Fatal().Msg("required tool can not install") } - utils.LogInfo("Starting CM-Ant server initialization...") // Initialize the configuration for CM-Ant server err = config.InitConfig() if err != nil { - log.Fatalf("[ERROR] CM-Ant server config error: %v", err) + log.Fatal().Msgf("CM-Ant server config error: %v", err) } // Create a new instance of the CM-Ant server s, err := app.NewAntServer() if err != nil { - log.Fatalf("[ERROR] CM-Ant server creation error: %v", err) + log.Fatal().Msgf("CM-Ant server creation error: %v", err) } // Initialize the router for the CM-Ant server err = s.InitRouter() if err != nil { - log.Fatalf("[ERROR] CM-Ant server init router error: %v", err) + log.Fatal().Msgf("CM-Ant server init router error: %v", err) } - utils.LogInfo("CM-Ant server initialization completed successfully.") + log.Info().Msgf("CM-Ant server initialization completed successfully.") // Create a channel to listen for OS signals stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt, syscall.SIGTERM) - utils.LogInfo("Starting the CM-Ant server...") + log.Info().Msgf("Starting the CM-Ant server...") go func() { if err := s.Start(); err != nil { - log.Fatalf("[ERROR] CM-Ant start server error: %v", err) + log.Fatal().Msgf("CM-Ant start server error: %v", err) } }() - utils.LogInfo("CM-Ant server started successfully. Waiting for termination signal...") + log.Info().Msgf("CM-Ant server started successfully. Waiting for termination signal...") // Wait for termination signal <-stop - utils.LogInfo("Shutting down CM-Ant server...") + log.Info().Msgf("Shutting down CM-Ant server...") // Perform any necessary cleanup actions here, such as closing connections or saving state. // Optionally wait for pending operations to complete gracefully. - utils.LogInfo("CM-Ant server stopped gracefully.") + log.Info().Msgf("CM-Ant server stopped gracefully.") os.Exit(0) } diff --git a/config.yaml b/config.yaml index 0a9f4db..0285095 100644 --- a/config.yaml +++ b/config.yaml @@ -15,8 +15,7 @@ tumblebug: cost: estimation: - forecast: - priceUpdateInterval: 7d + updateInterval: "168h" load: retry: 2 @@ -24,7 +23,7 @@ load: dir: "/opt/ant/jmeter" version: 5.6 -logging: +log: level: info database: diff --git a/docker-compose.yaml b/docker-compose.yaml index 0f13a7f..beae8cb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -32,6 +32,7 @@ services: restart: unless-stopped ant-postgres: + container_name: ant-postgres image: timescale/timescaledb:latest-pg16 ports: - "5432:5432" @@ -50,7 +51,7 @@ services: restart: unless-stopped cb-tumblebug: - image: cloudbaristaorg/cb-tumblebug:0.9.13 + image: cloudbaristaorg/cb-tumblebug:0.9.21 container_name: cb-tumblebug platform: linux/amd64 ports: @@ -147,7 +148,7 @@ services: restart: unless-stopped cb-spider: - image: cloudbaristaorg/cb-spider:0.9.4 + image: cloudbaristaorg/cb-spider:0.9.8 container_name: cb-spider platform: linux/amd64 networks: diff --git a/internal/app/cost_estimation_handler.go b/internal/app/cost_estimation_handler.go deleted file mode 100644 index da94e1e..0000000 --- a/internal/app/cost_estimation_handler.go +++ /dev/null @@ -1,326 +0,0 @@ -package app - -import ( - "fmt" - "net/http" - "strings" - "time" - - "github.com/cloud-barista/cm-ant/internal/config" - "github.com/cloud-barista/cm-ant/internal/core/common/constant" - "github.com/cloud-barista/cm-ant/internal/core/cost" - "github.com/cloud-barista/cm-ant/internal/utils" - "github.com/labstack/echo/v4" -) - -// @Id EstimateForecastCost -// @Summary Estimate Forecast Cost -// @Description Estimate the forecast cost for cloud resources based on recommended specifications. Requires either RecommendSpecs or RecommendSpecsWithFormat in the request body. Returns an error if the required properties are missing or if the request is invalid. -// @Tags [Cost Estimation] -// @Accept json -// @Produce json -// @Param body body EstimateForecastCostReq true "Request body containing estimation parameters" -// @Success 200 {object} app.AntResponse[cost.EstimateForecastCostResult] "Successfully estimated forecast cost" -// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" -// @Failure 500 {object} app.AntResponse[string] "Failed to estimate forecast cost" -// @Router /api/v1/cost-estimation/forecast [post] -func (a *AntServer) estimateForecastCost(c echo.Context) error { - var req EstimateForecastCostReq - if err := c.Bind(&req); err != nil { - return errorResponseJson(http.StatusBadRequest, err.Error()) - } - - if len(req.Specs) == 0 && len(req.SpecsWithFormat) == 0 { - return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") - } - - pastTime := time.Now().Add(-config.AppConfig.Cost.Estimation.Forcast.PriceUpdateInterval) - - recommendSpecs := make([]cost.RecommendSpecParam, 0) - - if len(req.Specs) > 0 { - for _, v := range req.Specs { - param := cost.RecommendSpecParam{ - ProviderName: strings.TrimSpace(strings.ToLower(v.ProviderName)), - RegionName: strings.TrimSpace(v.RegionName), - InstanceType: strings.TrimSpace(v.InstanceType), - Image: strings.TrimSpace(v.Image), - } - recommendSpecs = append(recommendSpecs, param) - } - } - - if len(req.SpecsWithFormat) > 0 { - delim := "+" - - for _, v := range req.SpecsWithFormat { - - cs := strings.TrimSpace(v.CommonSpec) - ci := strings.TrimSpace(v.CommonImage) - - splitedCommonSpec := strings.Split(cs, delim) - splitedCommonImage := strings.Split(ci, delim) - - if len(splitedCommonSpec) != 3 { - utils.LogErrorf("common spec format is not correct; image: %s; spec: %s", ci, cs) - return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common spec format is not correct; image: %s; spec: %s", ci, cs)) - } - - if len(splitedCommonImage) == 3 && (splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1]) { - utils.LogErrorf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs) - return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs)) - } - - param := cost.RecommendSpecParam{ - ProviderName: strings.TrimSpace(strings.ToLower(splitedCommonSpec[0])), - RegionName: strings.TrimSpace(splitedCommonSpec[1]), - InstanceType: strings.TrimSpace(splitedCommonSpec[2]), - } - - if len(splitedCommonImage) == 3 { - param.Image = strings.TrimSpace(splitedCommonImage[2]) - } - - recommendSpecs = append(recommendSpecs, param) - } - } - - arg := cost.EstimateForecastCostParam{ - RecommendSpecs: recommendSpecs, - TimeStandard: time.Date(pastTime.Year(), pastTime.Month(), pastTime.Day(), 0, 0, 0, 0, pastTime.Location()), - PricePolicy: constant.OnDemand, - } - - res, err := a.services.costService.EstimateForecastCost(arg) - - if err != nil { - return errorResponseJson(http.StatusInternalServerError, err.Error()) - } - - return successResponseJson( - c, - "retrieved pricing information", - res, - ) -} - -// @Id UpdatePriceInfos -// @Summar Update Price Information -// @Description Retrieve pricing information for cloud resources based on specified parameters. If saved data is more than 7 days, fetch new data and insert new price data even if same price as before. -// @Tags [Price Management] -// @Accept json -// @Produce json -// @Param body body app.UpdatePriceInfosReq true "Request body containing get price information" -// @Success 200 {object} app.AntResponse[string] "Successfully retrieved pricing information" -// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" -// @Failure 500 {object} app.AntResponse[string] "Failed to retrieve pricing information" -// @Router /api/v1/price/info [post] -func (server *AntServer) updatePriceInfos(c echo.Context) error { - var req UpdatePriceInfosReq - if err := c.Bind(&req); err != nil { - return errorResponseJson(http.StatusBadRequest, err.Error()) - } - - if strings.TrimSpace(req.RegionName) == "" || - // strings.TrimSpace(req.InstanceType) == "" || - strings.TrimSpace(req.ProviderName) == "" { - return errorResponseJson(http.StatusBadRequest, "provier name, region name, instance type must be set") - } - - arg := cost.UpdatePriceInfosParam{ - ProviderName: strings.TrimSpace(req.ProviderName), - RegionName: strings.TrimSpace(req.RegionName), - InstanceType: strings.TrimSpace(req.InstanceType), - } - - err := server.services.costService.UpdatePriceInfos(arg) - - if err != nil { - return errorResponseJson(http.StatusInternalServerError, err.Error()) - } - - return successResponseJson( - c, - "retrieved pricing information", - fmt.Sprintf("%s csp price information updated. region: %s, instance type: %s", arg.ProviderName, arg.RegionName, arg.InstanceType), - ) -} - -// @Id GetPriceInfos -// @Summary Get Price Information -// @Description Retrieve pricing information for cloud resources based on specified query parameters. Returns price data based on provider, region, instance type, vCPU, memory, and OS type. It offer instances with the lowest monthly prices what in the database. -// @Tags Price Management -// @Accept json -// @Produce json -// @Param providerName query string true "Cloud provider name - aws|alibaba|tencent|gcp|azure|ibm" -// @Param regionName query string true "Region name" -// @Param instanceType query string false "Instance type" -// @Param vCpu query string false "Number of vCPUs" -// @Param memory query string false "Amount of memory" -// @Param osType query string false "Operating system type" -// @Success 200 {object} AntResponse[string] "Successfully retrieved pricing information" -// @Failure 400 {object} AntResponse[string] "Invalid request parameters" -// @Failure 500 {object} AntResponse[string] "Failed to retrieve pricing information" -// @Router /api/v1/price/info [get] -func (server *AntServer) getPriceInfos(c echo.Context) error { - var req GetPriceInfosReq - if err := c.Bind(&req); err != nil { - return errorResponseJson(http.StatusBadRequest, err.Error()) - } - - arg := cost.GetPriceInfosParam{ - ProviderName: strings.TrimSpace(req.ProviderName), - RegionName: strings.TrimSpace(req.RegionName), - InstanceType: strings.TrimSpace(req.InstanceType), - VCpu: strings.TrimSpace(req.VCpu), - Memory: strings.TrimSpace(req.Memory), - OsType: strings.TrimSpace(req.OsType), - } - - r, err := server.services.costService.GetPriceInfos(arg) - - if err != nil { - return errorResponseJson(http.StatusInternalServerError, err.Error()) - } - - return successResponseJson( - c, - "Successfully get price info.", - r, - ) -} - -// @Id UpdateCostInfo -// @Summary Update Cost Information -// @Description Update cost information for specified resources, including details such as migration ID, cost resources, and additional AWS info if applicable. The request body must include a valid migration ID and a list of cost resources. If AWS-specific details are provided, ensure all required fields are populated. -// @Tags [Cost Management] -// @Accept json -// @Produce json -// @Param body body app.UpdateCostInfoReq true "Request body containing cost update information" -// @Success 200 {object} app.AntResponse[cost.UpdateCostInfoResult] "Successfully updated cost information" -// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" -// @Failure 500 {object} app.AntResponse[string] "Failed to update cost information" -// @Router /api/v1/cost/info [post] -func (server *AntServer) updateCostInfos(c echo.Context) error { - var req UpdateCostInfoReq - - if err := c.Bind(&req); err != nil { - return errorResponseJson(http.StatusBadRequest, "request body binding error") - } - - if strings.TrimSpace(req.MigrationId) == "" { - return errorResponseJson(http.StatusBadRequest, "migration id is required") - } - - if len(req.CostResources) == 0 { - return errorResponseJson(http.StatusBadRequest, "Migrated resource id list are required") - } - - costResources := make([]cost.CostResourceParam, 0) - - for _, v := range req.CostResources { - costResources = append(costResources, cost.CostResourceParam{ - ResourceType: v.ResourceType, - ResourceIds: v.ResourceIds, - }) - } - - endDate := time.Now().Truncate(24*time.Hour).AddDate(0, 0, 1) - startDate := endDate.AddDate(0, 0, -14) - param := cost.UpdateCostInfoParam{ - MigrationId: req.MigrationId, - Provider: "aws", - StartDate: startDate, - EndDate: endDate, - CostResources: costResources, - ConnectionName: req.ConnectionName, - AwsAdditionalInfo: cost.AwsAdditionalInfoParam{ - OwnerId: req.AwsAdditionalInfo.OwnerId, - Regions: req.AwsAdditionalInfo.Regions, - }, - } - - r, err := server.services.costService.UpdateCostInfo(param) - - if err != nil { - return errorResponseJson(http.StatusInternalServerError, err.Error()) - } - - return successResponseJson( - c, - "Successfully updated cost info.", - r, - ) -} - -// @Id GetCostInfo -// @Summary Get Cost Information -// @Description Retrieve cost information for specified parameters within a defined date range. The date range must be within a 6-month period. Optionally, you can specify cost aggregation type and date order for the results. -// @Tags [Cost Management] -// @Accept json -// @Produce json -// @Param startDate query string true "Start date for the cost information retrieval in 'YYYY-MM-DD' format" -// @Param endDate query string true "End date for the cost information retrieval in 'YYYY-MM-DD' format" -// @Param migrationIds query []string false "List of migration IDs to filter the cost information" -// @Param provider query []string false "List of cloud providers to filter the cost information" -// @Param resourceTypes query []string false "List of resource types to filter the cost information" -// @Param resourceIds query []string false "List of resource IDs to filter the cost information" -// @Param costAggregationType query string true "Type of cost aggregation for the results (e.g., 'daily', 'weekly', 'monthly')" -// @Param dateOrder query string false "Order of dates in the result (e.g., 'asc', 'desc')" -// @Param resourceTypeOrder query string false "Order of resource types in the result (e.g., 'asc', 'desc')" -// @Success 200 {object} app.AntResponse[[]cost.GetCostInfoResult] "Successfully retrieved cost information" -// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" -// @Failure 500 {object} app.AntResponse[string] "Failed to retrieve cost information" -// @Router /api/v1/cost/info [get] -func (s *AntServer) getCostInfos(c echo.Context) error { - var req GetCostInfoReq - if err := c.Bind(&req); err != nil { - return errorResponseJson(http.StatusBadRequest, "Invalid request parameters") - } - - startDate, err := time.Parse("2006-01-02", req.StartDate) - - if err != nil { - return errorResponseJson(http.StatusBadRequest, "start date format is incorrect") - } - - endDate, err := time.Parse("2006-01-02", req.EndDate) - - if err != nil { - return errorResponseJson(http.StatusBadRequest, "end date format is incorrect") - } - - sixMonthsLater := startDate.AddDate(0, 6, 0) - - if endDate.After(sixMonthsLater) { - return errorResponseJson(http.StatusBadRequest, "date range must in 6 month") - } - - if req.CostAggregationType == "" { - req.CostAggregationType = constant.Daily - } - - if req.DateOrder == "" { - req.DateOrder = constant.Asc - } - - arg := cost.GetCostInfoParam{ - StartDate: startDate, - EndDate: endDate, - MigrationIds: req.MigrationIds, - Providers: req.Providers, - ResourceTypes: req.ResourceTypes, - ResourceIds: req.ResourceIds, - CostAggregationType: req.CostAggregationType, - DateOrder: req.DateOrder, - ResourceTypeOrder: req.ResourceTypeOrder, - } - - result, err := s.services.costService.GetCostInfos(arg) - - if err != nil { - return errorResponseJson(http.StatusInternalServerError, "Failed to retrieve load test result") - } - - return successResponseJson(c, "Successfully retrieved load test result", result) -} diff --git a/internal/app/estimate_cost_handler.go b/internal/app/estimate_cost_handler.go new file mode 100644 index 0000000..777d2fc --- /dev/null +++ b/internal/app/estimate_cost_handler.go @@ -0,0 +1,363 @@ +package app + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/cloud-barista/cm-ant/internal/config" + "github.com/cloud-barista/cm-ant/internal/core/common/constant" + "github.com/cloud-barista/cm-ant/internal/core/cost" + "github.com/cloud-barista/cm-ant/internal/utils" + "github.com/labstack/echo/v4" + "github.com/rs/zerolog/log" +) + +// @Id UpdateAndGetEstimateCost +// @Summary Update and Retrieve Estimated Cost Information +// @Description Update the estimate cost based on provided specifications and retrieve the updated cost estimation. Required fields for each specification include `ProviderName`, `RegionName`, and `InstanceType`. Specifications can also be provided in a formatted string using `+` delimiter. +// @Tags [Cost Estimate] +// @Accept json +// @Produce json +// @Param body body UpdateAndGetEstimateCostReq true "Request body for updating and retrieving estimated cost information" +// @Success 200 {object} app.AntResponse[cost.EstimateCostResults] "Successfully updated and retrieved estimated cost information" +// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters or format" +// @Failure 500 {object} app.AntResponse[string] "Failed to update or retrieve estimated cost information" +// @Router /api/v1/cost/estimate [post] +func (a *AntServer) updateAndGetEstimateCost(c echo.Context) error { + var req UpdateAndGetEstimateCostReq + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, err.Error()) + } + + if len(req.Specs) == 0 && len(req.SpecsWithFormat) == 0 { + return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") + } + + pastTime := time.Now().Add(-config.AppConfig.Cost.Estimation.UpdateInterval) + + recommendSpecs := make([]cost.RecommendSpecParam, 0) + + if len(req.Specs) > 0 { + for _, v := range req.Specs { + pn := strings.TrimSpace(strings.ToLower(v.ProviderName)) + rn := strings.TrimSpace(v.RegionName) + it := strings.TrimSpace(v.InstanceType) + + if pn == "" || rn == "" || it == "" { + return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") + } + + param := cost.RecommendSpecParam{ + ProviderName: pn, + RegionName: rn, + InstanceType: it, + Image: strings.TrimSpace(v.Image), + } + recommendSpecs = append(recommendSpecs, param) + } + } + + if len(req.SpecsWithFormat) > 0 { + delim := "+" + + for _, v := range req.SpecsWithFormat { + + cs := strings.TrimSpace(v.CommonSpec) + ci := strings.TrimSpace(v.CommonImage) + + if cs == "" { + return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") + } + + splitedCommonSpec := strings.Split(cs, delim) + splitedCommonImage := strings.Split(ci, delim) + + if len(splitedCommonSpec) != 3 { + utils.LogErrorf("common spec format is not correct; image: %s; spec: %s", ci, cs) + return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common spec format is not correct; image: %s; spec: %s", ci, cs)) + } + + if len(splitedCommonImage) == 3 && (splitedCommonImage[0] != splitedCommonSpec[0] || splitedCommonImage[1] != splitedCommonSpec[1]) { + utils.LogErrorf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs) + return errorResponseJson(http.StatusBadRequest, fmt.Sprintf("common image and spec recommendation is wrong; image: %s; spec: %s", ci, cs)) + } + + pn := strings.TrimSpace(strings.ToLower(splitedCommonSpec[0])) + rn := strings.TrimSpace(splitedCommonSpec[1]) + it := strings.TrimSpace(splitedCommonSpec[2]) + + if pn == "" || rn == "" || it == "" { + return errorResponseJson(http.StatusBadRequest, "request is invalid. check the required request body properties") + } + + param := cost.RecommendSpecParam{ + ProviderName: pn, + RegionName: rn, + InstanceType: it, + } + + if len(splitedCommonImage) == 3 { + param.Image = strings.TrimSpace(splitedCommonImage[2]) + } + + recommendSpecs = append(recommendSpecs, param) + } + } + + arg := cost.UpdateAndGetEstimateCostParam{ + RecommendSpecs: recommendSpecs, + TimeStandard: time.Date(pastTime.Year(), pastTime.Month(), pastTime.Day(), 0, 0, 0, 0, pastTime.Location()), + PricePolicy: constant.OnDemand, + } + + res, err := a.services.costService.UpdateAndGetEstimateCost(arg) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, err.Error()) + } + + return successResponseJson( + c, + "Successfully update and get estimate cost info", + res, + ) +} + +// @Id UpdateEstimateForecastCost +// @Summary Update and Retrieve Estimated Forecast Cost +// @Description Update and retrieve forecasted cost estimates for a specified namespace and migration configuration ID over the past 14 days. +// @Tags [Cost Estimate] +// @Accept json +// @Produce json +// @Param body body UpdateEstimateForecastCostReq true "Request body containing NsId (Namespace ID) and MciId (Migration Configuration ID) for cost estimation forecast" +// @Success 200 {object} app.AntResponse[cost.UpdateEstimateForecastCostInfoResult] "Successfully updated and retrieved estimated forecast cost information" +// @Failure 400 {object} app.AntResponse[string] "Request body binding error" +// @Failure 500 {object} app.AntResponse[string] "Failed to update or retrieve forecast cost information" +// @Router /api/v1/cost/estimate/forecast [post] +func (a *AntServer) updateEstimateForecastCost(c echo.Context) error { + var req UpdateEstimateForecastCostReq + + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, "request body binding error") + } + + endDate := time.Now().Truncate(24*time.Hour).AddDate(0, 0, 1) + startDate := endDate.AddDate(0, 0, -14) + + param := cost.UpdateEstimateForecastCostParam{ + NsId: req.NsId, + MciId: req.MciId, + StartDate: startDate, + EndDate: endDate, + } + + r, err := a.services.costService.UpdateEstimateForecastCost(param) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, err.Error()) + } + + return successResponseJson( + c, + "Successfully update estimate forecast cost info.", + r, + ) +} + +// @Id GetEstimateCost +// @Summary Retrieve Estimated Cost Information +// @Description Fetch estimated cost details based on provider, region, instance type, and resource specifications. Pagination support is provided through `Page` and `Size` parameters. +// @Tags [Cost Estimate] +// @Accept json +// @Produce json +// @Param providerName query string false "Cloud provider name to filter estimated costs" +// @Param regionName query string false "Region name to filter estimated costs" +// @Param instanceType query string false "Instance type to filter estimated costs" +// @Param vCpu query string false "Number of vCPUs to filter estimated costs" +// @Param memory query string false "Memory size to filter estimated costs" +// @Param osType query string false "Operating system type to filter estimated costs" +// @Param page query int false "Page number for pagination (default: 1)" +// @Param size query int false "Number of records per page (default: 100, max: 100)" +// @Success 200 {object} app.AntResponse[cost.EstimateCostInfoResults] "Successfully retrieved estimated cost information" +// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters" +// @Failure 500 {object} app.AntResponse[string] "Failed to retrieve estimated cost information" +// @Router /api/v1/cost/estimate [get] +func (server *AntServer) getEstimateCost(c echo.Context) error { + var req GetEstimateCostInfosReq + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, err.Error()) + } + + if req.Page < 1 { + req.Page = 1 + } + + if req.Size < 1 || req.Size > 100 { + req.Size = 100 + } + + arg := cost.GetEstimateCostParam{ + ProviderName: strings.TrimSpace(req.ProviderName), + RegionName: strings.TrimSpace(req.RegionName), + InstanceType: strings.TrimSpace(req.InstanceType), + VCpu: strings.TrimSpace(req.VCpu), + Memory: strings.TrimSpace(req.Memory), + OsType: strings.TrimSpace(req.OsType), + Page: req.Page, + Size: req.Size, + } + + r, err := server.services.costService.GetEstimateCost(arg) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, err.Error()) + } + + return successResponseJson( + c, + "Successfully get price info.", + r, + ) +} + +// @Id GetEstimateForecastCost +// @Summary Retrieve Estimated Forecast Cost Information +// @Description Fetch estimated forecast cost data based on specified parameters, including a date range that must be within 6 months. Supports pagination and filtering by namespace IDs, migration configuration IDs, and resource types. +// @Tags [Cost Estimate] +// @Accept json +// @Produce json +// @Param startDate query string true "Start date for the forecast cost retrieval in 'YYYY-MM-DD' format" +// @Param endDate query string true "End date for the forecast cost retrieval in 'YYYY-MM-DD' format" +// @Param nsIds query []string false "List of namespace IDs to filter forecast cost information" +// @Param mciIds query []string false "List of migration configuration IDs to filter forecast cost information" +// @Param providers query []string false "List of cloud providers to filter forecast cost information" +// @Param resourceTypes query []string false "List of resource types to filter forecast cost information" +// @Param resourceIds query []string false "List of resource IDs to filter forecast cost information" +// @Param costAggregationType query string false "Type of cost aggregation (e.g., 'daily', 'weekly', 'monthly')" +// @Param dateOrder query string false "Order of dates in the result (e.g., 'asc', 'desc')" +// @Param resourceTypeOrder query string false "Order of resource types in the result (e.g., 'asc', 'desc')" +// @Param page query int false "Page number for pagination (default: 1)" +// @Param size query int false "Number of records per page (default: 10000, max: 10000)" +// @Success 200 {object} app.AntResponse[cost.GetEstimateForecastCostInfoResults] "Successfully retrieved estimated forecast cost information" +// @Failure 400 {object} app.AntResponse[string] "Invalid request parameters or date format errors" +// @Failure 500 {object} app.AntResponse[string] "Failed to retrieve estimated forecast cost information" +// @Router /api/v1/cost/estimate/forecast [get] +func (s *AntServer) getEstimateForecastCost(c echo.Context) error { + + log.Info().Msgf("hello~") + var req GetEstimateForecastCostReq + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, "Invalid request parameters") + } + + startDate, err := time.Parse("2006-01-02", req.StartDate) + + if err != nil { + return errorResponseJson(http.StatusBadRequest, "start date format is incorrect") + } + + endDate, err := time.Parse("2006-01-02", req.EndDate) + + if err != nil { + return errorResponseJson(http.StatusBadRequest, "end date format is incorrect") + } + + if endDate.Before(startDate) { + return errorResponseJson(http.StatusBadRequest, "end date must be after than start date") + } + + sixMonthsLater := startDate.AddDate(0, 6, 0) + + if endDate.After(sixMonthsLater) { + return errorResponseJson(http.StatusBadRequest, "date range must in 6 month") + } + + if req.CostAggregationType == "" { + req.CostAggregationType = constant.Daily + } + + if req.DateOrder == "" { + req.DateOrder = constant.Asc + } + + if req.Page < 1 { + req.Page = 1 + } + + if req.Size < 1 || req.Size > 10000 { + req.Size = 10000 + } + + arg := cost.GetEstimateForecastCostParam{ + Page: req.Page, + Size: req.Size, + StartDate: startDate, + EndDate: endDate, + NsIds: req.NsIds, + MciIds: req.MciIds, + Providers: req.Providers, + ResourceTypes: req.ResourceTypes, + ResourceIds: req.ResourceIds, + CostAggregationType: req.CostAggregationType, + DateOrder: req.DateOrder, + ResourceTypeOrder: req.ResourceTypeOrder, + } + + result, err := s.services.costService.GetEstimateForecastCostInfos(arg) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, "Failed to get estimate forecast cost") + } + + return successResponseJson(c, "Successfully get estimate forecast cost", result) +} + +// --------------------------------------------------------------------------- + +func (server *AntServer) updateCostInfos(c echo.Context) error { + var req UpdateCostInfoReq + + if err := c.Bind(&req); err != nil { + return errorResponseJson(http.StatusBadRequest, "request body binding error") + } + + if len(req.CostResources) == 0 { + return errorResponseJson(http.StatusBadRequest, "Migrated resource id list are required") + } + + costResources := make([]cost.CostResourceParam, 0) + + for _, v := range req.CostResources { + costResources = append(costResources, cost.CostResourceParam{ + ResourceType: v.ResourceType, + ResourceIds: v.ResourceIds, + }) + } + + endDate := time.Now().Truncate(24*time.Hour).AddDate(0, 0, 1) + startDate := endDate.AddDate(0, 0, -14) + param := cost.UpdateCostInfoParam{ + Provider: "aws", + StartDate: startDate, + EndDate: endDate, + CostResources: costResources, + AwsAdditionalInfo: cost.AwsAdditionalInfoParam{ + OwnerId: req.AwsAdditionalInfo.OwnerId, + Regions: req.AwsAdditionalInfo.Regions, + }, + } + + r, err := server.services.costService.UpdateCostInfo(param) + + if err != nil { + return errorResponseJson(http.StatusInternalServerError, err.Error()) + } + + return successResponseJson( + c, + "Successfully updated cost info.", + r, + ) +} diff --git a/internal/app/cost_estimation_req.go b/internal/app/estimate_cost_req.go similarity index 71% rename from internal/app/cost_estimation_req.go rename to internal/app/estimate_cost_req.go index f951277..931dcb0 100644 --- a/internal/app/cost_estimation_req.go +++ b/internal/app/estimate_cost_req.go @@ -2,38 +2,54 @@ package app import "github.com/cloud-barista/cm-ant/internal/core/common/constant" -type EstimateForecastCostReq struct { +type UpdateAndGetEstimateCostReq struct { Specs []struct { ProviderName string `json:"providerName" validate:"required"` RegionName string `json:"regionName" validate:"required"` InstanceType string `json:"instanceType" validate:"required"` Image string `json:"image"` - } `json:"specs" validate:"required"` + } `json:"specs"` SpecsWithFormat []struct { CommonSpec string `json:"commonSpec" validate:"required"` CommonImage string `json:"commonImage"` - } `json:"specsWithFormat" validate:"required"` + } `json:"specsWithFormat"` } -type UpdatePriceInfosReq struct { - ProviderName string `json:"providerName" validate:"required"` - RegionName string `json:"regionName" validate:"required"` - InstanceType string `json:"instanceType" validate:"required"` +type UpdateEstimateForecastCostReq struct { + NsId string `json:"nsId"` + MciId string `json:"mciId"` } -type GetPriceInfosReq struct { +type GetEstimateCostInfosReq struct { ProviderName string `query:"providerName" validate:"required"` RegionName string `query:"regionName" validate:"required"` InstanceType string `query:"instanceType"` VCpu string `query:"vCpu"` Memory string `query:"memory"` OsType string `query:"osType"` + Page int `query:"page"` + Size int `query:"size"` } +type GetEstimateForecastCostReq struct { + Page int `query:"page"` + Size int `query:"size"` + StartDate string `query:"startDate" validate:"required"` + EndDate string `query:"endDate" validate:"required"` + NsIds []string `query:"nsIds"` + MciIds []string `query:"mciIds"` + Providers []string `query:"provider"` + ResourceTypes []constant.ResourceType `query:"resourceTypes"` + ResourceIds []string `query:"resourceIds"` + CostAggregationType constant.CostAggregationType `query:"costAggregationType" validate:"required"` + DateOrder constant.OrderType `query:"dateOrder"` + ResourceTypeOrder constant.OrderType `query:"resourceTypeOrder"` +} + +// ------------------------------------------------------------------------------------------------------------------- + type UpdateCostInfoReq struct { - MigrationId string `json:"migrationId"` - ConnectionName string `json:"connectionName" validate:"required"` CostResources []CostResourceReq `json:"costResources" validate:"required"` AwsAdditionalInfo AwsAdditionalInfoReq `json:"awsAdditionalInfo"` } @@ -47,15 +63,3 @@ type AwsAdditionalInfoReq struct { OwnerId string `json:"ownerId"` Regions []string `json:"regions"` } - -type GetCostInfoReq struct { - StartDate string `query:"startDate" validate:"required"` - EndDate string `query:"endDate" validate:"required"` - MigrationIds []string `query:"migrationIds"` - Providers []string `query:"provider"` - ResourceTypes []constant.ResourceType `query:"resourceTypes"` - ResourceIds []string `query:"resourceIds"` - CostAggregationType constant.CostAggregationType `query:"costAggregationType" validate:"required"` - DateOrder constant.OrderType `query:"dateOrder"` - ResourceTypeOrder constant.OrderType `query:"resourceTypeOrder"` -} diff --git a/internal/app/middlewares.go b/internal/app/middlewares.go index 54d5bed..73b318e 100644 --- a/internal/app/middlewares.go +++ b/internal/app/middlewares.go @@ -1,6 +1,7 @@ package app import ( + "net/http" "strings" "time" @@ -10,78 +11,24 @@ import ( "github.com/cloud-barista/cm-ant/internal/utils" "github.com/labstack/echo/v4" - zerolog "github.com/rs/zerolog/log" + "github.com/rs/zerolog/log" ) -// setMiddleware configures middleware for the Echo server. -func setMiddleware(e *echo.Echo) { - logSkipPattern := [][]string{ +var ( + logSkipPattern = [][]string{ {"/ant/swagger/*"}, + {"/ant/readyz"}, } +) + +func setMiddleware(e *echo.Echo) { e.Use( - middleware.RequestLoggerWithConfig( - middleware.RequestLoggerConfig{ - Skipper: func(c echo.Context) bool { - path := c.Request().URL.Path - query := c.Request().URL.RawQuery - for _, patterns := range logSkipPattern { - isAllMatched := true - for _, pattern := range patterns { - if !strings.Contains(path+query, pattern) { - isAllMatched = false - break - } - } - if isAllMatched { - return true - } - } - return false - }, - LogError: true, - LogRequestID: true, - LogRemoteIP: true, - LogHost: true, - LogMethod: true, - LogURI: true, - LogUserAgent: false, - LogStatus: true, - LogLatency: true, - LogContentLength: true, - LogResponseSize: true, - LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { - if v.Error == nil { - zerolog.Info(). - Str("id", v.RequestID). - Str("client_ip", v.RemoteIP). - // Str("host", v.Host). - Str("method", v.Method). - Str("URI", v.URI). - Int("status", v.Status). - // Int64("latency", v.Latency.Nanoseconds()). - Str("latency_human", v.Latency.String()). - Str("bytes_in", v.ContentLength). - Int64("bytes_out", v.ResponseSize). - Msg("request") - } else { - zerolog.Error(). - Err(v.Error). - Str("id", v.RequestID). - Str("client_ip", v.RemoteIP). - // Str("host", v.Host). - Str("method", v.Method). - Str("URI", v.URI). - Int("status", v.Status). - // Int64("latency", v.Latency.Nanoseconds()). - Str("latency_human", v.Latency.String()). - Str("bytes_in", v.ContentLength). - Int64("bytes_out", v.ResponseSize). - Msg("request error") - } - return nil - }, - }, - ), + middleware.Secure(), + middleware.RequestID(), + middleware.Recover(), + middleware.Gzip(), + middleware.CORS(), + Zerologger(logSkipPattern), middleware.TimeoutWithConfig( middleware.TimeoutConfig{ Skipper: middleware.DefaultSkipper, @@ -92,27 +39,76 @@ func setMiddleware(e *echo.Echo) { Timeout: 300 * time.Second, }, ), - middleware.Recover(), - middleware.RequestID(), + middleware.RateLimiter(middleware.NewRateLimiterMemoryStore(20)), - middleware.CORS(), ) } -func RequestIdAndDetailsIssuer(next echo.HandlerFunc) echo.HandlerFunc { - return func(c echo.Context) error { - // Make X-Request-Id visible to all handlers - c.Response().Header().Set("Access-Control-Expose-Headers", echo.HeaderXRequestID) - - // Get or generate Request ID - reqID := c.Request().Header.Get(echo.HeaderXRequestID) - if reqID == "" { - reqID = utils.CreateUniqIdBaseOnUnixTime() - } - - // Set Request on the context - c.Set("RequestID", reqID) - - return next(c) - } +func Zerologger(skipPatterns [][]string) echo.MiddlewareFunc { + return middleware.RequestLoggerWithConfig(middleware.RequestLoggerConfig{ + Skipper: func(c echo.Context) bool { + path := c.Request().URL.Path + query := c.Request().URL.RawQuery + for _, patterns := range skipPatterns { + isAllMatched := true + for _, pattern := range patterns { + if !strings.Contains(path+query, pattern) { + isAllMatched = false + break + } + } + if isAllMatched { + return true + } + } + return false + }, + LogError: true, + LogRequestID: true, + LogRemoteIP: true, + LogHost: true, + LogMethod: true, + LogURI: true, + LogUserAgent: false, + LogStatus: true, + LogLatency: true, + LogContentLength: true, + LogResponseSize: true, + // HandleError: true, // forwards error to the global error handler, so it can decide appropriate status code + LogValuesFunc: func(c echo.Context, v middleware.RequestLoggerValues) error { + if v.Error == nil { + if v.Method != http.MethodOptions { + log.Info(). + Str("ID", v.RequestID). + Str("Method", v.Method). + Str("URI", v.URI). + Str("clientIP", v.RemoteIP). + //Str("host", v.Host). + //Str("user_agent", v.UserAgent). + Int("status", v.Status). + //Int64("latency", v.Latency.Nanoseconds()). + Str("latency", v.Latency.String()). + //Str("bytes_in", v.ContentLength). + //Int64("bytes_out", v.ResponseSize). + Msg("") + } + } else { + log.Error(). + Err(v.Error). + Str("ID", v.RequestID). + Str("Method", v.Method). + Str("URI", v.URI). + Str("clientIP", v.RemoteIP). + // Str("host", v.Host). + //Str("user_agent", v.UserAgent). + Int("status", v.Status). + // Int64("latency", v.Latency.Nanoseconds()). + Str("latency", v.Latency.String()). + //Str("bytes_in", v.ContentLength). + //Int64("bytes_out", v.ResponseSize). + Msg("") + } + return nil + }, + }) } diff --git a/internal/app/router.go b/internal/app/router.go index 1cfaa68..84c7ac2 100644 --- a/internal/app/router.go +++ b/internal/app/router.go @@ -56,24 +56,15 @@ func (server *AntServer) InitRouter() error { } } } - - - { - costEstimationHandler := versionRouter.Group("/cost-estimation") - costEstimationHandler.POST("/forecast", server.estimateForecastCost) + { + costEstimationHandler := versionRouter.Group("/cost/estimate") - priceRouter := versionRouter.Group("/price") - { - priceRouter.POST("/info", server.updatePriceInfos) - priceRouter.GET("/info", server.getPriceInfos) - } + costEstimationHandler.POST("", server.updateAndGetEstimateCost) + costEstimationHandler.GET("", server.getEstimateCost) - costRouter := versionRouter.Group("/cost") - { - costRouter.POST("/info", server.updateCostInfos) - costRouter.GET("/info", server.getCostInfos) - } + costEstimationHandler.POST("/forecast", server.updateEstimateForecastCost) + costEstimationHandler.GET("/forecast", server.getEstimateForecastCost) } return nil diff --git a/internal/app/server.go b/internal/app/server.go index e538a57..338b36c 100644 --- a/internal/app/server.go +++ b/internal/app/server.go @@ -90,7 +90,7 @@ func initializeRepositories(conn *gorm.DB) *antRepositories { func initializeServices(repos *antRepositories, tbClient *tumblebug.TumblebugClient, sClient *spider.SpiderClient) *antServices { loadServ := load.NewLoadService(repos.loadRepo, tbClient) - cc := cost.NewAwsCostExplorerSpiderCostCollector(sClient) + cc := cost.NewAwsCostExplorerSpiderCostCollector(sClient, tbClient) pc := cost.NewSpiderPriceCollector(sClient) costServ := cost.NewCostService(repos.costRepo, pc, cc) diff --git a/internal/config/config.go b/internal/config/config.go index f92162b..4ccf806 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ import ( "time" "github.com/cloud-barista/cm-ant/internal/utils" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -35,9 +36,7 @@ type AntConfig struct { Cost struct { Estimation struct { - Forcast struct { - PriceUpdateInterval time.Duration `yaml:"priceUpdateInterval"` - } `yaml:"forecast"` + UpdateInterval time.Duration `yaml:"updateInterval"` } `yaml:"estimation"` } `yaml:"cost"` Load struct { @@ -47,9 +46,9 @@ type AntConfig struct { Version string `yaml:"version"` } `yaml:"jmeter"` } `yaml:"load"` - Logging struct { + Log struct { Level string `yaml:"level"` - } `yaml:"logging"` + } `yaml:"log"` Database struct { Driver string `yaml:"driver"` Host string `yaml:"host"` @@ -61,7 +60,7 @@ type AntConfig struct { } func InitConfig() error { - utils.LogInfo("Initializing configuration...") + log.Info().Msg("Initializing configuration...") cfg := AntConfig{} @@ -75,17 +74,17 @@ func InitConfig() error { err := viper.ReadInConfig() if err != nil { - utils.LogErrorf("Fatal error while reading config file: %v", err) + log.Error().Msgf("Fatal error while reading config file: %v", err) return fmt.Errorf("fatal error while read config file: %w", err) } err = viper.Unmarshal(&cfg) if err != nil { - utils.LogErrorf("Fatal error while unmarshaling config: %v", err) + log.Error().Msgf("Fatal error while unmarshaling config: %v", err) return fmt.Errorf("fatal error while unmarshal from config to ant config: %w", err) } - utils.LogInfof("Configuration loaded successfully: %+v", cfg) + log.Info().Msgf("Configuration loaded successfully: %+v", cfg) AppConfig = cfg return nil diff --git a/internal/core/cost/cost_collector.go b/internal/core/cost/cost_collector.go index 9033f2c..de9d01c 100644 --- a/internal/core/cost/cost_collector.go +++ b/internal/core/cost/cost_collector.go @@ -11,25 +11,29 @@ import ( "github.com/cloud-barista/cm-ant/internal/core/common/constant" "github.com/cloud-barista/cm-ant/internal/infra/outbound/spider" + "github.com/cloud-barista/cm-ant/internal/infra/outbound/tumblebug" "github.com/cloud-barista/cm-ant/internal/utils" ) type CostCollector interface { Readyz(context.Context) error - GetCostInfos(context.Context, UpdateCostInfoParam) (CostInfos, error) + UpdateEstimateForecastCost(context.Context, UpdateEstimateForecastCostParam) (EstimateForecastCostInfos, error) + GetCostInfos(context.Context, UpdateCostInfoParam) (EstimateForecastCostInfos, error) } -type AwsCostExplorerSpiderCostCollector struct { +type AwsCostExplorerBaristaCostCollector struct { sc *spider.SpiderClient + tc *tumblebug.TumblebugClient } -func NewAwsCostExplorerSpiderCostCollector(sc *spider.SpiderClient) CostCollector { - return &AwsCostExplorerSpiderCostCollector{ +func NewAwsCostExplorerSpiderCostCollector(sc *spider.SpiderClient, tc *tumblebug.TumblebugClient) CostCollector { + return &AwsCostExplorerBaristaCostCollector{ sc: sc, + tc: tc, } } -func (a *AwsCostExplorerSpiderCostCollector) Readyz(ctx context.Context) error { +func (a *AwsCostExplorerBaristaCostCollector) Readyz(ctx context.Context) error { err := a.sc.ReadyzWithContext(ctx) if err != nil { return err @@ -88,7 +92,7 @@ type groupBy struct { Type string `json:"type"` // DIMENSION | TAG | COST_CATEGORY } -func (a *AwsCostExplorerSpiderCostCollector) generateFilterValue( +func (a *AwsCostExplorerBaristaCostCollector) generateFilterValue( costResources []CostResourceParam, awsAdditionalInfo AwsAdditionalInfoParam, ) ( []string, []string, error, @@ -124,7 +128,12 @@ func (a *AwsCostExplorerSpiderCostCollector) generateFilterValue( return serviceValue, resourceIdValues, nil } -func (a *AwsCostExplorerSpiderCostCollector) GetCostInfos(ctx context.Context, param UpdateCostInfoParam) (CostInfos, error) { +func (a *AwsCostExplorerBaristaCostCollector) GetCostInfos(ctx context.Context, param UpdateCostInfoParam) (EstimateForecastCostInfos, error) { + + if param.ConnectionName == "" { + param.ConnectionName = costExplorerConnectionName + } + serviceFilterValue, resourceIdFilterValue, err := a.generateFilterValue(param.CostResources, param.AwsAdditionalInfo) if err != nil { utils.LogError("parsing service and resource id for filtering cost explorer value") @@ -218,7 +227,7 @@ func (a *AwsCostExplorerSpiderCostCollector) GetCostInfos(ctx context.Context, p return nil, ErrCostResultEmpty } - var costInfos = make([]CostInfo, 0) + var costInfos = make([]EstimateForecastCostInfo, 0) for _, result := range res.ResultsByTime { if result.Groups == nil { utils.LogError("groups is nil; it must not be nil") @@ -275,8 +284,8 @@ func (a *AwsCostExplorerSpiderCostCollector) GetCostInfos(ctx context.Context, p continue } - costInfo := CostInfo{ - MigrationId: param.MigrationId, + costInfo := EstimateForecastCostInfo{ + // MigrationId: param.MigrationId, Provider: param.Provider, ConnectionName: param.ConnectionName, ResourceType: resourceType, @@ -296,3 +305,87 @@ func (a *AwsCostExplorerSpiderCostCollector) GetCostInfos(ctx context.Context, p return costInfos, nil } + +const ( + nsKey = "sys.namespace" + provider = "aws" + costExplorerConnectionName = "aws-us-east-1" + defaultNsId = "ns01" + defaultMciId = "mmci01" +) + +func (a *AwsCostExplorerBaristaCostCollector) UpdateEstimateForecastCost(ctx context.Context, param UpdateEstimateForecastCostParam) (EstimateForecastCostInfos, error) { + + if param.NsId == "" { + param.NsId = defaultNsId + } + + if param.MciId == "" { + param.MciId = defaultMciId + } + + res := EstimateForecastCostInfos{} + + mci, err := a.tc.GetMciWithContext(ctx, param.NsId, param.MciId) + + if err != nil { + utils.LogError("error while get mci from tumblebug; ", err) + return res, err + } + + if len(mci.Vm) == 0 { + return nil, errors.New("there is no vm in mci") + } + + mciLabels := mci.Label + _ = mciLabels[nsKey] + + arg := UpdateCostInfoParam{ + Provider: provider, + ConnectionName: costExplorerConnectionName, + StartDate: param.StartDate, + EndDate: param.EndDate, + CostResources: make([]CostResourceParam, 0), + } + + vmIds := make([]string, 0) + + for _, mci := range mci.Vm { + pn := mci.ConnectionConfig.ProviderName + + if pn == "" || !strings.EqualFold(strings.ToLower(pn), "aws") { + utils.LogWarnf("CSP: %s, does not support yet", pn) + continue + } + + vmId := mci.CspResourceId + _ = mci.ConnectionName + _ = mci.Label + + vmIds = append(vmIds, vmId) + } + + if len(vmIds) != 0 { + arg.CostResources = append(arg.CostResources, CostResourceParam{ + ResourceType: constant.VM, + ResourceIds: vmIds, + }) + } else { + return res, errors.New("no vm resource create on aws") + } + + infos, err := a.GetCostInfos(ctx, arg) + + if err != nil { + utils.LogError("error while get cost info from spider;", err) + return res, fmt.Errorf("error from get cost infos +%w", err) + } + + for i := range infos { + info := infos[i] + info.NsId = param.NsId + info.MciId = param.MciId + } + + return infos, nil +} diff --git a/internal/core/cost/dtos.go b/internal/core/cost/dtos.go index cae46f9..4f4872c 100644 --- a/internal/core/cost/dtos.go +++ b/internal/core/cost/dtos.go @@ -8,7 +8,7 @@ import ( "github.com/cloud-barista/cm-ant/internal/core/common/constant" ) -type EstimateForecastCostParam struct { +type UpdateAndGetEstimateCostParam struct { RecommendSpecs []RecommendSpecParam `json:"recommendSpecs"` TimeStandard time.Time `json:"timeStandard"` @@ -34,23 +34,23 @@ func (r RecommendSpecParam) Hash() string { return hex.EncodeToString(hashBytes) } -type EstimateForecastCostResult struct { - TotalMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` - TotalMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` - EsimateForecastCostSpecResults []EsimateForecastCostSpecResult `json:"esimateForecastCostSpecResults"` +type EstimateCostResults struct { + TotalMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` + TotalMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` + EsimateCostSpecResults []EsimateCostSpecResults `json:"esimateForecastCostSpecResults"` } -type EsimateForecastCostSpecResult struct { - ProviderName string `json:"providerName"` - RegionName string `json:"regionName"` - InstanceType string `json:"instanceType"` - ImageName string `json:"imageName"` - SpecMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` - SpecMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` - EstimateForecastCostSpecDetailResults []EstimateForecastCostSpecDetailResult `json:"estimateForecastCostSpecDetailResults"` +type EsimateCostSpecResults struct { + ProviderName string `json:"providerName"` + RegionName string `json:"regionName"` + InstanceType string `json:"instanceType"` + ImageName string `json:"imageName"` + SpecMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` + SpecMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` + EstimateCostSpecDetailResults []EstimateCostSpecDetailResult `json:"estimateForecastCostSpecDetailResults"` } -type EstimateForecastCostSpecDetailResult struct { +type EstimateCostSpecDetailResult struct { ID uint `json:"id"` VCpu string `json:"vCpu,omitempty"` Memory string `json:"memory,omitempty"` @@ -77,7 +77,7 @@ type UpdatePriceInfosParam struct { PricePolicy constant.PricePolicy } -type GetPriceInfosParam struct { +type GetEstimateCostParam struct { ProviderName string RegionName string InstanceType string @@ -88,14 +88,16 @@ type GetPriceInfosParam struct { TimeStandard time.Time PricePolicy constant.PricePolicy + Page int + Size int } -type AllPriceInfoResult struct { - PriceInfoList []PriceInfoResult `json:"priceInfoList,omitempty"` - ResultCount int64 `json:"resultCount"` +type EstimateCostInfoResults struct { + EstimateCostInfoResult []EstimateCostInfoResult `json:"estimateCostInfoResult,omitempty"` + ResultCount int64 `json:"resultCount"` } -type PriceInfoResult struct { +type EstimateCostInfoResult struct { ID uint `json:"id"` ProviderName string `json:"providerName"` RegionName string `json:"regionName"` @@ -116,36 +118,26 @@ type PriceInfoResult struct { LastUpdatedAt time.Time `json:"lastUpdatedAt,omitempty"` } -type UpdateCostInfoParam struct { - MigrationId string - Provider string // currently only aws - ConnectionName string - StartDate time.Time - EndDate time.Time - CostResources []CostResourceParam - AwsAdditionalInfo AwsAdditionalInfoParam +type UpdateEstimateForecastCostParam struct { + NsId string + MciId string + StartDate time.Time + EndDate time.Time } -type CostResourceParam struct { - ResourceType constant.ResourceType - ResourceIds []string -} - -type AwsAdditionalInfoParam struct { - OwnerId string `json:"ownerId"` - Regions []string `json:"regions"` -} - -type UpdateCostInfoResult struct { +type UpdateEstimateForecastCostInfoResult struct { FetchedDataCount int64 `json:"fetchedDataCount"` UpdatedDataCount int64 `json:"updatedDataCount"` - InsertedDataCount int64 `insertedDataCount` + InsertedDataCount int64 `json:"insertedDataCount"` } -type GetCostInfoParam struct { +type GetEstimateForecastCostParam struct { + Page int + Size int StartDate time.Time EndDate time.Time - MigrationIds []string + NsIds []string + MciIds []string Providers []string ResourceTypes []constant.ResourceType ResourceIds []string @@ -154,7 +146,11 @@ type GetCostInfoParam struct { ResourceTypeOrder constant.OrderType } -type GetCostInfoResult struct { +type GetEstimateForecastCostInfoResults struct { + GetEstimateForecastCostInfoResults []GetEstimateForecastCostInfoResult `json:"getEstimateForecastCostInfoResults,omitempty"` + ResultCount int64 `json:"resultCount"` +} +type GetEstimateForecastCostInfoResult struct { Provider string `json:"provider"` ResourceType string `json:"resourceType"` Category string `json:"category"` @@ -163,3 +159,25 @@ type GetCostInfoResult struct { Date time.Time `json:"date"` TotalCost float64 `json:"totalCost"` } + +// ------------------------------------------------------------------- + +type UpdateCostInfoParam struct { + // MigrationId string + Provider string // currently only aws + ConnectionName string + StartDate time.Time + EndDate time.Time + CostResources []CostResourceParam + AwsAdditionalInfo AwsAdditionalInfoParam +} + +type CostResourceParam struct { + ResourceType constant.ResourceType + ResourceIds []string +} + +type AwsAdditionalInfoParam struct { + OwnerId string `json:"ownerId"` + Regions []string `json:"regions"` +} diff --git a/internal/core/cost/models.go b/internal/core/cost/models.go index 4eb2c1e..3c66532 100644 --- a/internal/core/cost/models.go +++ b/internal/core/cost/models.go @@ -7,9 +7,9 @@ import ( "gorm.io/gorm" ) -type PriceInfos []*PriceInfo +type EstimateCostInfos []*EstimateCostInfo -type PriceInfo struct { +type EstimateCostInfo struct { gorm.Model ProviderName string `gorm:"index"` RegionName string `gorm:"index"` @@ -35,11 +35,10 @@ type PriceInfo struct { ImageName string `gorm:"index"` } -type CostInfos []CostInfo +type EstimateForecastCostInfos []EstimateForecastCostInfo -type CostInfo struct { +type EstimateForecastCostInfo struct { gorm.Model - MigrationId string `gorm:"index"` Provider string `gorm:"index"` ConnectionName string ResourceType constant.ResourceType `gorm:"index"` @@ -51,10 +50,6 @@ type CostInfo struct { Granularity string `gorm:"index"` StartDate time.Time `gorm:"index"` EndDate time.Time `gorm:"index"` -} - -type CostUpdateRestrict struct { - gorm.Model - StandardDate time.Time - UpdateCount int64 + NsId string `gorm:"index"` + MciId string `gorm:"index"` } diff --git a/internal/core/cost/price_collector.go b/internal/core/cost/price_collector.go index f68e362..6ca512e 100644 --- a/internal/core/cost/price_collector.go +++ b/internal/core/cost/price_collector.go @@ -17,8 +17,7 @@ import ( type PriceCollector interface { Readyz(context.Context) error - GetPriceInfos(context.Context, UpdatePriceInfosParam) (PriceInfos, error) - FetchPriceInfos(context.Context, RecommendSpecParam) (PriceInfos, error) + FetchPriceInfos(context.Context, RecommendSpecParam) (EstimateCostInfos, error) } var ( @@ -33,21 +32,21 @@ var ( "ncpvpc": "Monthly Flat Rate", } - priceValidator = map[string]func(res *PriceInfo) bool{ - "aws": func(res *PriceInfo) bool { + priceValidator = map[string]func(res *EstimateCostInfo) bool{ + "aws": func(res *EstimateCostInfo) bool { return !strings.Contains(res.PriceDescription, "Reservation") }, - "gcp": func(res *PriceInfo) bool { return true }, - "azure": func(res *PriceInfo) bool { return true }, - "tencent": func(res *PriceInfo) bool { + "gcp": func(res *EstimateCostInfo) bool { return true }, + "azure": func(res *EstimateCostInfo) bool { return true }, + "tencent": func(res *EstimateCostInfo) bool { return res.OriginalPricePolicy == onDemandPricingPolicyMap[res.ProviderName] }, - "alibaba": func(res *PriceInfo) bool { return true }, - "ibm": func(res *PriceInfo) bool { + "alibaba": func(res *EstimateCostInfo) bool { return true }, + "ibm": func(res *EstimateCostInfo) bool { return strings.Contains(res.OriginalUnit, "Instance-Hour") }, - "ncp": func(res *PriceInfo) bool { return true }, - "ncpvpc": func(res *PriceInfo) bool { return true }, + "ncp": func(res *EstimateCostInfo) bool { return true }, + "ncpvpc": func(res *EstimateCostInfo) bool { return true }, } units = map[string]bool{ @@ -79,121 +78,7 @@ func (s *SpiderPriceCollector) Readyz(ctx context.Context) error { return nil } -func (s *SpiderPriceCollector) GetPriceInfos(ctx context.Context, param UpdatePriceInfosParam) (PriceInfos, error) { - connectionName := fmt.Sprintf("%s-%s", strings.ToLower(param.ProviderName), strings.ToLower(param.RegionName)) - - req := spider.PriceInfoReq{ - ConnectionName: connectionName, - FilterList: s.generateFilterList(param), - } - - result, err := s.sc.GetPriceInfoWithContext(ctx, param.RegionName, req) - - if err != nil { - - if strings.Contains(err.Error(), "you don't have any permission") { - return nil, fmt.Errorf("you don't have permission to query the price for %s", param.ProviderName) - } - return nil, err - } - - createdPriceInfo := make([]*PriceInfo, 0) - if result.CloudPriceList != nil { - for i := range result.CloudPriceList { - p := result.CloudPriceList[i] - - if p.PriceList != nil { - for j := range p.PriceList { - - pl := p.PriceList[j] - - productInfo := pl.ProductInfo - vCpu := s.naChecker(productInfo.Vcpu) - originalMemory := s.naChecker(productInfo.Memory) - - if vCpu == "" || originalMemory == "" { - continue - } - - memory, memoryUnit := s.splitMemory(originalMemory) - zoneName := s.naChecker(productInfo.ZoneName) - osType := s.naChecker(productInfo.OperatingSystem) - storage := s.naChecker(productInfo.Storage) - productDescription := s.naChecker(productInfo.Description) - - var price, originalCurrency, originalUnit, priceDescription string - var unit constant.PriceUnit - var currency constant.PriceCurrency - - priceInfo := pl.PriceInfo - - if priceInfo.PricingPolicies != nil { - for k := range priceInfo.PricingPolicies { - policy := priceInfo.PricingPolicies[k] - originalPricePolicy := s.naChecker(policy.PricingPolicy) - priceDescription = s.naChecker(policy.Description) - originalCurrency = s.naChecker(policy.Currency) - originalUnit = s.naChecker(policy.Unit) - unit = s.parseUnit(originalUnit) - currency = s.parseCurrency(policy.Currency) - convertedPrice, err := strconv.ParseFloat(policy.Price, 64) - if err != nil { - continue - } - - if convertedPrice == float64(0) { - continue - } - price = s.naChecker(policy.Price) - - if price == "" { - continue - } - - pi := PriceInfo{ - ProviderName: param.ProviderName, - RegionName: productInfo.RegionName, - InstanceType: productInfo.InstanceType, - ZoneName: zoneName, - VCpu: vCpu, - OriginalMemory: originalMemory, - Memory: memory, - MemoryUnit: memoryUnit, - Storage: storage, - OsType: osType, - ProductDescription: productDescription, - OriginalPricePolicy: originalPricePolicy, - PricePolicy: constant.OnDemand, - Price: price, - Currency: currency, - Unit: unit, - OriginalUnit: originalUnit, - OriginalCurrency: originalCurrency, - PriceDescription: priceDescription, - CalculatedMonthlyPrice: s.calculatePrice(price, unit), - } - - if !priceValidator[param.ProviderName](&pi) { - continue - } - - createdPriceInfo = append(createdPriceInfo, &pi) - } - } - - } - } - } - } - - sort.Slice(createdPriceInfo, func(i, j int) bool { - return createdPriceInfo[i].Price < createdPriceInfo[j].Price - }) - - return createdPriceInfo, nil -} - -func (s *SpiderPriceCollector) FetchPriceInfos(ctx context.Context, param RecommendSpecParam) (PriceInfos, error) { +func (s *SpiderPriceCollector) FetchPriceInfos(ctx context.Context, param RecommendSpecParam) (EstimateCostInfos, error) { connectionName := fmt.Sprintf("%s-%s", strings.ToLower(param.ProviderName), strings.ToLower(param.RegionName)) req := spider.PriceInfoReq{ @@ -211,7 +96,7 @@ func (s *SpiderPriceCollector) FetchPriceInfos(ctx context.Context, param Recomm return nil, err } - createdPriceInfo := make([]*PriceInfo, 0) + createdPriceInfo := make([]*EstimateCostInfo, 0) if result.CloudPriceList != nil { for i := range result.CloudPriceList { p := result.CloudPriceList[i] @@ -272,7 +157,7 @@ func (s *SpiderPriceCollector) FetchPriceInfos(ctx context.Context, param Recomm continue } - pi := PriceInfo{ + pi := EstimateCostInfo{ ProviderName: param.ProviderName, RegionName: productInfo.RegionName, InstanceType: productInfo.InstanceType, @@ -340,32 +225,6 @@ func (s *SpiderPriceCollector) generateFilter(param RecommendSpecParam) []spider return ret } -func (s *SpiderPriceCollector) generateFilterList(param UpdatePriceInfosParam) []spider.FilterReq { - - providerName := strings.ToLower(param.ProviderName) - param.ProviderName = providerName - - ret := []spider.FilterReq{ - { - Key: "pricingPolicy", - Value: onDemandPricingPolicyMap[providerName], - }, - { - Key: "regionName", - Value: param.RegionName, - }, - } - - if param.InstanceType != "" { - ret = append(ret, spider.FilterReq{ - Key: "instanceType", - Value: param.InstanceType, - }) - } - - return ret -} - func (s *SpiderPriceCollector) parseUnit(p string) constant.PriceUnit { ret := constant.PerHour diff --git a/internal/core/cost/repository.go b/internal/core/cost/repository.go index d7ebbd8..fd0831a 100644 --- a/internal/core/cost/repository.go +++ b/internal/core/cost/repository.go @@ -37,11 +37,12 @@ func (r *CostRepository) execInTransaction(ctx context.Context, fn func(*gorm.DB return tx.Commit().Error } -func (r *CostRepository) GetAllMatchingPriceInfoList(ctx context.Context, param GetPriceInfosParam) (PriceInfos, error) { - var priceInfoList []*PriceInfo +func (r *CostRepository) GetMatchingEstimateCostInfosTx(ctx context.Context, param GetEstimateCostParam) (EstimateCostInfos, int64, error) { + var priceInfoList []*EstimateCostInfo + var totalRows int64 err := r.execInTransaction(ctx, func(d *gorm.DB) error { - q := d.Model(&PriceInfo{}). + q := d.Model(&EstimateCostInfo{}). Where( "price_policy = ? AND updated_at >= ?", param.PricePolicy, param.TimeStandard, @@ -73,6 +74,14 @@ func (r *CostRepository) GetAllMatchingPriceInfoList(ctx context.Context, param q = q.Where("LOWER(os_type) = ?", strings.ToLower(param.OsType)) } + if err := q.Count(&totalRows).Error; err != nil { + return err + } + + offset := (param.Page - 1) * param.Size + q = q.Offset(offset). + Limit(param.Size) + if err := q.Find(&priceInfoList).Error; err != nil { return err } @@ -80,14 +89,14 @@ func (r *CostRepository) GetAllMatchingPriceInfoList(ctx context.Context, param return nil }) - return priceInfoList, err + return priceInfoList, totalRows, err } -func (r *CostRepository) GetMatchingForecastCost(ctx context.Context, param RecommendSpecParam, timeStandard time.Time, pricePolicy constant.PricePolicy) (PriceInfos, error) { - var priceInfos []*PriceInfo +func (r *CostRepository) GetMatchingEstimateCostTx(ctx context.Context, param RecommendSpecParam, timeStandard time.Time, pricePolicy constant.PricePolicy) (EstimateCostInfos, error) { + var priceInfos []*EstimateCostInfo err := r.execInTransaction(ctx, func(d *gorm.DB) error { - q := d.Model(&PriceInfo{}). + q := d.Model(&EstimateCostInfo{}). Where( "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND instance_type = ? AND image_name = ? AND price_policy = ? AND last_updated_at >= ?", strings.ToLower(param.ProviderName), @@ -112,28 +121,7 @@ func (r *CostRepository) GetMatchingForecastCost(ctx context.Context, param Reco return priceInfos, nil } -func (r *CostRepository) CountMatchingPriceInfoList(ctx context.Context, param UpdatePriceInfosParam) (int64, error) { - var totalCount int64 - - err := r.execInTransaction(ctx, func(d *gorm.DB) error { - q := d.Model(&PriceInfo{}). - Where( - "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND price_policy = ? AND updated_at >= ?", - strings.ToLower(param.ProviderName), strings.ToLower(param.RegionName), param.PricePolicy, param.TimeStandard, - ) - - if param.InstanceType != "" { - q = q.Where("LOWER(instance_type) = ?", strings.ToLower(param.InstanceType)) - } - - return q.Count(&totalCount).Error - }) - - return totalCount, err - -} - -func (r *CostRepository) BatchInsertAllForecastCostResult(ctx context.Context, created PriceInfos) error { +func (r *CostRepository) BatchInsertAllEstimateCostResultTx(ctx context.Context, created EstimateCostInfos) error { batchSize := 100 err := r.execInTransaction(ctx, func(d *gorm.DB) error { @@ -155,14 +143,13 @@ func (r *CostRepository) BatchInsertAllForecastCostResult(ctx context.Context, c } -func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo CostInfo) (int64, int64, error) { +func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo EstimateForecastCostInfo) (int64, int64, error) { var updateCount = int64(0) var insertCount = int64(0) err := r.execInTransaction(ctx, func(d *gorm.DB) error { err := d. Model(costInfo). - Where(&CostInfo{ - MigrationId: costInfo.MigrationId, + Where(&EstimateForecastCostInfo{ Provider: costInfo.Provider, ResourceType: costInfo.ResourceType, Category: costInfo.Category, @@ -170,6 +157,8 @@ func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo CostInfo) Granularity: costInfo.Granularity, StartDate: costInfo.StartDate, EndDate: costInfo.EndDate, + NsId: costInfo.NsId, + MciId: costInfo.MciId, }).First(&costInfo).Error if err != nil && err != gorm.ErrRecordNotFound { @@ -188,7 +177,6 @@ func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo CostInfo) }).Error; err != nil { return err } - updateCount++ } @@ -199,11 +187,12 @@ func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo CostInfo) } -func (r *CostRepository) GetCostInfoWithFilter(ctx context.Context, param GetCostInfoParam) ([]GetCostInfoResult, error) { - var costInfo []GetCostInfoResult +func (r *CostRepository) GetEstimateForecastCostInfosTx(ctx context.Context, param GetEstimateForecastCostParam) ([]GetEstimateForecastCostInfoResult, int64, error) { + var costInfo []GetEstimateForecastCostInfoResult + var totalRows int64 err := r.execInTransaction(ctx, func(d *gorm.DB) error { - query := d.Model(&CostInfo{}) + query := d.Model(&EstimateForecastCostInfo{}) if len(param.Providers) > 0 { query = query.Where("provider IN ?", param.Providers) @@ -216,6 +205,14 @@ func (r *CostRepository) GetCostInfoWithFilter(ctx context.Context, param GetCos query = query.Where("actual_resource_id IN ?", param.ResourceIds) } + if len(param.NsIds) > 0 { + query = query.Where("ns_id IN ?", param.NsIds) + } + + if len(param.MciIds) > 0 { + query = query.Where("mci_id IN ?", param.MciIds) + } + query = query.Where("start_date >= ? AND end_date <= ?", param.StartDate, param.EndDate) switch param.CostAggregationType { @@ -238,6 +235,14 @@ func (r *CostRepository) GetCostInfoWithFilter(ctx context.Context, param GetCos query = query.Order("resource_type " + string(param.ResourceTypeOrder)) } + if err := query.Count(&totalRows).Error; err != nil { + return err + } + + offset := (param.Page - 1) * param.Size + query = query.Offset(offset). + Limit(param.Size) + if err := query.Find(&costInfo).Error; err != nil { return err } @@ -245,5 +250,5 @@ func (r *CostRepository) GetCostInfoWithFilter(ctx context.Context, param GetCos return nil }) - return costInfo, err + return costInfo, totalRows, err } diff --git a/internal/core/cost/service.go b/internal/core/cost/service.go index 0522322..5a516e6 100644 --- a/internal/core/cost/service.go +++ b/internal/core/cost/service.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "math" - "strings" "sync" "time" @@ -54,19 +53,19 @@ func (c *CostService) Readyz() error { return nil } -var forecastUpdateLockMap sync.Map +var estimateCostUpdateLockMap sync.Map -func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (EstimateForecastCostResult, error) { +func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostParam) (EstimateCostResults, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() var wg sync.WaitGroup var mu sync.Mutex - var results []EsimateForecastCostSpecResult + var results []EsimateCostSpecResults var errList []error - var esimateForecastCostSpecResult EstimateForecastCostResult + var esimateCostSpecResult EstimateCostResults - utils.LogInfof("Fetching price information for spec: %+v", param) + utils.LogInfof("Fetching estimate cost info for spec: %+v", param) for _, v := range param.RecommendSpecs { wg.Add(1) @@ -74,38 +73,37 @@ func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (Est defer wg.Done() // memory lock - rl, _ := forecastUpdateLockMap.LoadOrStore(p.Hash(), &sync.Mutex{}) + rl, _ := estimateCostUpdateLockMap.LoadOrStore(p.Hash(), &sync.Mutex{}) lock := rl.(*sync.Mutex) lock.Lock() defer lock.Unlock() - priceInfos, err := c.costRepo.GetMatchingForecastCost(ctx, v, param.TimeStandard, param.PricePolicy) + estimateCostInfos, err := c.costRepo.GetMatchingEstimateCostTx(ctx, v, param.TimeStandard, param.PricePolicy) if err != nil { mu.Lock() errList = append(errList, err) mu.Unlock() - utils.LogErrorf("Error fetching price info for spec %+v: %v", v, err) + utils.LogErrorf("Error fetching estimate cost info for spec %+v: %v", v, err) return - } - if len(priceInfos) == 0 { - utils.LogInfof("No matching forecast cost found for spec: %+v, fetching from price collector", v) + if len(estimateCostInfos) == 0 { + utils.LogInfof("No matching estimate cost found for spec: %+v, fetching from price collector", v) resList, err := c.priceCollector.FetchPriceInfos(ctx, v) if err != nil { mu.Lock() - errList = append(errList, fmt.Errorf("error retrieving prices for %+v: %w", v, err)) + errList = append(errList, fmt.Errorf("error retrieving estimate cost info for %+v: %w", v, err)) mu.Unlock() return } if len(resList) > 0 { - utils.LogInfof("Inserting fetched price results for spec: %+v", v) + utils.LogInfof("Inserting fetched estimate cost info results for spec: %+v", v) - err = c.costRepo.BatchInsertAllForecastCostResult(ctx, resList) + err = c.costRepo.BatchInsertAllEstimateCostResultTx(ctx, resList) if err != nil { mu.Lock() errList = append(errList, fmt.Errorf("error batch inserting results for %+v: %w", v, err)) @@ -113,24 +111,23 @@ func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (Est return } } - priceInfos = resList + estimateCostInfos = resList } - if len(priceInfos) > 0 { + if len(estimateCostInfos) > 0 { minPrice := float64(math.MaxFloat64) maxPrice := float64(math.SmallestNonzeroFloat64) - res := EsimateForecastCostSpecResult{ - ProviderName: v.ProviderName, - RegionName: v.RegionName, - InstanceType: v.InstanceType, - ImageName: v.Image, - EstimateForecastCostSpecDetailResults: make([]EstimateForecastCostSpecDetailResult, 0), + res := EsimateCostSpecResults{ + ProviderName: v.ProviderName, + RegionName: v.RegionName, + InstanceType: v.InstanceType, + ImageName: v.Image, + EstimateCostSpecDetailResults: make([]EstimateCostSpecDetailResult, 0), } - for _, v := range priceInfos { - + for _, v := range estimateCostInfos { calculatedPrice := v.CalculatedMonthlyPrice utils.LogInfof("Price calculated for spec %+v: %f", v, calculatedPrice) @@ -141,7 +138,7 @@ func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (Est maxPrice = calculatedPrice } - specDetail := EstimateForecastCostSpecDetailResult{ + specDetail := EstimateCostSpecDetailResult{ ID: v.ID, VCpu: v.VCpu, Memory: fmt.Sprintf("%s %s", v.Memory, v.MemoryUnit), @@ -160,13 +157,13 @@ func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (Est res.SpecMinMonthlyPrice = minPrice res.SpecMaxMonthlyPrice = maxPrice - res.EstimateForecastCostSpecDetailResults = append(res.EstimateForecastCostSpecDetailResults, specDetail) + res.EstimateCostSpecDetailResults = append(res.EstimateCostSpecDetailResults, specDetail) } mu.Lock() results = append(results, res) mu.Unlock() - utils.LogInfof("Successfully calculated forecast cost for spec: %+v", param) + utils.LogInfof("Successfully calculated cost for spec: %+v", param) } }(v) @@ -174,74 +171,42 @@ func (c *CostService) EstimateForecastCost(param EstimateForecastCostParam) (Est wg.Wait() if len(errList) > 0 { - return esimateForecastCostSpecResult, fmt.Errorf("errors occurred during processing: %v", errList) + return esimateCostSpecResult, fmt.Errorf("errors occurred during processing: %v", errList) } if len(results) > 0 { - esimateForecastCostSpecResult.EsimateForecastCostSpecResults = results + esimateCostSpecResult.EsimateCostSpecResults = results for _, v := range results { - esimateForecastCostSpecResult.TotalMinMonthlyPrice += v.SpecMinMonthlyPrice - esimateForecastCostSpecResult.TotalMaxMonthlyPrice += v.SpecMaxMonthlyPrice + esimateCostSpecResult.TotalMinMonthlyPrice += v.SpecMinMonthlyPrice + esimateCostSpecResult.TotalMaxMonthlyPrice += v.SpecMaxMonthlyPrice } - utils.LogInfof("Total min monthly price: %f, Total max monthly price: %f", esimateForecastCostSpecResult.TotalMinMonthlyPrice, esimateForecastCostSpecResult.TotalMaxMonthlyPrice) + utils.LogInfof("Total min monthly price: %f, Total max monthly price: %f", esimateCostSpecResult.TotalMinMonthlyPrice, esimateCostSpecResult.TotalMaxMonthlyPrice) } - return esimateForecastCostSpecResult, nil -} - -func (c *CostService) UpdatePriceInfos(param UpdatePriceInfosParam) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer cancel() - - param.TimeStandard = time.Now().AddDate(0, 0, -7).Truncate(24 * time.Hour) - param.PricePolicy = constant.OnDemand - - count, err := c.costRepo.CountMatchingPriceInfoList(ctx, param) - if err != nil { - return err - } - - if count <= int64(0) { - resList, err := c.priceCollector.GetPriceInfos(ctx, param) - - if err != nil { - if strings.Contains(err.Error(), "you don't have any permission") { - return fmt.Errorf("you don't have permission to query the price for %s", param.ProviderName) - } - return err - } - - if len(resList) > 0 { - err := c.costRepo.BatchInsertAllForecastCostResult(ctx, resList) - if err != nil { - return err - } - } - } - return nil + return esimateCostSpecResult, nil } -func (c *CostService) GetPriceInfos(param GetPriceInfosParam) (AllPriceInfoResult, error) { +func (c *CostService) GetEstimateCost(param GetEstimateCostParam) (EstimateCostInfoResults, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() param.TimeStandard = time.Now().AddDate(0, 0, -7).Truncate(24 * time.Hour) param.PricePolicy = constant.OnDemand - var res AllPriceInfoResult + var res EstimateCostInfoResults - priceInfos, err := c.costRepo.GetAllMatchingPriceInfoList(ctx, param) + estimateCostInfos, totalCount, err := c.costRepo.GetMatchingEstimateCostInfosTx(ctx, param) if err != nil { return res, err } - priceInfoList := make([]PriceInfoResult, 0) + priceInfoList := make([]EstimateCostInfoResult, 0) - if len(priceInfos) > 0 { - for _, v := range priceInfos { - result := PriceInfoResult{ + if len(estimateCostInfos) > 0 { + for _, v := range estimateCostInfos { + result := EstimateCostInfoResult{ ID: v.ID, ProviderName: v.ProviderName, RegionName: v.RegionName, @@ -263,8 +228,8 @@ func (c *CostService) GetPriceInfos(param GetPriceInfosParam) (AllPriceInfoResul priceInfoList = append(priceInfoList, result) } - res.PriceInfoList = priceInfoList - res.ResultCount = int64(len(priceInfoList)) + res.EstimateCostInfoResult = priceInfoList + res.ResultCount = int64(totalCount) return res, nil } @@ -272,17 +237,68 @@ func (c *CostService) GetPriceInfos(param GetPriceInfosParam) (AllPriceInfoResul return res, nil } +func (c *CostService) UpdateEstimateForecastCost(param UpdateEstimateForecastCostParam) (UpdateEstimateForecastCostInfoResult, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + var updateEstimateForecastCostInfoResult UpdateEstimateForecastCostInfoResult + + r, err := c.costCollector.UpdateEstimateForecastCost(ctx, param) + if err != nil { + return updateEstimateForecastCostInfoResult, err + } + + updateEstimateForecastCostInfoResult.FetchedDataCount = int64(len(r)) + + var updatedCount int64 + var insertedCount int64 + + for _, costInfo := range r { + u, i, err := c.costRepo.UpsertCostInfo(ctx, costInfo) + if err != nil { + utils.LogErrorf("upsert error: %+v", costInfo) + } + + updatedCount += u + insertedCount += i + } + + utils.LogInfof("updated count: %d; inserted count : %d", updatedCount, insertedCount) + + updateEstimateForecastCostInfoResult.UpdatedDataCount = updatedCount + updateEstimateForecastCostInfoResult.InsertedDataCount = insertedCount + + return updateEstimateForecastCostInfoResult, nil +} + +func (c *CostService) GetEstimateForecastCostInfos(param GetEstimateForecastCostParam) (GetEstimateForecastCostInfoResults, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + res := GetEstimateForecastCostInfoResults{} + + r, totalCount, err := c.costRepo.GetEstimateForecastCostInfosTx(ctx, param) + if err != nil { + return res, err + } + + res.GetEstimateForecastCostInfoResults = r + res.ResultCount = totalCount + + return res, nil +} + var ( ErrRequestResourceEmpty = errors.New("cost request info is not enough") ErrCostResultEmpty = errors.New("cost information does not exist") ErrCostResultFormatInvalid = errors.New("cost result does not matching with interface") ) -func (c *CostService) UpdateCostInfo(param UpdateCostInfoParam) (UpdateCostInfoResult, error) { +func (c *CostService) UpdateCostInfo(param UpdateCostInfoParam) (UpdateEstimateForecastCostInfoResult, error) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - var updateCostInfoResult UpdateCostInfoResult + var updateCostInfoResult UpdateEstimateForecastCostInfoResult r, err := c.costCollector.GetCostInfos(ctx, param) if err != nil { @@ -310,13 +326,3 @@ func (c *CostService) UpdateCostInfo(param UpdateCostInfoParam) (UpdateCostInfoR return updateCostInfoResult, nil } - -func (c *CostService) GetCostInfos(param GetCostInfoParam) ([]GetCostInfoResult, error) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - r, err := c.costRepo.GetCostInfoWithFilter(ctx, param) - if err != nil { - return nil, err - } - return r, nil -} diff --git a/internal/infra/db/db.go b/internal/infra/db/db.go index b9e56c6..fec3b35 100644 --- a/internal/infra/db/db.go +++ b/internal/infra/db/db.go @@ -27,8 +27,8 @@ func migrateDB(defaultDb *gorm.DB) error { &load.LoadTestExecutionHttpInfo{}, &load.LoadTestExecutionState{}, - &cost.PriceInfo{}, - &cost.CostInfo{}, + &cost.EstimateCostInfo{}, + &cost.EstimateForecastCostInfo{}, ) if err != nil { From f664749b0b21482d8393dde6d2f07f1d77bfa609 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Wed, 30 Oct 2024 17:50:26 +0900 Subject: [PATCH 6/8] add api for forecast cost update in raw --- api/docs.go | 107 ++++++++++++++++++++++++++ api/swagger.json | 107 ++++++++++++++++++++++++++ api/swagger.yaml | 74 ++++++++++++++++++ internal/app/estimate_cost_handler.go | 24 +++--- internal/app/estimate_cost_req.go | 2 +- internal/app/router.go | 2 + internal/core/cost/cost_collector.go | 7 +- internal/core/cost/dtos.go | 3 +- internal/core/cost/repository.go | 17 ++-- internal/core/cost/service.go | 2 +- 10 files changed, 323 insertions(+), 22 deletions(-) diff --git a/api/docs.go b/api/docs.go index 7f8aff9..440775b 100644 --- a/api/docs.go +++ b/api/docs.go @@ -324,6 +324,53 @@ const docTemplate = `{ } } }, + "/api/v1/cost/estimate/forecast/raw": { + "post": { + "description": "Update and retrieve raw forecasted cost estimates for specified cost resources and additional AWS information over the past 14 days.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimate]" + ], + "summary": "Update and Retrieve Raw Estimated Forecast Cost", + "operationId": "UpdateEstimateForecastCostRaw", + "parameters": [ + { + "description": "Request body containing details for cost estimation forecast", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/app.UpdateEstimateForecastCostRawReq" + } + } + ], + "responses": { + "200": { + "description": "Successfully updated and retrieved raw estimated forecast cost information in raw data", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult" + } + }, + "400": { + "description": "Migrated resource id list is required", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Error updating or retrieving forecast cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + } + }, "/api/v1/load/generators": { "get": { "description": "Retrieve a list of all installed load generators with pagination support.", @@ -1375,6 +1422,34 @@ const docTemplate = `{ } } }, + "app.AwsAdditionalInfoReq": { + "type": "object", + "properties": { + "ownerId": { + "type": "string" + }, + "regions": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "app.CostResourceReq": { + "type": "object", + "properties": { + "resourceIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "resourceType": { + "$ref": "#/definitions/constant.ResourceType" + } + } + }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1525,6 +1600,23 @@ const docTemplate = `{ } } }, + "app.UpdateEstimateForecastCostRawReq": { + "type": "object", + "required": [ + "costResources" + ], + "properties": { + "awsAdditionalInfo": { + "$ref": "#/definitions/app.AwsAdditionalInfoReq" + }, + "costResources": { + "type": "array", + "items": { + "$ref": "#/definitions/app.CostResourceReq" + } + } + } + }, "app.UpdateEstimateForecastCostReq": { "type": "object", "properties": { @@ -1607,6 +1699,21 @@ const docTemplate = `{ "PerYear" ] }, + "constant.ResourceType": { + "type": "string", + "enum": [ + "VM", + "VNet", + "DataDisk", + "Etc" + ], + "x-enum-varnames": [ + "VM", + "VNet", + "DataDisk", + "Etc" + ] + }, "cost.EsimateCostSpecResults": { "type": "object", "properties": { diff --git a/api/swagger.json b/api/swagger.json index b3fc3d6..6da4b83 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -316,6 +316,53 @@ } } }, + "/api/v1/cost/estimate/forecast/raw": { + "post": { + "description": "Update and retrieve raw forecasted cost estimates for specified cost resources and additional AWS information over the past 14 days.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Cost Estimate]" + ], + "summary": "Update and Retrieve Raw Estimated Forecast Cost", + "operationId": "UpdateEstimateForecastCostRaw", + "parameters": [ + { + "description": "Request body containing details for cost estimation forecast", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/app.UpdateEstimateForecastCostRawReq" + } + } + ], + "responses": { + "200": { + "description": "Successfully updated and retrieved raw estimated forecast cost information in raw data", + "schema": { + "$ref": "#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult" + } + }, + "400": { + "description": "Migrated resource id list is required", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + }, + "500": { + "description": "Error updating or retrieving forecast cost information", + "schema": { + "$ref": "#/definitions/app.AntResponse-string" + } + } + } + } + }, "/api/v1/load/generators": { "get": { "description": "Retrieve a list of all installed load generators with pagination support.", @@ -1367,6 +1414,34 @@ } } }, + "app.AwsAdditionalInfoReq": { + "type": "object", + "properties": { + "ownerId": { + "type": "string" + }, + "regions": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "app.CostResourceReq": { + "type": "object", + "properties": { + "resourceIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "resourceType": { + "$ref": "#/definitions/constant.ResourceType" + } + } + }, "app.InstallLoadGeneratorReq": { "type": "object", "properties": { @@ -1517,6 +1592,23 @@ } } }, + "app.UpdateEstimateForecastCostRawReq": { + "type": "object", + "required": [ + "costResources" + ], + "properties": { + "awsAdditionalInfo": { + "$ref": "#/definitions/app.AwsAdditionalInfoReq" + }, + "costResources": { + "type": "array", + "items": { + "$ref": "#/definitions/app.CostResourceReq" + } + } + } + }, "app.UpdateEstimateForecastCostReq": { "type": "object", "properties": { @@ -1599,6 +1691,21 @@ "PerYear" ] }, + "constant.ResourceType": { + "type": "string", + "enum": [ + "VM", + "VNet", + "DataDisk", + "Etc" + ], + "x-enum-varnames": [ + "VM", + "VNet", + "DataDisk", + "Etc" + ] + }, "cost.EsimateCostSpecResults": { "type": "object", "properties": { diff --git a/api/swagger.yaml b/api/swagger.yaml index 09c4003..afbc3ec 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -193,6 +193,24 @@ definitions: successMessage: type: string type: object + app.AwsAdditionalInfoReq: + properties: + ownerId: + type: string + regions: + items: + type: string + type: array + type: object + app.CostResourceReq: + properties: + resourceIds: + items: + type: string + type: array + resourceType: + $ref: '#/definitions/constant.ResourceType' + type: object app.InstallLoadGeneratorReq: properties: installLocation: @@ -291,6 +309,17 @@ definitions: type: object type: array type: object + app.UpdateEstimateForecastCostRawReq: + properties: + awsAdditionalInfo: + $ref: '#/definitions/app.AwsAdditionalInfoReq' + costResources: + items: + $ref: '#/definitions/app.CostResourceReq' + type: array + required: + - costResources + type: object app.UpdateEstimateForecastCostReq: properties: mciId: @@ -354,6 +383,18 @@ definitions: x-enum-varnames: - PerHour - PerYear + constant.ResourceType: + enum: + - VM + - VNet + - DataDisk + - Etc + type: string + x-enum-varnames: + - VM + - VNet + - DataDisk + - Etc cost.EsimateCostSpecResults: properties: estimateForecastCostSpecDetailResults: @@ -1014,6 +1055,39 @@ paths: summary: Update and Retrieve Estimated Forecast Cost tags: - '[Cost Estimate]' + /api/v1/cost/estimate/forecast/raw: + post: + consumes: + - application/json + description: Update and retrieve raw forecasted cost estimates for specified + cost resources and additional AWS information over the past 14 days. + operationId: UpdateEstimateForecastCostRaw + parameters: + - description: Request body containing details for cost estimation forecast + in: body + name: body + required: true + schema: + $ref: '#/definitions/app.UpdateEstimateForecastCostRawReq' + produces: + - application/json + responses: + "200": + description: Successfully updated and retrieved raw estimated forecast cost + information in raw data + schema: + $ref: '#/definitions/app.AntResponse-cost_UpdateEstimateForecastCostInfoResult' + "400": + description: Migrated resource id list is required + schema: + $ref: '#/definitions/app.AntResponse-string' + "500": + description: Error updating or retrieving forecast cost information + schema: + $ref: '#/definitions/app.AntResponse-string' + summary: Update and Retrieve Raw Estimated Forecast Cost + tags: + - '[Cost Estimate]' /api/v1/load/generators: get: consumes: diff --git a/internal/app/estimate_cost_handler.go b/internal/app/estimate_cost_handler.go index 777d2fc..0b5846e 100644 --- a/internal/app/estimate_cost_handler.go +++ b/internal/app/estimate_cost_handler.go @@ -11,7 +11,6 @@ import ( "github.com/cloud-barista/cm-ant/internal/core/cost" "github.com/cloud-barista/cm-ant/internal/utils" "github.com/labstack/echo/v4" - "github.com/rs/zerolog/log" ) // @Id UpdateAndGetEstimateCost @@ -245,8 +244,6 @@ func (server *AntServer) getEstimateCost(c echo.Context) error { // @Failure 500 {object} app.AntResponse[string] "Failed to retrieve estimated forecast cost information" // @Router /api/v1/cost/estimate/forecast [get] func (s *AntServer) getEstimateForecastCost(c echo.Context) error { - - log.Info().Msgf("hello~") var req GetEstimateForecastCostReq if err := c.Bind(&req); err != nil { return errorResponseJson(http.StatusBadRequest, "Invalid request parameters") @@ -314,10 +311,19 @@ func (s *AntServer) getEstimateForecastCost(c echo.Context) error { return successResponseJson(c, "Successfully get estimate forecast cost", result) } -// --------------------------------------------------------------------------- - -func (server *AntServer) updateCostInfos(c echo.Context) error { - var req UpdateCostInfoReq +// @Id UpdateEstimateForecastCostRaw +// @Summary Update and Retrieve Raw Estimated Forecast Cost +// @Description Update and retrieve raw forecasted cost estimates for specified cost resources and additional AWS information over the past 14 days. +// @Tags [Cost Estimate] +// @Accept json +// @Produce json +// @Param body body UpdateEstimateForecastCostRawReq true "Request body containing details for cost estimation forecast" +// @Success 200 {object} app.AntResponse[cost.UpdateEstimateForecastCostInfoResult] "Successfully updated and retrieved raw estimated forecast cost information in raw data" +// @Failure 400 {object} app.AntResponse[string] "Migrated resource id list is required" +// @Failure 500 {object} app.AntResponse[string] "Error updating or retrieving forecast cost information" +// @Router /api/v1/cost/estimate/forecast/raw [post] +func (server *AntServer) updateEstimateForecastCostRaw(c echo.Context) error { + var req UpdateEstimateForecastCostRawReq if err := c.Bind(&req); err != nil { return errorResponseJson(http.StatusBadRequest, "request body binding error") @@ -338,7 +344,7 @@ func (server *AntServer) updateCostInfos(c echo.Context) error { endDate := time.Now().Truncate(24*time.Hour).AddDate(0, 0, 1) startDate := endDate.AddDate(0, 0, -14) - param := cost.UpdateCostInfoParam{ + param := cost.UpdateEstimateForecastCostRawParam{ Provider: "aws", StartDate: startDate, EndDate: endDate, @@ -349,7 +355,7 @@ func (server *AntServer) updateCostInfos(c echo.Context) error { }, } - r, err := server.services.costService.UpdateCostInfo(param) + r, err := server.services.costService.UpdateEstimateForecastCostRaw(param) if err != nil { return errorResponseJson(http.StatusInternalServerError, err.Error()) diff --git a/internal/app/estimate_cost_req.go b/internal/app/estimate_cost_req.go index 931dcb0..b8186f6 100644 --- a/internal/app/estimate_cost_req.go +++ b/internal/app/estimate_cost_req.go @@ -49,7 +49,7 @@ type GetEstimateForecastCostReq struct { // ------------------------------------------------------------------------------------------------------------------- -type UpdateCostInfoReq struct { +type UpdateEstimateForecastCostRawReq struct { CostResources []CostResourceReq `json:"costResources" validate:"required"` AwsAdditionalInfo AwsAdditionalInfoReq `json:"awsAdditionalInfo"` } diff --git a/internal/app/router.go b/internal/app/router.go index 84c7ac2..270b806 100644 --- a/internal/app/router.go +++ b/internal/app/router.go @@ -65,6 +65,8 @@ func (server *AntServer) InitRouter() error { costEstimationHandler.POST("/forecast", server.updateEstimateForecastCost) costEstimationHandler.GET("/forecast", server.getEstimateForecastCost) + + costEstimationHandler.POST("/forecast/raw", server.updateEstimateForecastCostRaw) } return nil diff --git a/internal/core/cost/cost_collector.go b/internal/core/cost/cost_collector.go index de9d01c..c74c070 100644 --- a/internal/core/cost/cost_collector.go +++ b/internal/core/cost/cost_collector.go @@ -18,7 +18,7 @@ import ( type CostCollector interface { Readyz(context.Context) error UpdateEstimateForecastCost(context.Context, UpdateEstimateForecastCostParam) (EstimateForecastCostInfos, error) - GetCostInfos(context.Context, UpdateCostInfoParam) (EstimateForecastCostInfos, error) + GetCostInfos(context.Context, UpdateEstimateForecastCostRawParam) (EstimateForecastCostInfos, error) } type AwsCostExplorerBaristaCostCollector struct { @@ -128,7 +128,7 @@ func (a *AwsCostExplorerBaristaCostCollector) generateFilterValue( return serviceValue, resourceIdValues, nil } -func (a *AwsCostExplorerBaristaCostCollector) GetCostInfos(ctx context.Context, param UpdateCostInfoParam) (EstimateForecastCostInfos, error) { +func (a *AwsCostExplorerBaristaCostCollector) GetCostInfos(ctx context.Context, param UpdateEstimateForecastCostRawParam) (EstimateForecastCostInfos, error) { if param.ConnectionName == "" { param.ConnectionName = costExplorerConnectionName @@ -285,7 +285,6 @@ func (a *AwsCostExplorerBaristaCostCollector) GetCostInfos(ctx context.Context, } costInfo := EstimateForecastCostInfo{ - // MigrationId: param.MigrationId, Provider: param.Provider, ConnectionName: param.ConnectionName, ResourceType: resourceType, @@ -340,7 +339,7 @@ func (a *AwsCostExplorerBaristaCostCollector) UpdateEstimateForecastCost(ctx con mciLabels := mci.Label _ = mciLabels[nsKey] - arg := UpdateCostInfoParam{ + arg := UpdateEstimateForecastCostRawParam{ Provider: provider, ConnectionName: costExplorerConnectionName, StartDate: param.StartDate, diff --git a/internal/core/cost/dtos.go b/internal/core/cost/dtos.go index 4f4872c..6fdfcfa 100644 --- a/internal/core/cost/dtos.go +++ b/internal/core/cost/dtos.go @@ -162,8 +162,7 @@ type GetEstimateForecastCostInfoResult struct { // ------------------------------------------------------------------- -type UpdateCostInfoParam struct { - // MigrationId string +type UpdateEstimateForecastCostRawParam struct { Provider string // currently only aws ConnectionName string StartDate time.Time diff --git a/internal/core/cost/repository.go b/internal/core/cost/repository.go index fd0831a..93f860c 100644 --- a/internal/core/cost/repository.go +++ b/internal/core/cost/repository.go @@ -2,6 +2,7 @@ package cost import ( "context" + "errors" "fmt" "strings" "time" @@ -157,8 +158,6 @@ func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo EstimateFo Granularity: costInfo.Granularity, StartDate: costInfo.StartDate, EndDate: costInfo.EndDate, - NsId: costInfo.NsId, - MciId: costInfo.MciId, }).First(&costInfo).Error if err != nil && err != gorm.ErrRecordNotFound { @@ -172,8 +171,10 @@ func (r *CostRepository) UpsertCostInfo(ctx context.Context, costInfo EstimateFo insertCount++ } else { if err := d.Model(&costInfo).Updates(map[string]interface{}{ - "cost": costInfo.Cost, - "unit": costInfo.Unit, + "cost": costInfo.Cost, + "unit": costInfo.Unit, + "ns_id": costInfo.NsId, + "mci_id": costInfo.MciId, }).Error; err != nil { return err } @@ -250,5 +251,11 @@ func (r *CostRepository) GetEstimateForecastCostInfosTx(ctx context.Context, par return nil }) - return costInfo, totalRows, err + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + return costInfo, totalRows, err + } + } + + return costInfo, totalRows, nil } diff --git a/internal/core/cost/service.go b/internal/core/cost/service.go index 5a516e6..5142854 100644 --- a/internal/core/cost/service.go +++ b/internal/core/cost/service.go @@ -294,7 +294,7 @@ var ( ErrCostResultFormatInvalid = errors.New("cost result does not matching with interface") ) -func (c *CostService) UpdateCostInfo(param UpdateCostInfoParam) (UpdateEstimateForecastCostInfoResult, error) { +func (c *CostService) UpdateEstimateForecastCostRaw(param UpdateEstimateForecastCostRawParam) (UpdateEstimateForecastCostInfoResult, error) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() From 091d8404ff34d66d457a78ed22246dcc4d0d66e8 Mon Sep 17 00:00:00 2001 From: hippo-an Date: Wed, 30 Oct 2024 18:44:37 +0900 Subject: [PATCH 7/8] update filtering for instance type when update estimate cost --- internal/core/cost/dtos.go | 18 ++-- internal/core/cost/repository.go | 38 +++++++- internal/core/cost/service.go | 147 ++++++++++++++++++++----------- 3 files changed, 139 insertions(+), 64 deletions(-) diff --git a/internal/core/cost/dtos.go b/internal/core/cost/dtos.go index 6fdfcfa..7770efd 100644 --- a/internal/core/cost/dtos.go +++ b/internal/core/cost/dtos.go @@ -35,19 +35,17 @@ func (r RecommendSpecParam) Hash() string { } type EstimateCostResults struct { - TotalMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` - TotalMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` - EsimateCostSpecResults []EsimateCostSpecResults `json:"esimateForecastCostSpecResults"` + EsimateCostSpecResults []EsimateCostSpecResults `json:"esimateCostSpecResults,omitempty"` } type EsimateCostSpecResults struct { - ProviderName string `json:"providerName"` - RegionName string `json:"regionName"` - InstanceType string `json:"instanceType"` - ImageName string `json:"imageName"` - SpecMinMonthlyPrice float64 `json:"totalMinMonthlyPrice"` - SpecMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice"` - EstimateCostSpecDetailResults []EstimateCostSpecDetailResult `json:"estimateForecastCostSpecDetailResults"` + ProviderName string `json:"providerName,omitempty"` + RegionName string `json:"regionName,omitempty"` + InstanceType string `json:"instanceType,omitempty"` + ImageName string `json:"imageName,omitempty"` + SpecMinMonthlyPrice float64 `json:"totalMinMonthlyPrice,omitempty"` + SpecMaxMonthlyPrice float64 `json:"totalMaxMonthlyPrice,omitempty"` + EstimateCostSpecDetailResults []EstimateCostSpecDetailResult `json:"estimateForecastCostSpecDetailResults,omitempty"` } type EstimateCostSpecDetailResult struct { diff --git a/internal/core/cost/repository.go b/internal/core/cost/repository.go index 93f860c..57f934f 100644 --- a/internal/core/cost/repository.go +++ b/internal/core/cost/repository.go @@ -93,21 +93,55 @@ func (r *CostRepository) GetMatchingEstimateCostInfosTx(ctx context.Context, par return priceInfoList, totalRows, err } +func (r *CostRepository) GetMatchingEstimateCostWithoutTypeTx(ctx context.Context, param RecommendSpecParam, timeStandard time.Time, pricePolicy constant.PricePolicy) (EstimateCostInfos, error) { + var priceInfos []*EstimateCostInfo + + err := r.execInTransaction(ctx, func(d *gorm.DB) error { + q := d.Model(&EstimateCostInfo{}). + Where( + "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND price_policy = ? AND last_updated_at >= ?", + strings.ToLower(param.ProviderName), + strings.ToLower(param.RegionName), + pricePolicy, + timeStandard, + ) + + if param.Image != "" { + q = q.Where("image_name = ?", strings.ToLower(param.Image)) + } + + if err := q.Find(&priceInfos).Error; err != nil { + return err + } + + return nil + }) + + if err != nil { + return nil, err + } + + return priceInfos, nil +} + func (r *CostRepository) GetMatchingEstimateCostTx(ctx context.Context, param RecommendSpecParam, timeStandard time.Time, pricePolicy constant.PricePolicy) (EstimateCostInfos, error) { var priceInfos []*EstimateCostInfo err := r.execInTransaction(ctx, func(d *gorm.DB) error { q := d.Model(&EstimateCostInfo{}). Where( - "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND instance_type = ? AND image_name = ? AND price_policy = ? AND last_updated_at >= ?", + "LOWER(provider_name) = ? AND LOWER(region_name) = ? AND instance_type = ? AND price_policy = ? AND last_updated_at >= ?", strings.ToLower(param.ProviderName), strings.ToLower(param.RegionName), strings.ToLower(param.InstanceType), - strings.ToLower(param.Image), pricePolicy, timeStandard, ) + if param.Image != "" { + q = q.Where("image_name = ?", strings.ToLower(param.Image)) + } + if err := q.Find(&priceInfos).Error; err != nil { return err } diff --git a/internal/core/cost/service.go b/internal/core/cost/service.go index 5142854..8170b72 100644 --- a/internal/core/cost/service.go +++ b/internal/core/cost/service.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" - "math" + "sort" + "strings" "sync" "time" "github.com/cloud-barista/cm-ant/internal/core/common/constant" "github.com/cloud-barista/cm-ant/internal/utils" + "github.com/rs/zerolog/log" ) type CostService struct { @@ -65,7 +67,14 @@ func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostPar var errList []error var esimateCostSpecResult EstimateCostResults - utils.LogInfof("Fetching estimate cost info for spec: %+v", param) + log.Info().Msgf("Fetching estimate cost info for spec: %+v", param) + + fail := func(msgFormat string, err error, p RecommendSpecParam) { + mu.Lock() + errList = append(errList, err) + mu.Unlock() + log.Error().Msgf(msgFormat, p, err) + } for _, v := range param.RecommendSpecs { wg.Add(1) @@ -79,57 +88,92 @@ func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostPar lock.Lock() defer lock.Unlock() - estimateCostInfos, err := c.costRepo.GetMatchingEstimateCostTx(ctx, v, param.TimeStandard, param.PricePolicy) - if err != nil { - mu.Lock() - errList = append(errList, err) - mu.Unlock() - utils.LogErrorf("Error fetching estimate cost info for spec %+v: %v", v, err) + var err error + var estimateCostInfos EstimateCostInfos + possibleFetch := true + if p.ProviderName == "ibm" || p.ProviderName == "azure" { + r, err := c.costRepo.GetMatchingEstimateCostWithoutTypeTx(ctx, v, param.TimeStandard, param.PricePolicy) + if err != nil { + fail("Error fetching estimate cost info: %v; %s", err, p) + return + } + + // ibm and azure price fetching filter is not working so if user put never exist instance type, + // it always fetch price data from collector. + // check and matching with database values + if len(r) != 0 { + temp := make([]*EstimateCostInfo, 0) + + for i := range r { + val := r[i] + + if val == nil { + log.Warn().Msg("estimate cost info value is nil") + continue + } + + if strings.EqualFold(v.InstanceType, p.InstanceType) { + temp = append(temp, val) + } + } + + if len(temp) > 0 { + estimateCostInfos = temp + } else { + possibleFetch = false + } + } + } else { + estimateCostInfos, err = c.costRepo.GetMatchingEstimateCostTx(ctx, p, param.TimeStandard, param.PricePolicy) + } + + if err != nil { + fail("Error fetching estimate cost info for spec: %v; %s", err, p) return } - if len(estimateCostInfos) == 0 { - utils.LogInfof("No matching estimate cost found for spec: %+v, fetching from price collector", v) + if len(estimateCostInfos) == 0 || possibleFetch { + log.Info().Msgf("No matching estimate cost found from database for spec: %+v, fetching from price collector", p) - resList, err := c.priceCollector.FetchPriceInfos(ctx, v) + resList, err := c.priceCollector.FetchPriceInfos(ctx, p) if err != nil { - mu.Lock() - errList = append(errList, fmt.Errorf("error retrieving estimate cost info for %+v: %w", v, err)) - mu.Unlock() + fail("Error retrieving estimate cost info spec: %v; %s", fmt.Errorf("error retrieving estimate cost info for %+v: %w", p, err), p) return } if len(resList) > 0 { - utils.LogInfof("Inserting fetched estimate cost info results for spec: %+v", v) + log.Info().Msgf("Inserting fetched estimate cost info results for spec: %+v", p) err = c.costRepo.BatchInsertAllEstimateCostResultTx(ctx, resList) if err != nil { - mu.Lock() - errList = append(errList, fmt.Errorf("error batch inserting results for %+v: %w", v, err)) - mu.Unlock() + fail("Error batch inserting estimate cost info spec: %v; %s", fmt.Errorf("error batch inserting results for %+v: %w", p, err), p) return } } estimateCostInfos = resList } - if len(estimateCostInfos) > 0 { + res := EsimateCostSpecResults{ + ProviderName: p.ProviderName, + RegionName: p.RegionName, + InstanceType: p.InstanceType, + ImageName: p.Image, + EstimateCostSpecDetailResults: make([]EstimateCostSpecDetailResult, 0), + } - minPrice := float64(math.MaxFloat64) - maxPrice := float64(math.SmallestNonzeroFloat64) + if len(estimateCostInfos) > 0 { + minPrice := estimateCostInfos[0].CalculatedMonthlyPrice + maxPrice := estimateCostInfos[0].CalculatedMonthlyPrice - res := EsimateCostSpecResults{ - ProviderName: v.ProviderName, - RegionName: v.RegionName, - InstanceType: v.InstanceType, - ImageName: v.Image, - EstimateCostSpecDetailResults: make([]EstimateCostSpecDetailResult, 0), - } + for _, va := range estimateCostInfos { - for _, v := range estimateCostInfos { - calculatedPrice := v.CalculatedMonthlyPrice - utils.LogInfof("Price calculated for spec %+v: %f", v, calculatedPrice) + if !strings.EqualFold(v.InstanceType, p.InstanceType) { + log.Warn().Msgf("%s instance type is not matching with provided condition %s", va.InstanceType, p.InstanceType) + continue + } + calculatedPrice := va.CalculatedMonthlyPrice + log.Info().Msgf("Price calculated for spec %+v: %f", p, calculatedPrice) if calculatedPrice < minPrice { minPrice = calculatedPrice @@ -139,20 +183,20 @@ func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostPar } specDetail := EstimateCostSpecDetailResult{ - ID: v.ID, - VCpu: v.VCpu, - Memory: fmt.Sprintf("%s %s", v.Memory, v.MemoryUnit), - Storage: v.Storage, - OsType: v.OsType, - ProductDescription: v.ProductDescription, - OriginalPricePolicy: v.OriginalPricePolicy, - PricePolicy: v.PricePolicy, - Unit: v.Unit, - Currency: v.Currency, - Price: v.Price, + ID: va.ID, + VCpu: va.VCpu, + Memory: fmt.Sprintf("%s %s", va.Memory, va.MemoryUnit), + Storage: va.Storage, + OsType: va.OsType, + ProductDescription: va.ProductDescription, + OriginalPricePolicy: va.OriginalPricePolicy, + PricePolicy: va.PricePolicy, + Unit: va.Unit, + Currency: va.Currency, + Price: va.Price, CalculatedMonthlyPrice: calculatedPrice, - PriceDescription: v.PriceDescription, - LastUpdatedAt: v.LastUpdatedAt, + PriceDescription: va.PriceDescription, + LastUpdatedAt: va.LastUpdatedAt, } res.SpecMinMonthlyPrice = minPrice @@ -160,10 +204,16 @@ func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostPar res.EstimateCostSpecDetailResults = append(res.EstimateCostSpecDetailResults, specDetail) } + if len(res.EstimateCostSpecDetailResults) > 0 { + sort.Slice(res.EstimateCostSpecDetailResults, func(i, j int) bool { + return res.EstimateCostSpecDetailResults[i].CalculatedMonthlyPrice < res.EstimateCostSpecDetailResults[j].CalculatedMonthlyPrice + }) + } + mu.Lock() results = append(results, res) mu.Unlock() - utils.LogInfof("Successfully calculated cost for spec: %+v", param) + log.Info().Msgf("Successfully calculated cost for spec: %+v", param) } }(v) @@ -176,13 +226,6 @@ func (c *CostService) UpdateAndGetEstimateCost(param UpdateAndGetEstimateCostPar if len(results) > 0 { esimateCostSpecResult.EsimateCostSpecResults = results - - for _, v := range results { - esimateCostSpecResult.TotalMinMonthlyPrice += v.SpecMinMonthlyPrice - esimateCostSpecResult.TotalMaxMonthlyPrice += v.SpecMaxMonthlyPrice - } - utils.LogInfof("Total min monthly price: %f, Total max monthly price: %f", esimateCostSpecResult.TotalMinMonthlyPrice, esimateCostSpecResult.TotalMaxMonthlyPrice) - } return esimateCostSpecResult, nil From 6358a9bf20f3fe73e2903ff26dbdfbb14bccbe9b Mon Sep 17 00:00:00 2001 From: hippo-an Date: Wed, 30 Oct 2024 18:47:02 +0900 Subject: [PATCH 8/8] update swagger for estimate cost --- api/docs.go | 8 +------- api/swagger.json | 8 +------- api/swagger.yaml | 6 +----- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/api/docs.go b/api/docs.go index 440775b..0376d82 100644 --- a/api/docs.go +++ b/api/docs.go @@ -1816,17 +1816,11 @@ const docTemplate = `{ "cost.EstimateCostResults": { "type": "object", "properties": { - "esimateForecastCostSpecResults": { + "esimateCostSpecResults": { "type": "array", "items": { "$ref": "#/definitions/cost.EsimateCostSpecResults" } - }, - "totalMaxMonthlyPrice": { - "type": "number" - }, - "totalMinMonthlyPrice": { - "type": "number" } } }, diff --git a/api/swagger.json b/api/swagger.json index 6da4b83..601693c 100644 --- a/api/swagger.json +++ b/api/swagger.json @@ -1808,17 +1808,11 @@ "cost.EstimateCostResults": { "type": "object", "properties": { - "esimateForecastCostSpecResults": { + "esimateCostSpecResults": { "type": "array", "items": { "$ref": "#/definitions/cost.EsimateCostSpecResults" } - }, - "totalMaxMonthlyPrice": { - "type": "number" - }, - "totalMinMonthlyPrice": { - "type": "number" } } }, diff --git a/api/swagger.yaml b/api/swagger.yaml index afbc3ec..83317a5 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -462,14 +462,10 @@ definitions: type: object cost.EstimateCostResults: properties: - esimateForecastCostSpecResults: + esimateCostSpecResults: items: $ref: '#/definitions/cost.EsimateCostSpecResults' type: array - totalMaxMonthlyPrice: - type: number - totalMinMonthlyPrice: - type: number type: object cost.EstimateCostSpecDetailResult: properties: