diff --git a/README.md b/README.md index d6b5822..65aeee3 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ This API provides simple restful API access to Amazon's Cost explorer and cloudw ## Endpoints -``` +```text GET /v1/cost/ping GET /v1/cost/version GET /v1/cost/metrics GET /v1/cost/{account}/spaces/{spaceid}[?start=2019-10-01&end=2019-10-30][&groupBy=SERVICE] -GET /v1/cost/{account}/spaces/{spaceid}/{resourcename}[?start=2019-10-01&end=2019-10-30] POST /v1/cost/{account}/spaces/{spaceid}/budgets GET /v1/cost/{account}/spaces/{spaceid}/budgets @@ -18,11 +17,6 @@ DELETE /v1/cost/{account}/spaces/{spaceid}/budgets/{budget} GET /v1/cost/{account}/spaces/{space}/instances/{id}/optimizer -### DEPRECATED ### -GET /v1/cost/{account}/instances/{id}/metrics/graph.png?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] -GET /v1/cost/{account}/instances/{id}/metrics/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] -################## - GET /v1/inventory/{account}/spaces/{spaceid} GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] @@ -40,9 +34,7 @@ different dimensions is supported by passing query parameters. #### Request month to date costs for a space -``` GET /v1/cost/{account}/spaces/{spaceid} -``` #### Response @@ -75,9 +67,7 @@ GET /v1/cost/{account}/spaces/{spaceid} #### Request costs for a space for a date range -``` GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31 -``` #### Response @@ -132,9 +122,7 @@ GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31 #### Request costs for a space by date range and grouped by a dimension -``` GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31&groupby=INSTANCE_TYPE_FAMILY -``` Supported default 'groupby' values are AZ, INSTANCE_TYPE, LINKED_ACCOUNT, OPERATION, PURCHASE_TYPE, SERVICE, USAGE_TYPE, PLATFORM, TENANCY, RECORD_TYPE, LEGAL_ENTITY_NAME, DEPLOYMENT_OPTION, DATABASE_ENGINE, CACHE_ENGINE, INSTANCE_TYPE_FAMILY, REGION, BILLING_ENTITY, RESERVATION_ID, SAVINGS_PLANS_TYPE, SAVINGS_PLAN_ARN, OPERATING_SYSTEM. In addition, the custom RESOURCE_NAME 'groupby' is supported using the Name tag. @@ -317,54 +305,6 @@ Supported default 'groupby' values are AZ, INSTANCE_TYPE, LINKED_ACCOUNT, OPERAT ] ``` -### Get the cost and usage for a resource (name) within a space ID - -By default, this will get the month to date costs for a resource name with a space id - -``` -GET /v1/cost/{account}/spaces/{spaceid}/{resourcename} -``` - -### How it works - -Costs are Filtered - the keys/values are resource tags - -```json -{ - "Filter": { - "And": [ - { - "Tags": { - "Key": "Name", - "Values": ["spinup-000cba.spinup.yale.edu"] - } - }, - { - "Tags": { - "Key": "spinup:spaceid", - "Values": ["spinup-0002a2"] - } - }, - { - "Or": [ - { - "Tags": { - "Key": "yale:org", - "Values": ["ss"] - } - }, - { - "Tags": { - "Key": "spinup:org", - "Values": ["ss"] - } - }] - } - ] - } -} -``` - ## Budget Usage ### Create Budgets Alerts @@ -733,28 +673,31 @@ GET /v1/inventory/{account}/spaces/{spaceid} ## Metrics Usage -### Get cloudwatch metrics widgets URL from S3 for an instance ID +### Get Cloudwatch metrics widgets URL from S3 for an instance ID This will get the passed metric(s) for the passed instance ID or container cluster/service in a `image/png` graph for the past 1 day by default, cache it in S3 and return the URL. URLs are cached in the API for 5 minutes, the images should be purged from the S3 cache on a schedule. It's also possible to pass the height, width, start time, end time and period (e. `300s` for 300 seconds, `5m` for 5 minutes). Query parameters must follow the [CloudWatch Metric Widget Structure](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html). -### Documentation on cloudwatch metrics +### Documentation on Cloudwatch metrics + +#### Get a list of metrics per AWS service +```bash +aws --region us-east-1 cloudwatch list-metrics --namespace AWS/RDS |grep MetricName |sort| uniq ``` -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html -Get you a list of metrics per AWS service -$ aws --region us-east-1 cloudwatch list-metrics --namespace AWS/RDS |grep MetricName |sort| uniq +#### Helpful references -GetMetricWidget gets a metric widget image for an instance id -https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html -https://docs.aws.amazon.com/AmazonECS/latest/developerguide/cloudwatch-metrics.html -https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/MonitoringOverview.html +* [Viewing Metrics with CloudWatch](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html) +* [CloudWatch Metrics Widget Structure](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html) +* [CloudWatch Metrics Developer Guide](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/cloudwatch-metrics.html) +* [AWS Aurora Monitoring Overview](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/MonitoringOverview.html) -Example metrics request +#### Example metrics request + +```json { "metrics": [ [ "AWS/ECS", "CPUUtilization", "ClusterName", "spinup-000393", "ServiceName", "spinup-0010a3-testsvc" ] @@ -768,7 +711,7 @@ Example metrics request #### Request -``` +```text GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&....] GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&start={start}&end={end}&period={period}] diff --git a/api/handlers_budgets.go b/api/handlers_budgets.go index 08e6c5e..910c453 100644 --- a/api/handlers_budgets.go +++ b/api/handlers_budgets.go @@ -15,7 +15,7 @@ import ( func (s *server) SpaceBudgetsCreatehandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) @@ -71,7 +71,7 @@ func (s *server) SpaceBudgetsCreatehandler(w http.ResponseWriter, r *http.Reques func (s *server) SpaceBudgetsListHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) @@ -120,7 +120,7 @@ func (s *server) SpaceBudgetsListHandler(w http.ResponseWriter, r *http.Request) func (s *server) SpaceBudgetsShowHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] budget := vars["budget"] @@ -170,7 +170,7 @@ func (s *server) SpaceBudgetsShowHandler(w http.ResponseWriter, r *http.Request) func (s *server) SpaceBudgetsDeleteHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] budget := vars["budget"] diff --git a/api/handlers_inventory.go b/api/handlers_inventory.go index 46b4601..9730a1c 100644 --- a/api/handlers_inventory.go +++ b/api/handlers_inventory.go @@ -16,7 +16,7 @@ import ( func (s *server) SpaceInventoryGetHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) diff --git a/api/handlers_metrics.go b/api/handlers_metrics.go index 02e4dc4..13471da 100644 --- a/api/handlers_metrics.go +++ b/api/handlers_metrics.go @@ -18,24 +18,29 @@ import ( func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) instanceId := vars["id"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -50,9 +55,9 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, instanceId, strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, instanceId, strings.Join(metrics, "-"), req.String()) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -85,7 +90,7 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -96,25 +101,30 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) cluster := vars["cluster"] service := vars["service"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -129,11 +139,11 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, fmt.Sprintf("%s-%s", cluster, service), strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, fmt.Sprintf("%s-%s", cluster, service), strings.Join(metrics, "-"), req.String()) log.Debugf("object key: %s", key) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -166,7 +176,7 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -177,25 +187,30 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) bucketName := vars["bucket"] metric := vars["metric"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) // only support NumberOfObjects and BucketSizeBytes var storageType string @@ -220,11 +235,11 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) }, } - key := fmt.Sprintf("%s/%s/%s%s", s.org, bucketName, metric, req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, bucketName, metric, req.String()) log.Debugf("object key: %s", key) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -251,7 +266,7 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -263,25 +278,30 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) queryType := vars["type"] instanceId := vars["id"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -296,9 +316,9 @@ func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, instanceId, strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, instanceId, strings.Join(metrics, "-"), req.String()) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -340,7 +360,7 @@ func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/api/handlers_optimizer.go b/api/handlers_optimizer.go index 19d1a9b..25bf65f 100644 --- a/api/handlers_optimizer.go +++ b/api/handlers_optimizer.go @@ -15,7 +15,7 @@ import ( func (s *server) SpaceInstanceOptimizer(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) instanceID := vars["id"] var j []byte diff --git a/api/handlers_spaces.go b/api/handlers_spaces.go index 8346704..d5d4400 100644 --- a/api/handlers_spaces.go +++ b/api/handlers_spaces.go @@ -4,15 +4,10 @@ import ( "encoding/json" "fmt" "net/http" - "time" "github.com/YaleSpinup/apierror" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/costexplorer" "github.com/gorilla/mux" - "github.com/pkg/errors" - ce "github.com/YaleSpinup/cost-api/costexplorer" log "github.com/sirupsen/logrus" ) @@ -20,224 +15,43 @@ import ( // it pulls data from the start of the month until now. func (s *server) SpaceGetHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} - - // loop thru and log given API input vars in debug vars := mux.Vars(r) - for k, v := range vars { - log.Debugf("key=%v, value=%v", k, v) - } - - // get vars from the API route - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) startTime := vars["start"] endTime := vars["end"] spaceID := vars["space"] groupBy := vars["groupby"] - ceService, ok := s.costExplorerServices[account] - if !ok { - msg := fmt.Sprintf("cost explorer service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) - return - } - log.Debugf("found cost explorer service %+v", ceService) - - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + policy, err := costExplorerReadPolicy() + if err != nil { + handleError(w, apierror.New(apierror.ErrInternalError, "failed to generate policy", err)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) - // vars for checking input times parse and are valid - var start string - var end string - var err error + orch, err := s.newCostExplorerOrchestrator(r.Context(), &sessionParams{ + inlinePolicy: policy, + role: role, + }) - // Did we get cost-explorer start and end times on the API? - // set defaults, else verify times given on API - if endTime == "" || startTime == "" { - log.Debug("no start or end time given on API input, assigning defaults") - start, end = getTimeDefault() - } else { - start, end, err = getTimeAPI(startTime, endTime) - if err != nil { - handleError(w, err) - return - } - } - - input := costexplorer.GetCostAndUsageInput{ - Filter: ce.And(inSpace(spaceID), inOrg(s.org), notTryIT()), - Granularity: aws.String("MONTHLY"), - Metrics: []*string{ - aws.String("BLENDED_COST"), - aws.String("UNBLENDED_COST"), - aws.String("USAGE_QUANTITY"), - }, - TimePeriod: &costexplorer.DateInterval{ - Start: aws.String(start), - End: aws.String(end), + out, cached, expire, err := orch.getCostAndUsageForSpace( + r.Context(), + &costAndUsageReq{ + account: account, + spaceID: spaceID, + start: startTime, + end: endTime, + groupBy: groupBy, }, - } - - switch { - case groupBy == "RESOURCE_NAME": - input.GroupBy = []*costexplorer.GroupDefinition{ - { - Key: aws.String("Name"), - Type: aws.String("TAG"), - }, - } - case groupBy != "": - input.GroupBy = []*costexplorer.GroupDefinition{ - { - Key: aws.String(groupBy), - Type: aws.String("DIMENSION"), - }, - } - } - - // create a cacheKey more unique than spaceID for managing cache objects. - // Since we will accept date-range cost exploring and grouping, concatenate - // the spaceID, the start time, end time and group by so we can cache each - // time-based result - cacheKey := fmt.Sprintf("%s_%s_%s_%s", spaceID, startTime, endTime, groupBy) - log.Debugf("cacheKey: %s", cacheKey) - - // the object is not found in the cache, call AWS cost-explorer and set cache - var out []*costexplorer.ResultByTime - c, expire, ok := resultCache.GetWithExpiration(cacheKey) - if !ok || c == nil { - log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", s.org, cacheKey) - // call cost-explorer - var err error - out, err = ceService.GetCostAndUsage(r.Context(), &input) - if err != nil { - msg := fmt.Sprintf("failed to get costs for space %s: %s", cacheKey, err.Error()) - handleError(w, errors.Wrap(err, msg)) - return - } - - // cache results - resultCache.SetDefault(cacheKey, out) - } else { - // cached object was found - out = c.([]*costexplorer.ResultByTime) - log.Debugf("found cached object: %s", out) - w.Header().Set("X-Cache-Hit", "true") - w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", time.Until(expire).Seconds())) - } - - j, err := json.Marshal(out) + ) if err != nil { - log.Errorf("cannot marshal response (%v) into JSON: %s", out, err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(j) - -} - -// SpaceResourceGetHandler gets the cost for a resource within a space -func (s *server) SpaceResourceGetHandler(w http.ResponseWriter, r *http.Request) { - w = LogWriter{w} - - log.Debugf("ResourceGetHandler Called") - - // loop thru and log given API input vars in debug - vars := mux.Vars(r) - for k, v := range vars { - log.Debugf("key=%v, value=%v", k, v) - } - - // get vars from the API route - account := vars["account"] - startTime := vars["start"] - endTime := vars["end"] - spaceID := vars["space"] - name := vars["resourcename"] - - ceService, ok := s.costExplorerServices[account] - if !ok { - msg := fmt.Sprintf("cost explorer service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) - return - } - log.Debugf("found cost explorer service %+v", ceService) - - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + handleError(w, err) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) - - // vars for checking input times parse and are valid - var start string - var end string - var err error - // Did we get cost-explorer start and end times on the API? - // set defaults, else verify times given on API - if endTime == "" || startTime == "" { - log.Debug("no start or end time given on API input, assigning defaults") - start, end = getTimeDefault() - } else { - start, end, err = getTimeAPI(startTime, endTime) - if err != nil { - handleError(w, err) - return - } - } - - input := costexplorer.GetCostAndUsageInput{ - Filter: ce.And(ofName(name), inSpace(spaceID), inOrg(s.org), notTryIT()), - Granularity: aws.String("MONTHLY"), - Metrics: []*string{ - aws.String("BLENDED_COST"), - aws.String("UNBLENDED_COST"), - aws.String("USAGE_QUANTITY"), - }, - TimePeriod: &costexplorer.DateInterval{ - Start: aws.String(start), - End: aws.String(end), - }, - } - - // create a cacheKey more unique than spaceID for managing cache objects. - // Since we will accept date-range cost exploring, concatenate the spaceID - // and the start and end time so we can cache each time-based result - cacheKey := fmt.Sprintf("%s_%s_%s_%s", spaceID, name, startTime, endTime) - log.Debugf("cacheKey: %s", cacheKey) - - // the object is not found in the cache, call AWS cost-explorer and set cache - var out []*costexplorer.ResultByTime - c, expire, ok := resultCache.GetWithExpiration(cacheKey) - if !ok || c == nil { - log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", s.org, cacheKey) - // call cost-explorer - var err error - out, err = ceService.GetCostAndUsage(r.Context(), &input) - if err != nil { - msg := fmt.Sprintf("failed to get costs for space %s: %s", cacheKey, err.Error()) - handleError(w, errors.Wrap(err, msg)) - return - } - - // cache results - resultCache.SetDefault(cacheKey, out) - } else { - // cached object was found - out = c.([]*costexplorer.ResultByTime) - log.Debugf("found cached object: %s", out) - w.Header().Set("X-Cache-Hit", "true") - w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", time.Until(expire).Seconds())) + w.Header().Set("X-Cache-Hit", fmt.Sprintf("%t", cached)) + if cached { + w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", expire.Seconds())) } j, err := json.Marshal(out) @@ -252,63 +66,3 @@ func (s *server) SpaceResourceGetHandler(w http.ResponseWriter, r *http.Request) w.Write(j) } - -// getTimeDefault returns time range from beginning of month to day-of-month now -func getTimeDefault() (string, string) { - // if it's the first day of the month, get today's usage thus far - y, m, d := time.Now().Date() - if d == 1 { - d = 3 - } - return fmt.Sprintf("%d-%02d-01", y, m), fmt.Sprintf("%d-%02d-%02d", y, m, d) -} - -// getTimeAPI returns time parsed from API input -func getTimeAPI(startTime, endTime string) (string, string, error) { - log.Debugf("startTime: %s, endTime: %s ", startTime, endTime) - - // sTmp and eTmp temporary vars to hold time.Time objects - sTmp, err := time.Parse("2006-01-02", startTime) - if err != nil { - return "", "", errors.Wrapf(err, "error parsing StartTime from input") - } - - eTmp, err := time.Parse("2006-01-02", endTime) - if err != nil { - return "", "", errors.Wrapf(err, "error parsing EndTime from input") - } - - // if time on the API input is already borked, don't continue - // end time is greater than start time, logically - timeValidity := eTmp.After(sTmp) - if !timeValidity { - return "", "", errors.Errorf("endTime should be greater than startTime") - } - - // convert time.Time to a string - return sTmp.Format("2006-01-02"), eTmp.Format("2006-01-02"), nil -} - -// inSpace returns the cost explorer expression to filter on spaceid -func inSpace(spaceID string) *costexplorer.Expression { - return ce.Tag("spinup:spaceid", []string{spaceID}) -} - -// ofName returns the cost explorer expression to filter on name -func ofName(name string) *costexplorer.Expression { - return ce.Tag("Name", []string{name}) -} - -// inOrg returns the cost explorer expression to filter on org -func inOrg(org string) *costexplorer.Expression { - yaleTag := ce.Tag("yale:org", []string{org}) - spinupTag := ce.Tag("spinup:org", []string{org}) - return ce.Or(yaleTag, spinupTag) -} - -// notTryIT returns the cost explorer expression to filter out tryits -func notTryIT() *costexplorer.Expression { - yaleTag := ce.Tag("yale:subsidized", []string{"true"}) - spinupTag := ce.Tag("spinup:subsidized", []string{"true"}) - return ce.Not(ce.Or(yaleTag, spinupTag)) -} diff --git a/api/handlers_spaces_test.go b/api/handlers_spaces_test.go index 8fc81e4..588d2c9 100644 --- a/api/handlers_spaces_test.go +++ b/api/handlers_spaces_test.go @@ -11,9 +11,12 @@ import ( "github.com/aws/aws-sdk-go/service/costexplorer" ) -func TestGetTimeDefault(t *testing.T) { +func TestParseTime(t *testing.T) { // use defaults derived in code - startTime, endTime := getTimeDefault() + startResult, endResult, err := parseTime("", "") + if err != nil { + t.Errorf("unexpected error from parseTime: %s", err) + } // tests should match defaults from getTimeDefault y, m, d := time.Now().Date() @@ -24,13 +27,13 @@ func TestGetTimeDefault(t *testing.T) { sTime := fmt.Sprintf("%d-%02d-01", y, m) eTime := fmt.Sprintf("%d-%02d-%02d", y, m, d) - if startTime == sTime { + if startResult == sTime { t.Logf("got expected default sTime: %s\n", sTime) } else { t.Errorf("got unexpected sTime: %s\n", sTime) } - if endTime == eTime { + if endResult == eTime { t.Logf("got expected default eTime: %s\n", eTime) } else { t.Errorf("got unexpected eTime: %s\n", eTime) @@ -40,24 +43,21 @@ func TestGetTimeDefault(t *testing.T) { sTime = "2006-01-02" eTime = "2006-13-40" - if startTime != sTime { - t.Logf("negative test sTime: %s does not match: %s", sTime, startTime) + if startResult != sTime { + t.Logf("negative test sTime: %s does not match: %s", sTime, startResult) } else { t.Errorf("got unexpected sTime: %s\n", sTime) } - if endTime != eTime { - t.Logf("negative test eTime: %s does not match: %s", eTime, endTime) + if endResult != eTime { + t.Logf("negative test eTime: %s does not match: %s", eTime, endResult) } else { t.Errorf("got unexpected eTime: %s\n", eTime) } -} - -func TestGetTimeAPI(t *testing.T) { startTime := "2019-11-01" endTime := "2019-11-30" - startResult, endResult, err := getTimeAPI(startTime, endTime) + startResult, endResult, err = parseTime(startTime, endTime) if err != nil { t.Errorf("got unexpected error: %s", err) } @@ -70,10 +70,10 @@ func TestGetTimeAPI(t *testing.T) { // negative tests for non-matching API inputs from getTimeDefault // bad start time fails - sTime := "2006-01-022" - eTime := "2006-12-02" + startTime = "2006-01-022" + endTime = "2006-12-02" - neg00startResult, neg00endResult, err := getTimeAPI(sTime, eTime) + neg00startResult, neg00endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error: %s", err) } @@ -85,10 +85,10 @@ func TestGetTimeAPI(t *testing.T) { } // bad end time fails - sTime = "2006-01-02" - eTime = "2006-12-403" + startTime = "2006-01-02" + endTime = "2006-12-403" - neg01startResult, neg01endResult, err := getTimeAPI(sTime, eTime) + neg01startResult, neg01endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error: %s", err) } @@ -100,10 +100,10 @@ func TestGetTimeAPI(t *testing.T) { } // start after end fails - sTime = "2006-01-30" - eTime = "2006-01-01" + startTime = "2006-01-30" + endTime = "2006-01-01" - neg02startResult, neg02endResult, err := getTimeAPI(sTime, eTime) + neg02startResult, neg02endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error for start after end : %s", err) } @@ -113,6 +113,7 @@ func TestGetTimeAPI(t *testing.T) { if neg02endResult == endTime { t.Logf("negative test got expected neg02endResult from getTimeAPI: %s", neg02endResult) } + } func TestInSpace(t *testing.T) { diff --git a/api/orchestration_costexplorer.go b/api/orchestration_costexplorer.go new file mode 100644 index 0000000..bee5247 --- /dev/null +++ b/api/orchestration_costexplorer.go @@ -0,0 +1,150 @@ +package api + +import ( + "context" + "fmt" + "time" + + ce "github.com/YaleSpinup/cost-api/costexplorer" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/costexplorer" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type costAndUsageReq struct { + account, spaceID, start, end, groupBy string +} + +func (o *costExplorerOrchestrator) getCostAndUsageForSpace(ctx context.Context, req *costAndUsageReq) ([]*costexplorer.ResultByTime, bool, time.Duration, error) { + start, end, err := parseTime(req.start, req.end) + if err != nil { + return nil, false, 0, nil + } + + input := costexplorer.GetCostAndUsageInput{ + Filter: ce.And(inSpace(req.spaceID), inOrg(o.server.org), notTryIT()), + Granularity: aws.String("MONTHLY"), + Metrics: []*string{ + aws.String("BLENDED_COST"), + aws.String("UNBLENDED_COST"), + aws.String("USAGE_QUANTITY"), + }, + TimePeriod: &costexplorer.DateInterval{ + Start: aws.String(start), + End: aws.String(end), + }, + } + + switch { + case req.groupBy == "RESOURCE_NAME": + input.GroupBy = []*costexplorer.GroupDefinition{ + { + Key: aws.String("Name"), + Type: aws.String("TAG"), + }, + } + case req.groupBy != "": + input.GroupBy = []*costexplorer.GroupDefinition{ + { + Key: aws.String(req.groupBy), + Type: aws.String("DIMENSION"), + }, + } + } + + // create a cacheKey more unique than spaceID for managing cache objects. + // Since we will accept date-range cost exploring and grouping, concatenate + // the spaceID, the start time, end time and group by so we can cache each + // time-based result + cacheKey := fmt.Sprintf("%s_%s_%s_%s_%s", req.account, req.spaceID, req.start, req.end, req.groupBy) + + log.Debugf("cacheKey: %s", cacheKey) + + // the object is not found in the cache, call AWS cost-explorer and set cache + c, expire, ok := o.server.resultCache.GetWithExpiration(cacheKey) + if !ok || c == nil { + log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", o.server.org, cacheKey) + + // call cost-explorer + out, err := o.client.GetCostAndUsage(ctx, &input) + if err != nil { + return nil, false, 0, err + } + + // cache results + o.server.resultCache.SetDefault(cacheKey, out) + + return out, false, 0, nil + } + + // cached object was found + out, ok := c.([]*costexplorer.ResultByTime) + if !ok { + return nil, false, 0, errors.New("value in cache is not a []*costexplorer.ResultByTime!") + } + + log.Debugf("found cached object: %s", out) + + return out, true, time.Until(expire), nil +} + +// parseTime returns time range from beginning of month to day-of-month now if the +// passed values are empty otherwise, it parses the string and returns the value (or an error) +func parseTime(start, end string) (string, string, error) { + // if it's the first day of the month, get today's usage thus far + // TODO: :confused: + y, m, d := time.Now().Date() + if d == 1 { + d = 3 + } + + if start == "" { + start = fmt.Sprintf("%d-%02d-01", y, m) + } + + startStamp, err := time.Parse("2006-01-02", start) + if err != nil { + return "", "", err + } + + if end == "" { + end = fmt.Sprintf("%d-%02d-%02d", y, m, d) + } + + endStamp, err := time.Parse("2006-01-02", end) + if err != nil { + return "", "", err + } + + if !endStamp.After(startStamp) { + return "", "", fmt.Errorf("end time should be after start time") + } + + // convert time.Time to a string + return startStamp.Format("2006-01-02"), endStamp.Format("2006-01-02"), nil +} + +// inSpace returns the cost explorer expression to filter on spaceid +func inSpace(spaceID string) *costexplorer.Expression { + return ce.Tag("spinup:spaceid", []string{spaceID}) +} + +// ofName returns the cost explorer expression to filter on name +func ofName(name string) *costexplorer.Expression { + return ce.Tag("Name", []string{name}) +} + +// inOrg returns the cost explorer expression to filter on org +func inOrg(org string) *costexplorer.Expression { + yaleTag := ce.Tag("yale:org", []string{org}) + spinupTag := ce.Tag("spinup:org", []string{org}) + return ce.Or(yaleTag, spinupTag) +} + +// notTryIT returns the cost explorer expression to filter out tryits +func notTryIT() *costexplorer.Expression { + yaleTag := ce.Tag("yale:subsidized", []string{"true"}) + spinupTag := ce.Tag("spinup:subsidized", []string{"true"}) + return ce.Not(ce.Or(yaleTag, spinupTag)) +} diff --git a/api/orchestrators.go b/api/orchestrators.go index 097947f..4fff285 100644 --- a/api/orchestrators.go +++ b/api/orchestrators.go @@ -1,18 +1,43 @@ package api import ( + "context" + "github.com/YaleSpinup/cost-api/budgets" "github.com/YaleSpinup/cost-api/computeoptimizer" + "github.com/YaleSpinup/cost-api/costexplorer" "github.com/YaleSpinup/cost-api/resourcegroupstaggingapi" "github.com/YaleSpinup/cost-api/sns" + log "github.com/sirupsen/logrus" ) +type sessionParams struct { + role string + inlinePolicy string + policyArns []string +} + type budgetsOrchestrator struct { client *budgets.Budgets snsClient *sns.SNS org string } +type costExplorerOrchestrator struct { + client *costexplorer.CostExplorer + server *server +} + +type optimizerOrchestrator struct { + client *computeoptimizer.ComputeOptimizer + org string +} + +type inventoryOrchestrator struct { + client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + org string +} + func newBudgetsOrchestrator(budgetsClient *budgets.Budgets, snsClient *sns.SNS, org string) *budgetsOrchestrator { return &budgetsOrchestrator{ client: budgetsClient, @@ -21,11 +46,6 @@ func newBudgetsOrchestrator(budgetsClient *budgets.Budgets, snsClient *sns.SNS, } } -type optimizerOrchestrator struct { - client *computeoptimizer.ComputeOptimizer - org string -} - func newOptimizerOrchestrator(client *computeoptimizer.ComputeOptimizer, org string) *optimizerOrchestrator { return &optimizerOrchestrator{ client: client, @@ -33,14 +53,29 @@ func newOptimizerOrchestrator(client *computeoptimizer.ComputeOptimizer, org str } } -type inventoryOrchestrator struct { - client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI - org string -} - func newInventoryOrchestrator(client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI, org string) *inventoryOrchestrator { return &inventoryOrchestrator{ client: client, org: org, } } + +func (s *server) newCostExplorerOrchestrator(ctx context.Context, sp *sessionParams) (*costExplorerOrchestrator, error) { + log.Debugf("initializing costExplorerOrchestrator") + + session, err := s.assumeRole( + ctx, + s.session.ExternalID, + sp.role, + sp.inlinePolicy, + sp.policyArns..., + ) + if err != nil { + return nil, err + } + + return &costExplorerOrchestrator{ + client: costexplorer.New(costexplorer.WithSession(session.Session)), + server: s, + }, nil +} diff --git a/api/policy.go b/api/policy.go index 34981fd..91468f3 100644 --- a/api/policy.go +++ b/api/policy.go @@ -37,7 +37,7 @@ func orgTagAccessPolicy(org string) (string, error) { } func budgetReadWritePolicy() (string, error) { - log.Debugf("generating org policy document") + log.Debugf("generating budget read/write policy document") policy := iam.PolicyDocument{ Version: "2012-10-17", @@ -64,6 +64,75 @@ func budgetReadWritePolicy() (string, error) { return string(j), nil } +func costExplorerReadPolicy() (string, error) { + log.Debugf("generating cost explorer read policy document") + + policy := iam.PolicyDocument{ + Version: "2012-10-17", + Statement: []iam.StatementEntry{ + { + Effect: "Allow", + Action: []string{ + "ce:DescribeCostCategoryDefinition", + "ce:GetRightsizingRecommendation", + "ce:GetCostAndUsage", + "ce:GetSavingsPlansUtilization", + "ce:GetAnomalies", + "ce:GetReservationPurchaseRecommendation", + "ce:ListCostCategoryDefinitions", + "ce:GetCostForecast", + "ce:GetPreferences", + "ce:GetReservationUtilization", + "ce:GetCostCategories", + "ce:GetSavingsPlansPurchaseRecommendation", + "ce:GetDimensionValues", + "ce:GetSavingsPlansUtilizationDetails", + "ce:GetAnomalySubscriptions", + "ce:GetCostAndUsageWithResources", + "ce:DescribeReport", + "ce:GetReservationCoverage", + "ce:GetSavingsPlansCoverage", + "ce:GetAnomalyMonitors", + "ce:DescribeNotificationSubscription", + "ce:GetTags", + "ce:GetUsageForecast", + }, + Resource: []string{"*"}, + }, + }, + } + + j, err := json.Marshal(policy) + if err != nil { + return "", err + } + + return string(j), nil +} + +func defaultCloudWatchMetricsPolicy() (string, error) { + policy := iam.PolicyDocument{ + Version: "2012-10-17", + Statement: []iam.StatementEntry{ + { + Sid: "CloudWatchMetricsPermissions", + Effect: "Allow", + Action: []string{ + "cloudwatch:GetMetricWidgetImage", + }, + Resource: []string{"*"}, + }, + }, + } + + j, err := json.Marshal(policy) + if err != nil { + return "", err + } + + return string(j), nil +} + func defaultBudgetTopicPolicy(arn string) (string, error) { policy := iam.PolicyDocument{ Version: "2012-10-17", diff --git a/api/routes.go b/api/routes.go index edbfe16..c5c7701 100644 --- a/api/routes.go +++ b/api/routes.go @@ -25,13 +25,6 @@ func (s *server) routes() { api.HandleFunc("/{account}/spaces/{space}/instances/{id}/optimizer", s.SpaceInstanceOptimizer).Methods(http.MethodGet) - // TODO remove this - these calls are too expensive - api.HandleFunc("/{account}/spaces/{space}/{resourcename}", s.SpaceResourceGetHandler).Methods(http.MethodGet).MatcherFunc(matchSpaceQueries) - - // metrics endpoints for EC2 instances - // TODO: deprecated but left for backwards compatability, remove me once the UI is updated - api.HandleFunc("/{account}/instances/{id}/metrics/graph", s.GetEC2MetricsURLHandler).Methods(http.MethodGet) - // metrics subrouter - /v1/metrics metricsApi := s.router.PathPrefix("/v1/metrics").Subrouter() metricsApi.HandleFunc("/ping", s.PingHandler).Methods(http.MethodGet) diff --git a/api/server.go b/api/server.go index dcd181f..1e5e7b8 100644 --- a/api/server.go +++ b/api/server.go @@ -8,9 +8,7 @@ import ( "time" "github.com/YaleSpinup/aws-go/services/session" - "github.com/YaleSpinup/cost-api/cloudwatch" "github.com/YaleSpinup/cost-api/common" - "github.com/YaleSpinup/cost-api/costexplorer" "github.com/YaleSpinup/cost-api/imagecache" "github.com/YaleSpinup/cost-api/s3cache" "github.com/gorilla/handlers" @@ -20,19 +18,23 @@ import ( log "github.com/sirupsen/logrus" ) +var ( + CacheExpireTime = 4 * time.Hour + CachePurgeTime = 15 * time.Minute +) + type server struct { - router *mux.Router - version common.Version - context context.Context - costExplorerServices map[string]costexplorer.CostExplorer - cloudwatchServices map[string]cloudwatch.Cloudwatch - session session.Session - orgPolicy string - optimizerCache *cache.Cache - resultCache map[string]*cache.Cache - imageCache imagecache.ImageCache - sessionCache *cache.Cache - org string + accountsMap map[string]string + router *mux.Router + version common.Version + context context.Context + session session.Session + orgPolicy string + optimizerCache *cache.Cache + resultCache *cache.Cache + imageCache imagecache.ImageCache + sessionCache *cache.Cache + org string } // NewServer creates a new server and starts it @@ -42,13 +44,11 @@ func NewServer(config common.Config) error { defer cancel() s := server{ - router: mux.NewRouter(), - version: config.Version, - context: ctx, - costExplorerServices: make(map[string]costexplorer.CostExplorer), - cloudwatchServices: make(map[string]cloudwatch.Cloudwatch), - resultCache: make(map[string]*cache.Cache), - sessionCache: cache.New(600*time.Second, 900*time.Second), + accountsMap: config.AccountsMap, + router: mux.NewRouter(), + version: config.Version, + context: ctx, + sessionCache: cache.New(600*time.Second, 900*time.Second), } if config.Org == "" { @@ -68,11 +68,12 @@ func NewServer(config common.Config) error { config.CacheExpireTime = "4h" } - expireTime, err := time.ParseDuration(config.CacheExpireTime) + exp, err := time.ParseDuration(config.CacheExpireTime) if err != nil { log.Error("Unexpected error with configured expiretime") return err } + CacheExpireTime = exp if config.CachePurgeTime == "" { // set default purgeTime @@ -80,26 +81,18 @@ func NewServer(config common.Config) error { config.CachePurgeTime = "15m" } - purgeTime, err := time.ParseDuration(config.CachePurgeTime) + pt, err := time.ParseDuration(config.CachePurgeTime) if err != nil { log.Error("Unexpected error with configured purgetime") return err } + CachePurgeTime = pt - log.Debugf("creating new optimizer cache with expire time: %s and purge time: %s", expireTime.String(), purgeTime.String()) - s.optimizerCache = cache.New(expireTime, purgeTime) - - // Create shared cost explorer sessions, cloudwatch sessions, and go-cache instances per account defined in the config - for name, c := range config.Accounts { - log.Debugf("creating new cost explorer service for account '%s' with key '%s' in region '%s' (org: %s)", name, c.Akid, c.Region, s.org) - s.costExplorerServices[name] = costexplorer.NewSession(c) - - log.Debugf("creating new cloudwatch service for account '%s' with key '%s' in region '%s' (org: %s)", name, c.Akid, c.Region, s.org) - s.cloudwatchServices[name] = cloudwatch.NewSession(c) + log.Debugf("creating new cost explorer result cache with expire time: %s and purge time: %s", CacheExpireTime, CachePurgeTime) + s.resultCache = cache.New(CacheExpireTime, CachePurgeTime) - log.Debugf("creating new result cache for account '%s' with expire time: %s and purge time: %s", name, expireTime.String(), purgeTime.String()) - s.resultCache[name] = cache.New(expireTime, purgeTime) - } + log.Debugf("creating new optimizer cache with expire time: %s and purge time: %s", CacheExpireTime, CachePurgeTime) + s.optimizerCache = cache.New(CacheExpireTime, CachePurgeTime) // Create a new session used for authentication and assuming cross account roles log.Debugf("Creating new session with key '%s' in region '%s'", config.Account.Akid, config.Account.Region) @@ -159,3 +152,11 @@ func (w LogWriter) Write(p []byte) (n int, err error) { } return } + +// if we have an entry for the account name, return the associated account number +func (s *server) mapAccountNumber(name string) string { + if a, ok := s.accountsMap[name]; ok { + return a + } + return name +} diff --git a/cloudwatch/cloudwatch.go b/cloudwatch/cloudwatch.go index 8441601..d139ecb 100644 --- a/cloudwatch/cloudwatch.go +++ b/cloudwatch/cloudwatch.go @@ -1,7 +1,6 @@ package cloudwatch import ( - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -12,17 +11,40 @@ import ( // Cloudwatch is a wrapper around the aws cloudwatch service with some default config info type Cloudwatch struct { + session *session.Session Service cloudwatchiface.CloudWatchAPI } -// NewSession creates a new cloudwatch session -func NewSession(account common.Account) Cloudwatch { - c := Cloudwatch{} - log.Infof("creating new aws session for costexplorer with key id %s in region %s", account.Akid, account.Region) - sess := session.Must(session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(account.Akid, account.Secret, ""), - Region: aws.String(account.Region), - })) - c.Service = cloudwatch.New(sess) - return c +type CloudwatchOption func(*Cloudwatch) + +func New(opts ...CloudwatchOption) *Cloudwatch { + client := Cloudwatch{} + + for _, opt := range opts { + opt(&client) + } + + if client.session != nil { + client.Service = cloudwatch.New(client.session) + } + + return &client +} + +func WithSession(sess *session.Session) CloudwatchOption { + return func(client *Cloudwatch) { + log.Debug("using aws session") + client.session = sess + } +} + +func WithCredentials(key, secret, token, region string) CloudwatchOption { + return func(client *Cloudwatch) { + log.Debugf("creating new session with key id %s in region %s", key, region) + sess := session.Must(session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(key, secret, token), + Region: aws.String(region), + })) + client.session = sess + } } diff --git a/cloudwatch/cloudwatch_test.go b/cloudwatch/cloudwatch_test.go index 196187a..4e166ad 100644 --- a/cloudwatch/cloudwatch_test.go +++ b/cloudwatch/cloudwatch_test.go @@ -4,7 +4,6 @@ import ( "reflect" "testing" - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" ) @@ -23,8 +22,8 @@ func newmockCloudwatchClient(t *testing.T, err error) cloudwatchiface.CloudWatch } func TestNewSession(t *testing.T) { - e := NewSession(common.Account{}) - if to := reflect.TypeOf(e).String(); to != "cloudwatch.Cloudwatch" { + e := New() + if to := reflect.TypeOf(e).String(); to != "*cloudwatch.Cloudwatch" { t.Errorf("expected type to be 'cloudwatch.Cloudwatch', got %s", to) } } diff --git a/common/config.go b/common/config.go index 4e79bf3..a93958e 100644 --- a/common/config.go +++ b/common/config.go @@ -10,16 +10,17 @@ import ( // Config is representation of the configuration data type Config struct { - ListenAddress string - Accounts map[string]Account Account Account - Token string - LogLevel string - Version Version - Org string + Accounts map[string]Account + AccountsMap map[string]string CacheExpireTime string CachePurgeTime string ImageCache *S3Cache + ListenAddress string + LogLevel string + Org string + Token string + Version Version } // Account is the configuration for an individual account diff --git a/config/config.example.json b/config/config.example.json index 0decd94..4d70e53 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -15,6 +15,11 @@ "prefix": "costapi", "hashingToken": "xxxxxxxx-yyyy-zzzz-aaaa-bbbbbbbbbbb" }, + "accountsMap": { + "spinup": "1234567890", + "spinupsec": "0987654321", + "sandbox": "00000000000" + }, "token": "xxxxxxxx-yyyy-zzzz-aaaa-bbbbbbbbbbb", "logLevel": "debug", "org": "localdev", diff --git a/costexplorer/costexplorer.go b/costexplorer/costexplorer.go index 6f7b26b..d4e356b 100644 --- a/costexplorer/costexplorer.go +++ b/costexplorer/costexplorer.go @@ -1,7 +1,6 @@ package costexplorer import ( - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -12,17 +11,40 @@ import ( // CostExplorer is a wrapper around the aws costexplorer service with some default config info type CostExplorer struct { + session *session.Session Service costexploreriface.CostExplorerAPI } -// NewSession creates a new costexplorer session -func NewSession(account common.Account) CostExplorer { - c := CostExplorer{} - log.Infof("creating new aws session for costexplorer with key id %s in region %s", account.Akid, account.Region) - sess := session.Must(session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(account.Akid, account.Secret, ""), - Region: aws.String(account.Region), - })) - c.Service = costexplorer.New(sess) - return c +type CostExplorerOption func(*CostExplorer) + +func New(opts ...CostExplorerOption) *CostExplorer { + client := CostExplorer{} + + for _, opt := range opts { + opt(&client) + } + + if client.session != nil { + client.Service = costexplorer.New(client.session) + } + + return &client +} + +func WithSession(sess *session.Session) CostExplorerOption { + return func(client *CostExplorer) { + log.Debug("using aws session") + client.session = sess + } +} + +func WithCredentials(key, secret, token, region string) CostExplorerOption { + return func(client *CostExplorer) { + log.Debugf("creating new session with key id %s in region %s", key, region) + sess := session.Must(session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(key, secret, token), + Region: aws.String(region), + })) + client.session = sess + } } diff --git a/costexplorer/costexplorer_test.go b/costexplorer/costexplorer_test.go index 843a477..b590220 100644 --- a/costexplorer/costexplorer_test.go +++ b/costexplorer/costexplorer_test.go @@ -4,7 +4,6 @@ import ( "reflect" "testing" - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/service/costexplorer/costexploreriface" ) @@ -23,8 +22,8 @@ func newmockCostExplorerClient(t *testing.T, err error) costexploreriface.CostEx } func TestNewSession(t *testing.T) { - e := NewSession(common.Account{}) - if to := reflect.TypeOf(e).String(); to != "costexplorer.CostExplorer" { - t.Errorf("expected type to be 'costexplorer.CostExplorer', got %s", to) + e := New() + if to := reflect.TypeOf(e).String(); to != "*costexplorer.CostExplorer" { + t.Errorf("expected type to be '*costexplorer.CostExplorer', got %s", to) } } diff --git a/docker/config.deco.json b/docker/config.deco.json index 201b8ab..b231225 100644 --- a/docker/config.deco.json +++ b/docker/config.deco.json @@ -19,6 +19,10 @@ "externalId": "{{ .external_id }}", "role": "{{ .cross_account_role }}" }, + "accountsMap": { + "spinup": "{{ .spinup_account_number }}", + "spinupsec": "{{ .spinupsec_account_number }}" + }, "imageCache": { "region": "us-east-1", "bucket": "{{ .imagecache_bucket }}",