From 1ac746358d3f8955a9ed8f85781e0ea4d72e0716 Mon Sep 17 00:00:00 2001 From: E Camden Fisher Date: Fri, 3 Dec 2021 07:37:01 -0500 Subject: [PATCH] Convert remaining endpoints to assumerole pattern, cleanup (#28) * convert remaining endpoints to assumerole pattern. cleanup README. remove deprecated routes. * disable circle, standard github actions, mod updates --- .circleci/config.yml | 45 ----- .github/workflows/go.yml | 47 ----- .github/workflows/test.yml | 27 +++ README.md | 89 ++------- api/handlers_budgets.go | 8 +- api/handlers_inventory.go | 2 +- api/handlers_metrics.go | 132 ++++++++------ api/handlers_optimizer.go | 2 +- api/handlers_spaces.go | 290 +++--------------------------- api/handlers_spaces_test.go | 43 ++--- api/orchestration_costexplorer.go | 150 ++++++++++++++++ api/orchestrators.go | 55 ++++-- api/policy.go | 71 +++++++- api/routes.go | 7 - api/server.go | 73 ++++---- cloudwatch/cloudwatch.go | 44 +++-- cloudwatch/cloudwatch_test.go | 5 +- common/config.go | 13 +- config/config.example.json | 17 +- costexplorer/costexplorer.go | 44 +++-- costexplorer/costexplorer_test.go | 7 +- docker/config.deco.json | 16 +- go.mod | 30 ++-- go.sum | 18 ++ 24 files changed, 602 insertions(+), 633 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/go.yml create mode 100644 .github/workflows/test.yml create mode 100644 api/orchestration_costexplorer.go diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 50e9fc4..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 -jobs: - build: - docker: - - image: circleci/golang - - environment: - TEST_RESULTS: /tmp/test-results - GO111MODULE: "on" - - steps: - - checkout - - run: mkdir -p $TEST_RESULTS - - restore_cache: - # Read about caching dependencies: https://circleci.com/docs/2.0/caching/ - keys: - - go-mod-v4-{{ checksum "go.sum" }} - - - run: go get github.com/jstemmer/go-junit-report - - run: go get ./... - - run: - name: Run unit tests - - # store the results of our tests in the $TEST_RESULTS directory - command: | - PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname) - gotestsum --junitfile ${TEST_RESULTS}/gotestsum-report.xml -- $PACKAGE_NAMES - - - save_cache: - key: go-mod-v4-{{ checksum "go.sum" }} - paths: - - "/go/pkg/mod" - - - store_artifacts: # Upload test summary for display in Artifacts: https://circleci.com/docs/2.0/artifacts/ - path: /tmp/test-results - destination: raw-test-output - - - store_test_results: # Upload test results for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/ - path: /tmp/test-results - -workflows: - version: 2 - build-workflow: - jobs: - - build diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml deleted file mode 100644 index 5cbf36b..0000000 --- a/.github/workflows/go.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Go Build and Test - -on: [pull_request, release] - -jobs: - - build: - name: Build - runs-on: ubuntu-latest - steps: - - - name: Set up Go 1.x - uses: actions/setup-go@v2 - with: - go-version: ^1.15 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v2 - - - uses: actions/cache@v2 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - - name: Get dependencies - run: | - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi - - - name: Run Gosec Security Scanner - run: | - export PATH=$PATH:$(go env GOPATH)/bin - go get github.com/securego/gosec/cmd/gosec - gosec -severity=medium -nosec ./... - - - name: Test - run: go test -v . - - - name: Build - run: go build -v . - diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e54be76 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Tests +on: + push: + +jobs: + tests-off: + name: ${{ matrix.os }} - Go v${{ matrix.go-version }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: + - "1.17.x" + os: + - "ubuntu-latest" + + steps: + - uses: actions/checkout@v2 + + - name: Setup Go ${{ matrix.go }} + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + + - name: Test + run: | + go mod tidy -v + go test -cover ./... diff --git a/README.md b/README.md index d6b5822..65aeee3 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,12 @@ This API provides simple restful API access to Amazon's Cost explorer and cloudw ## Endpoints -``` +```text GET /v1/cost/ping GET /v1/cost/version GET /v1/cost/metrics GET /v1/cost/{account}/spaces/{spaceid}[?start=2019-10-01&end=2019-10-30][&groupBy=SERVICE] -GET /v1/cost/{account}/spaces/{spaceid}/{resourcename}[?start=2019-10-01&end=2019-10-30] POST /v1/cost/{account}/spaces/{spaceid}/budgets GET /v1/cost/{account}/spaces/{spaceid}/budgets @@ -18,11 +17,6 @@ DELETE /v1/cost/{account}/spaces/{spaceid}/budgets/{budget} GET /v1/cost/{account}/spaces/{space}/instances/{id}/optimizer -### DEPRECATED ### -GET /v1/cost/{account}/instances/{id}/metrics/graph.png?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] -GET /v1/cost/{account}/instances/{id}/metrics/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] -################## - GET /v1/inventory/{account}/spaces/{spaceid} GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300] @@ -40,9 +34,7 @@ different dimensions is supported by passing query parameters. #### Request month to date costs for a space -``` GET /v1/cost/{account}/spaces/{spaceid} -``` #### Response @@ -75,9 +67,7 @@ GET /v1/cost/{account}/spaces/{spaceid} #### Request costs for a space for a date range -``` GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31 -``` #### Response @@ -132,9 +122,7 @@ GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31 #### Request costs for a space by date range and grouped by a dimension -``` GET /v1/cost/{account}/spaces/{spaceid}?start=2021-04-01&end=2021-05-31&groupby=INSTANCE_TYPE_FAMILY -``` Supported default 'groupby' values are AZ, INSTANCE_TYPE, LINKED_ACCOUNT, OPERATION, PURCHASE_TYPE, SERVICE, USAGE_TYPE, PLATFORM, TENANCY, RECORD_TYPE, LEGAL_ENTITY_NAME, DEPLOYMENT_OPTION, DATABASE_ENGINE, CACHE_ENGINE, INSTANCE_TYPE_FAMILY, REGION, BILLING_ENTITY, RESERVATION_ID, SAVINGS_PLANS_TYPE, SAVINGS_PLAN_ARN, OPERATING_SYSTEM. In addition, the custom RESOURCE_NAME 'groupby' is supported using the Name tag. @@ -317,54 +305,6 @@ Supported default 'groupby' values are AZ, INSTANCE_TYPE, LINKED_ACCOUNT, OPERAT ] ``` -### Get the cost and usage for a resource (name) within a space ID - -By default, this will get the month to date costs for a resource name with a space id - -``` -GET /v1/cost/{account}/spaces/{spaceid}/{resourcename} -``` - -### How it works - -Costs are Filtered - the keys/values are resource tags - -```json -{ - "Filter": { - "And": [ - { - "Tags": { - "Key": "Name", - "Values": ["spinup-000cba.spinup.yale.edu"] - } - }, - { - "Tags": { - "Key": "spinup:spaceid", - "Values": ["spinup-0002a2"] - } - }, - { - "Or": [ - { - "Tags": { - "Key": "yale:org", - "Values": ["ss"] - } - }, - { - "Tags": { - "Key": "spinup:org", - "Values": ["ss"] - } - }] - } - ] - } -} -``` - ## Budget Usage ### Create Budgets Alerts @@ -733,28 +673,31 @@ GET /v1/inventory/{account}/spaces/{spaceid} ## Metrics Usage -### Get cloudwatch metrics widgets URL from S3 for an instance ID +### Get Cloudwatch metrics widgets URL from S3 for an instance ID This will get the passed metric(s) for the passed instance ID or container cluster/service in a `image/png` graph for the past 1 day by default, cache it in S3 and return the URL. URLs are cached in the API for 5 minutes, the images should be purged from the S3 cache on a schedule. It's also possible to pass the height, width, start time, end time and period (e. `300s` for 300 seconds, `5m` for 5 minutes). Query parameters must follow the [CloudWatch Metric Widget Structure](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html). -### Documentation on cloudwatch metrics +### Documentation on Cloudwatch metrics + +#### Get a list of metrics per AWS service +```bash +aws --region us-east-1 cloudwatch list-metrics --namespace AWS/RDS |grep MetricName |sort| uniq ``` -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html -Get you a list of metrics per AWS service -$ aws --region us-east-1 cloudwatch list-metrics --namespace AWS/RDS |grep MetricName |sort| uniq +#### Helpful references -GetMetricWidget gets a metric widget image for an instance id -https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html -https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html -https://docs.aws.amazon.com/AmazonECS/latest/developerguide/cloudwatch-metrics.html -https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/MonitoringOverview.html +* [Viewing Metrics with CloudWatch](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/viewing_metrics_with_cloudwatch.html) +* [CloudWatch Metrics Widget Structure](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/CloudWatch-Metric-Widget-Structure.html) +* [CloudWatch Metrics Developer Guide](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/cloudwatch-metrics.html) +* [AWS Aurora Monitoring Overview](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/MonitoringOverview.html) -Example metrics request +#### Example metrics request + +```json { "metrics": [ [ "AWS/ECS", "CPUUtilization", "ClusterName", "spinup-000393", "ServiceName", "spinup-0010a3-testsvc" ] @@ -768,7 +711,7 @@ Example metrics request #### Request -``` +```text GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&....] GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&start={start}&end={end}&period={period}] diff --git a/api/handlers_budgets.go b/api/handlers_budgets.go index 08e6c5e..910c453 100644 --- a/api/handlers_budgets.go +++ b/api/handlers_budgets.go @@ -15,7 +15,7 @@ import ( func (s *server) SpaceBudgetsCreatehandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) @@ -71,7 +71,7 @@ func (s *server) SpaceBudgetsCreatehandler(w http.ResponseWriter, r *http.Reques func (s *server) SpaceBudgetsListHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) @@ -120,7 +120,7 @@ func (s *server) SpaceBudgetsListHandler(w http.ResponseWriter, r *http.Request) func (s *server) SpaceBudgetsShowHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] budget := vars["budget"] @@ -170,7 +170,7 @@ func (s *server) SpaceBudgetsShowHandler(w http.ResponseWriter, r *http.Request) func (s *server) SpaceBudgetsDeleteHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] budget := vars["budget"] diff --git a/api/handlers_inventory.go b/api/handlers_inventory.go index 46b4601..9730a1c 100644 --- a/api/handlers_inventory.go +++ b/api/handlers_inventory.go @@ -16,7 +16,7 @@ import ( func (s *server) SpaceInventoryGetHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) spaceID := vars["space"] role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) diff --git a/api/handlers_metrics.go b/api/handlers_metrics.go index 02e4dc4..13471da 100644 --- a/api/handlers_metrics.go +++ b/api/handlers_metrics.go @@ -18,24 +18,29 @@ import ( func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) instanceId := vars["id"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -50,9 +55,9 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, instanceId, strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, instanceId, strings.Join(metrics, "-"), req.String()) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -85,7 +90,7 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -96,25 +101,30 @@ func (s *server) GetEC2MetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) cluster := vars["cluster"] service := vars["service"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -129,11 +139,11 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, fmt.Sprintf("%s-%s", cluster, service), strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, fmt.Sprintf("%s-%s", cluster, service), strings.Join(metrics, "-"), req.String()) log.Debugf("object key: %s", key) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -166,7 +176,7 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -177,25 +187,30 @@ func (s *server) GetECSMetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) bucketName := vars["bucket"] metric := vars["metric"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) // only support NumberOfObjects and BucketSizeBytes var storageType string @@ -220,11 +235,11 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) }, } - key := fmt.Sprintf("%s/%s/%s%s", s.org, bucketName, metric, req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, bucketName, metric, req.String()) log.Debugf("object key: %s", key) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -251,7 +266,7 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) @@ -263,25 +278,30 @@ func (s *server) GetS3MetricsURLHandler(w http.ResponseWriter, r *http.Request) func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) queryType := vars["type"] instanceId := vars["id"] - cwService, ok := s.cloudwatchServices[account] - if !ok { - msg := fmt.Sprintf("cloudwatch service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + policy, err := defaultCloudWatchMetricsPolicy() + if err != nil { + handleError(w, err) return } - log.Debugf("found cloudwatch service %+v", cwService) - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + session, err := s.assumeRole( + r.Context(), + s.session.ExternalID, + role, + policy, + ) + if err != nil { + msg := fmt.Sprintf("failed to assume role in account: %s", account) + handleError(w, apierror.New(apierror.ErrForbidden, msg, nil)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) + + cwService := cloudwatch.New(cloudwatch.WithSession(session.Session)) queries := r.URL.Query() metrics := queries["metric"] @@ -296,9 +316,9 @@ func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) return } - key := fmt.Sprintf("%s/%s/%s%s", s.org, instanceId, strings.Join(metrics, "-"), req.String()) + key := fmt.Sprintf("%s/%s/%s/%s%s", account, s.org, instanceId, strings.Join(metrics, "-"), req.String()) hashedCacheKey := s.imageCache.HashedKey(key) - if res, expire, ok := resultCache.GetWithExpiration(hashedCacheKey); ok { + if res, expire, ok := s.resultCache.GetWithExpiration(hashedCacheKey); ok { log.Debugf("found cached object: %s", res) if body, ok := res.([]byte); ok { @@ -340,7 +360,7 @@ func (s *server) GetRDSMetricsURLHandler(w http.ResponseWriter, r *http.Request) handleError(w, err) return } - resultCache.Set(hashedCacheKey, meta, 300*time.Second) + s.resultCache.Set(hashedCacheKey, meta, 300*time.Second) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) diff --git a/api/handlers_optimizer.go b/api/handlers_optimizer.go index 19d1a9b..25bf65f 100644 --- a/api/handlers_optimizer.go +++ b/api/handlers_optimizer.go @@ -15,7 +15,7 @@ import ( func (s *server) SpaceInstanceOptimizer(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} vars := mux.Vars(r) - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) instanceID := vars["id"] var j []byte diff --git a/api/handlers_spaces.go b/api/handlers_spaces.go index 8346704..d5d4400 100644 --- a/api/handlers_spaces.go +++ b/api/handlers_spaces.go @@ -4,15 +4,10 @@ import ( "encoding/json" "fmt" "net/http" - "time" "github.com/YaleSpinup/apierror" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/costexplorer" "github.com/gorilla/mux" - "github.com/pkg/errors" - ce "github.com/YaleSpinup/cost-api/costexplorer" log "github.com/sirupsen/logrus" ) @@ -20,224 +15,43 @@ import ( // it pulls data from the start of the month until now. func (s *server) SpaceGetHandler(w http.ResponseWriter, r *http.Request) { w = LogWriter{w} - - // loop thru and log given API input vars in debug vars := mux.Vars(r) - for k, v := range vars { - log.Debugf("key=%v, value=%v", k, v) - } - - // get vars from the API route - account := vars["account"] + account := s.mapAccountNumber(vars["account"]) startTime := vars["start"] endTime := vars["end"] spaceID := vars["space"] groupBy := vars["groupby"] - ceService, ok := s.costExplorerServices[account] - if !ok { - msg := fmt.Sprintf("cost explorer service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) - return - } - log.Debugf("found cost explorer service %+v", ceService) - - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName) + policy, err := costExplorerReadPolicy() + if err != nil { + handleError(w, apierror.New(apierror.ErrInternalError, "failed to generate policy", err)) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) - // vars for checking input times parse and are valid - var start string - var end string - var err error + orch, err := s.newCostExplorerOrchestrator(r.Context(), &sessionParams{ + inlinePolicy: policy, + role: role, + }) - // Did we get cost-explorer start and end times on the API? - // set defaults, else verify times given on API - if endTime == "" || startTime == "" { - log.Debug("no start or end time given on API input, assigning defaults") - start, end = getTimeDefault() - } else { - start, end, err = getTimeAPI(startTime, endTime) - if err != nil { - handleError(w, err) - return - } - } - - input := costexplorer.GetCostAndUsageInput{ - Filter: ce.And(inSpace(spaceID), inOrg(s.org), notTryIT()), - Granularity: aws.String("MONTHLY"), - Metrics: []*string{ - aws.String("BLENDED_COST"), - aws.String("UNBLENDED_COST"), - aws.String("USAGE_QUANTITY"), - }, - TimePeriod: &costexplorer.DateInterval{ - Start: aws.String(start), - End: aws.String(end), + out, cached, expire, err := orch.getCostAndUsageForSpace( + r.Context(), + &costAndUsageReq{ + account: account, + spaceID: spaceID, + start: startTime, + end: endTime, + groupBy: groupBy, }, - } - - switch { - case groupBy == "RESOURCE_NAME": - input.GroupBy = []*costexplorer.GroupDefinition{ - { - Key: aws.String("Name"), - Type: aws.String("TAG"), - }, - } - case groupBy != "": - input.GroupBy = []*costexplorer.GroupDefinition{ - { - Key: aws.String(groupBy), - Type: aws.String("DIMENSION"), - }, - } - } - - // create a cacheKey more unique than spaceID for managing cache objects. - // Since we will accept date-range cost exploring and grouping, concatenate - // the spaceID, the start time, end time and group by so we can cache each - // time-based result - cacheKey := fmt.Sprintf("%s_%s_%s_%s", spaceID, startTime, endTime, groupBy) - log.Debugf("cacheKey: %s", cacheKey) - - // the object is not found in the cache, call AWS cost-explorer and set cache - var out []*costexplorer.ResultByTime - c, expire, ok := resultCache.GetWithExpiration(cacheKey) - if !ok || c == nil { - log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", s.org, cacheKey) - // call cost-explorer - var err error - out, err = ceService.GetCostAndUsage(r.Context(), &input) - if err != nil { - msg := fmt.Sprintf("failed to get costs for space %s: %s", cacheKey, err.Error()) - handleError(w, errors.Wrap(err, msg)) - return - } - - // cache results - resultCache.SetDefault(cacheKey, out) - } else { - // cached object was found - out = c.([]*costexplorer.ResultByTime) - log.Debugf("found cached object: %s", out) - w.Header().Set("X-Cache-Hit", "true") - w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", time.Until(expire).Seconds())) - } - - j, err := json.Marshal(out) + ) if err != nil { - log.Errorf("cannot marshal response (%v) into JSON: %s", out, err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(j) - -} - -// SpaceResourceGetHandler gets the cost for a resource within a space -func (s *server) SpaceResourceGetHandler(w http.ResponseWriter, r *http.Request) { - w = LogWriter{w} - - log.Debugf("ResourceGetHandler Called") - - // loop thru and log given API input vars in debug - vars := mux.Vars(r) - for k, v := range vars { - log.Debugf("key=%v, value=%v", k, v) - } - - // get vars from the API route - account := vars["account"] - startTime := vars["start"] - endTime := vars["end"] - spaceID := vars["space"] - name := vars["resourcename"] - - ceService, ok := s.costExplorerServices[account] - if !ok { - msg := fmt.Sprintf("cost explorer service not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) - return - } - log.Debugf("found cost explorer service %+v", ceService) - - resultCache, ok := s.resultCache[account] - if !ok { - msg := fmt.Sprintf("result cache not found for account: %s", account) - handleError(w, apierror.New(apierror.ErrNotFound, msg, nil)) + handleError(w, err) return } - log.Debugf("found cost explorer result cache %+v", *resultCache) - - // vars for checking input times parse and are valid - var start string - var end string - var err error - // Did we get cost-explorer start and end times on the API? - // set defaults, else verify times given on API - if endTime == "" || startTime == "" { - log.Debug("no start or end time given on API input, assigning defaults") - start, end = getTimeDefault() - } else { - start, end, err = getTimeAPI(startTime, endTime) - if err != nil { - handleError(w, err) - return - } - } - - input := costexplorer.GetCostAndUsageInput{ - Filter: ce.And(ofName(name), inSpace(spaceID), inOrg(s.org), notTryIT()), - Granularity: aws.String("MONTHLY"), - Metrics: []*string{ - aws.String("BLENDED_COST"), - aws.String("UNBLENDED_COST"), - aws.String("USAGE_QUANTITY"), - }, - TimePeriod: &costexplorer.DateInterval{ - Start: aws.String(start), - End: aws.String(end), - }, - } - - // create a cacheKey more unique than spaceID for managing cache objects. - // Since we will accept date-range cost exploring, concatenate the spaceID - // and the start and end time so we can cache each time-based result - cacheKey := fmt.Sprintf("%s_%s_%s_%s", spaceID, name, startTime, endTime) - log.Debugf("cacheKey: %s", cacheKey) - - // the object is not found in the cache, call AWS cost-explorer and set cache - var out []*costexplorer.ResultByTime - c, expire, ok := resultCache.GetWithExpiration(cacheKey) - if !ok || c == nil { - log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", s.org, cacheKey) - // call cost-explorer - var err error - out, err = ceService.GetCostAndUsage(r.Context(), &input) - if err != nil { - msg := fmt.Sprintf("failed to get costs for space %s: %s", cacheKey, err.Error()) - handleError(w, errors.Wrap(err, msg)) - return - } - - // cache results - resultCache.SetDefault(cacheKey, out) - } else { - // cached object was found - out = c.([]*costexplorer.ResultByTime) - log.Debugf("found cached object: %s", out) - w.Header().Set("X-Cache-Hit", "true") - w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", time.Until(expire).Seconds())) + w.Header().Set("X-Cache-Hit", fmt.Sprintf("%t", cached)) + if cached { + w.Header().Set("X-Cache-Expire", fmt.Sprintf("%0.fs", expire.Seconds())) } j, err := json.Marshal(out) @@ -252,63 +66,3 @@ func (s *server) SpaceResourceGetHandler(w http.ResponseWriter, r *http.Request) w.Write(j) } - -// getTimeDefault returns time range from beginning of month to day-of-month now -func getTimeDefault() (string, string) { - // if it's the first day of the month, get today's usage thus far - y, m, d := time.Now().Date() - if d == 1 { - d = 3 - } - return fmt.Sprintf("%d-%02d-01", y, m), fmt.Sprintf("%d-%02d-%02d", y, m, d) -} - -// getTimeAPI returns time parsed from API input -func getTimeAPI(startTime, endTime string) (string, string, error) { - log.Debugf("startTime: %s, endTime: %s ", startTime, endTime) - - // sTmp and eTmp temporary vars to hold time.Time objects - sTmp, err := time.Parse("2006-01-02", startTime) - if err != nil { - return "", "", errors.Wrapf(err, "error parsing StartTime from input") - } - - eTmp, err := time.Parse("2006-01-02", endTime) - if err != nil { - return "", "", errors.Wrapf(err, "error parsing EndTime from input") - } - - // if time on the API input is already borked, don't continue - // end time is greater than start time, logically - timeValidity := eTmp.After(sTmp) - if !timeValidity { - return "", "", errors.Errorf("endTime should be greater than startTime") - } - - // convert time.Time to a string - return sTmp.Format("2006-01-02"), eTmp.Format("2006-01-02"), nil -} - -// inSpace returns the cost explorer expression to filter on spaceid -func inSpace(spaceID string) *costexplorer.Expression { - return ce.Tag("spinup:spaceid", []string{spaceID}) -} - -// ofName returns the cost explorer expression to filter on name -func ofName(name string) *costexplorer.Expression { - return ce.Tag("Name", []string{name}) -} - -// inOrg returns the cost explorer expression to filter on org -func inOrg(org string) *costexplorer.Expression { - yaleTag := ce.Tag("yale:org", []string{org}) - spinupTag := ce.Tag("spinup:org", []string{org}) - return ce.Or(yaleTag, spinupTag) -} - -// notTryIT returns the cost explorer expression to filter out tryits -func notTryIT() *costexplorer.Expression { - yaleTag := ce.Tag("yale:subsidized", []string{"true"}) - spinupTag := ce.Tag("spinup:subsidized", []string{"true"}) - return ce.Not(ce.Or(yaleTag, spinupTag)) -} diff --git a/api/handlers_spaces_test.go b/api/handlers_spaces_test.go index 8fc81e4..588d2c9 100644 --- a/api/handlers_spaces_test.go +++ b/api/handlers_spaces_test.go @@ -11,9 +11,12 @@ import ( "github.com/aws/aws-sdk-go/service/costexplorer" ) -func TestGetTimeDefault(t *testing.T) { +func TestParseTime(t *testing.T) { // use defaults derived in code - startTime, endTime := getTimeDefault() + startResult, endResult, err := parseTime("", "") + if err != nil { + t.Errorf("unexpected error from parseTime: %s", err) + } // tests should match defaults from getTimeDefault y, m, d := time.Now().Date() @@ -24,13 +27,13 @@ func TestGetTimeDefault(t *testing.T) { sTime := fmt.Sprintf("%d-%02d-01", y, m) eTime := fmt.Sprintf("%d-%02d-%02d", y, m, d) - if startTime == sTime { + if startResult == sTime { t.Logf("got expected default sTime: %s\n", sTime) } else { t.Errorf("got unexpected sTime: %s\n", sTime) } - if endTime == eTime { + if endResult == eTime { t.Logf("got expected default eTime: %s\n", eTime) } else { t.Errorf("got unexpected eTime: %s\n", eTime) @@ -40,24 +43,21 @@ func TestGetTimeDefault(t *testing.T) { sTime = "2006-01-02" eTime = "2006-13-40" - if startTime != sTime { - t.Logf("negative test sTime: %s does not match: %s", sTime, startTime) + if startResult != sTime { + t.Logf("negative test sTime: %s does not match: %s", sTime, startResult) } else { t.Errorf("got unexpected sTime: %s\n", sTime) } - if endTime != eTime { - t.Logf("negative test eTime: %s does not match: %s", eTime, endTime) + if endResult != eTime { + t.Logf("negative test eTime: %s does not match: %s", eTime, endResult) } else { t.Errorf("got unexpected eTime: %s\n", eTime) } -} - -func TestGetTimeAPI(t *testing.T) { startTime := "2019-11-01" endTime := "2019-11-30" - startResult, endResult, err := getTimeAPI(startTime, endTime) + startResult, endResult, err = parseTime(startTime, endTime) if err != nil { t.Errorf("got unexpected error: %s", err) } @@ -70,10 +70,10 @@ func TestGetTimeAPI(t *testing.T) { // negative tests for non-matching API inputs from getTimeDefault // bad start time fails - sTime := "2006-01-022" - eTime := "2006-12-02" + startTime = "2006-01-022" + endTime = "2006-12-02" - neg00startResult, neg00endResult, err := getTimeAPI(sTime, eTime) + neg00startResult, neg00endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error: %s", err) } @@ -85,10 +85,10 @@ func TestGetTimeAPI(t *testing.T) { } // bad end time fails - sTime = "2006-01-02" - eTime = "2006-12-403" + startTime = "2006-01-02" + endTime = "2006-12-403" - neg01startResult, neg01endResult, err := getTimeAPI(sTime, eTime) + neg01startResult, neg01endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error: %s", err) } @@ -100,10 +100,10 @@ func TestGetTimeAPI(t *testing.T) { } // start after end fails - sTime = "2006-01-30" - eTime = "2006-01-01" + startTime = "2006-01-30" + endTime = "2006-01-01" - neg02startResult, neg02endResult, err := getTimeAPI(sTime, eTime) + neg02startResult, neg02endResult, err := parseTime(startTime, endTime) if err != nil { t.Logf("negative test got expected error for start after end : %s", err) } @@ -113,6 +113,7 @@ func TestGetTimeAPI(t *testing.T) { if neg02endResult == endTime { t.Logf("negative test got expected neg02endResult from getTimeAPI: %s", neg02endResult) } + } func TestInSpace(t *testing.T) { diff --git a/api/orchestration_costexplorer.go b/api/orchestration_costexplorer.go new file mode 100644 index 0000000..bee5247 --- /dev/null +++ b/api/orchestration_costexplorer.go @@ -0,0 +1,150 @@ +package api + +import ( + "context" + "fmt" + "time" + + ce "github.com/YaleSpinup/cost-api/costexplorer" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/costexplorer" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type costAndUsageReq struct { + account, spaceID, start, end, groupBy string +} + +func (o *costExplorerOrchestrator) getCostAndUsageForSpace(ctx context.Context, req *costAndUsageReq) ([]*costexplorer.ResultByTime, bool, time.Duration, error) { + start, end, err := parseTime(req.start, req.end) + if err != nil { + return nil, false, 0, nil + } + + input := costexplorer.GetCostAndUsageInput{ + Filter: ce.And(inSpace(req.spaceID), inOrg(o.server.org), notTryIT()), + Granularity: aws.String("MONTHLY"), + Metrics: []*string{ + aws.String("BLENDED_COST"), + aws.String("UNBLENDED_COST"), + aws.String("USAGE_QUANTITY"), + }, + TimePeriod: &costexplorer.DateInterval{ + Start: aws.String(start), + End: aws.String(end), + }, + } + + switch { + case req.groupBy == "RESOURCE_NAME": + input.GroupBy = []*costexplorer.GroupDefinition{ + { + Key: aws.String("Name"), + Type: aws.String("TAG"), + }, + } + case req.groupBy != "": + input.GroupBy = []*costexplorer.GroupDefinition{ + { + Key: aws.String(req.groupBy), + Type: aws.String("DIMENSION"), + }, + } + } + + // create a cacheKey more unique than spaceID for managing cache objects. + // Since we will accept date-range cost exploring and grouping, concatenate + // the spaceID, the start time, end time and group by so we can cache each + // time-based result + cacheKey := fmt.Sprintf("%s_%s_%s_%s_%s", req.account, req.spaceID, req.start, req.end, req.groupBy) + + log.Debugf("cacheKey: %s", cacheKey) + + // the object is not found in the cache, call AWS cost-explorer and set cache + c, expire, ok := o.server.resultCache.GetWithExpiration(cacheKey) + if !ok || c == nil { + log.Debugf("cache empty for org, and space-cacheKey: %s, %s, calling cost-explorer", o.server.org, cacheKey) + + // call cost-explorer + out, err := o.client.GetCostAndUsage(ctx, &input) + if err != nil { + return nil, false, 0, err + } + + // cache results + o.server.resultCache.SetDefault(cacheKey, out) + + return out, false, 0, nil + } + + // cached object was found + out, ok := c.([]*costexplorer.ResultByTime) + if !ok { + return nil, false, 0, errors.New("value in cache is not a []*costexplorer.ResultByTime!") + } + + log.Debugf("found cached object: %s", out) + + return out, true, time.Until(expire), nil +} + +// parseTime returns time range from beginning of month to day-of-month now if the +// passed values are empty otherwise, it parses the string and returns the value (or an error) +func parseTime(start, end string) (string, string, error) { + // if it's the first day of the month, get today's usage thus far + // TODO: :confused: + y, m, d := time.Now().Date() + if d == 1 { + d = 3 + } + + if start == "" { + start = fmt.Sprintf("%d-%02d-01", y, m) + } + + startStamp, err := time.Parse("2006-01-02", start) + if err != nil { + return "", "", err + } + + if end == "" { + end = fmt.Sprintf("%d-%02d-%02d", y, m, d) + } + + endStamp, err := time.Parse("2006-01-02", end) + if err != nil { + return "", "", err + } + + if !endStamp.After(startStamp) { + return "", "", fmt.Errorf("end time should be after start time") + } + + // convert time.Time to a string + return startStamp.Format("2006-01-02"), endStamp.Format("2006-01-02"), nil +} + +// inSpace returns the cost explorer expression to filter on spaceid +func inSpace(spaceID string) *costexplorer.Expression { + return ce.Tag("spinup:spaceid", []string{spaceID}) +} + +// ofName returns the cost explorer expression to filter on name +func ofName(name string) *costexplorer.Expression { + return ce.Tag("Name", []string{name}) +} + +// inOrg returns the cost explorer expression to filter on org +func inOrg(org string) *costexplorer.Expression { + yaleTag := ce.Tag("yale:org", []string{org}) + spinupTag := ce.Tag("spinup:org", []string{org}) + return ce.Or(yaleTag, spinupTag) +} + +// notTryIT returns the cost explorer expression to filter out tryits +func notTryIT() *costexplorer.Expression { + yaleTag := ce.Tag("yale:subsidized", []string{"true"}) + spinupTag := ce.Tag("spinup:subsidized", []string{"true"}) + return ce.Not(ce.Or(yaleTag, spinupTag)) +} diff --git a/api/orchestrators.go b/api/orchestrators.go index 097947f..4fff285 100644 --- a/api/orchestrators.go +++ b/api/orchestrators.go @@ -1,18 +1,43 @@ package api import ( + "context" + "github.com/YaleSpinup/cost-api/budgets" "github.com/YaleSpinup/cost-api/computeoptimizer" + "github.com/YaleSpinup/cost-api/costexplorer" "github.com/YaleSpinup/cost-api/resourcegroupstaggingapi" "github.com/YaleSpinup/cost-api/sns" + log "github.com/sirupsen/logrus" ) +type sessionParams struct { + role string + inlinePolicy string + policyArns []string +} + type budgetsOrchestrator struct { client *budgets.Budgets snsClient *sns.SNS org string } +type costExplorerOrchestrator struct { + client *costexplorer.CostExplorer + server *server +} + +type optimizerOrchestrator struct { + client *computeoptimizer.ComputeOptimizer + org string +} + +type inventoryOrchestrator struct { + client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + org string +} + func newBudgetsOrchestrator(budgetsClient *budgets.Budgets, snsClient *sns.SNS, org string) *budgetsOrchestrator { return &budgetsOrchestrator{ client: budgetsClient, @@ -21,11 +46,6 @@ func newBudgetsOrchestrator(budgetsClient *budgets.Budgets, snsClient *sns.SNS, } } -type optimizerOrchestrator struct { - client *computeoptimizer.ComputeOptimizer - org string -} - func newOptimizerOrchestrator(client *computeoptimizer.ComputeOptimizer, org string) *optimizerOrchestrator { return &optimizerOrchestrator{ client: client, @@ -33,14 +53,29 @@ func newOptimizerOrchestrator(client *computeoptimizer.ComputeOptimizer, org str } } -type inventoryOrchestrator struct { - client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI - org string -} - func newInventoryOrchestrator(client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI, org string) *inventoryOrchestrator { return &inventoryOrchestrator{ client: client, org: org, } } + +func (s *server) newCostExplorerOrchestrator(ctx context.Context, sp *sessionParams) (*costExplorerOrchestrator, error) { + log.Debugf("initializing costExplorerOrchestrator") + + session, err := s.assumeRole( + ctx, + s.session.ExternalID, + sp.role, + sp.inlinePolicy, + sp.policyArns..., + ) + if err != nil { + return nil, err + } + + return &costExplorerOrchestrator{ + client: costexplorer.New(costexplorer.WithSession(session.Session)), + server: s, + }, nil +} diff --git a/api/policy.go b/api/policy.go index 34981fd..91468f3 100644 --- a/api/policy.go +++ b/api/policy.go @@ -37,7 +37,7 @@ func orgTagAccessPolicy(org string) (string, error) { } func budgetReadWritePolicy() (string, error) { - log.Debugf("generating org policy document") + log.Debugf("generating budget read/write policy document") policy := iam.PolicyDocument{ Version: "2012-10-17", @@ -64,6 +64,75 @@ func budgetReadWritePolicy() (string, error) { return string(j), nil } +func costExplorerReadPolicy() (string, error) { + log.Debugf("generating cost explorer read policy document") + + policy := iam.PolicyDocument{ + Version: "2012-10-17", + Statement: []iam.StatementEntry{ + { + Effect: "Allow", + Action: []string{ + "ce:DescribeCostCategoryDefinition", + "ce:GetRightsizingRecommendation", + "ce:GetCostAndUsage", + "ce:GetSavingsPlansUtilization", + "ce:GetAnomalies", + "ce:GetReservationPurchaseRecommendation", + "ce:ListCostCategoryDefinitions", + "ce:GetCostForecast", + "ce:GetPreferences", + "ce:GetReservationUtilization", + "ce:GetCostCategories", + "ce:GetSavingsPlansPurchaseRecommendation", + "ce:GetDimensionValues", + "ce:GetSavingsPlansUtilizationDetails", + "ce:GetAnomalySubscriptions", + "ce:GetCostAndUsageWithResources", + "ce:DescribeReport", + "ce:GetReservationCoverage", + "ce:GetSavingsPlansCoverage", + "ce:GetAnomalyMonitors", + "ce:DescribeNotificationSubscription", + "ce:GetTags", + "ce:GetUsageForecast", + }, + Resource: []string{"*"}, + }, + }, + } + + j, err := json.Marshal(policy) + if err != nil { + return "", err + } + + return string(j), nil +} + +func defaultCloudWatchMetricsPolicy() (string, error) { + policy := iam.PolicyDocument{ + Version: "2012-10-17", + Statement: []iam.StatementEntry{ + { + Sid: "CloudWatchMetricsPermissions", + Effect: "Allow", + Action: []string{ + "cloudwatch:GetMetricWidgetImage", + }, + Resource: []string{"*"}, + }, + }, + } + + j, err := json.Marshal(policy) + if err != nil { + return "", err + } + + return string(j), nil +} + func defaultBudgetTopicPolicy(arn string) (string, error) { policy := iam.PolicyDocument{ Version: "2012-10-17", diff --git a/api/routes.go b/api/routes.go index edbfe16..c5c7701 100644 --- a/api/routes.go +++ b/api/routes.go @@ -25,13 +25,6 @@ func (s *server) routes() { api.HandleFunc("/{account}/spaces/{space}/instances/{id}/optimizer", s.SpaceInstanceOptimizer).Methods(http.MethodGet) - // TODO remove this - these calls are too expensive - api.HandleFunc("/{account}/spaces/{space}/{resourcename}", s.SpaceResourceGetHandler).Methods(http.MethodGet).MatcherFunc(matchSpaceQueries) - - // metrics endpoints for EC2 instances - // TODO: deprecated but left for backwards compatability, remove me once the UI is updated - api.HandleFunc("/{account}/instances/{id}/metrics/graph", s.GetEC2MetricsURLHandler).Methods(http.MethodGet) - // metrics subrouter - /v1/metrics metricsApi := s.router.PathPrefix("/v1/metrics").Subrouter() metricsApi.HandleFunc("/ping", s.PingHandler).Methods(http.MethodGet) diff --git a/api/server.go b/api/server.go index dcd181f..1e5e7b8 100644 --- a/api/server.go +++ b/api/server.go @@ -8,9 +8,7 @@ import ( "time" "github.com/YaleSpinup/aws-go/services/session" - "github.com/YaleSpinup/cost-api/cloudwatch" "github.com/YaleSpinup/cost-api/common" - "github.com/YaleSpinup/cost-api/costexplorer" "github.com/YaleSpinup/cost-api/imagecache" "github.com/YaleSpinup/cost-api/s3cache" "github.com/gorilla/handlers" @@ -20,19 +18,23 @@ import ( log "github.com/sirupsen/logrus" ) +var ( + CacheExpireTime = 4 * time.Hour + CachePurgeTime = 15 * time.Minute +) + type server struct { - router *mux.Router - version common.Version - context context.Context - costExplorerServices map[string]costexplorer.CostExplorer - cloudwatchServices map[string]cloudwatch.Cloudwatch - session session.Session - orgPolicy string - optimizerCache *cache.Cache - resultCache map[string]*cache.Cache - imageCache imagecache.ImageCache - sessionCache *cache.Cache - org string + accountsMap map[string]string + router *mux.Router + version common.Version + context context.Context + session session.Session + orgPolicy string + optimizerCache *cache.Cache + resultCache *cache.Cache + imageCache imagecache.ImageCache + sessionCache *cache.Cache + org string } // NewServer creates a new server and starts it @@ -42,13 +44,11 @@ func NewServer(config common.Config) error { defer cancel() s := server{ - router: mux.NewRouter(), - version: config.Version, - context: ctx, - costExplorerServices: make(map[string]costexplorer.CostExplorer), - cloudwatchServices: make(map[string]cloudwatch.Cloudwatch), - resultCache: make(map[string]*cache.Cache), - sessionCache: cache.New(600*time.Second, 900*time.Second), + accountsMap: config.AccountsMap, + router: mux.NewRouter(), + version: config.Version, + context: ctx, + sessionCache: cache.New(600*time.Second, 900*time.Second), } if config.Org == "" { @@ -68,11 +68,12 @@ func NewServer(config common.Config) error { config.CacheExpireTime = "4h" } - expireTime, err := time.ParseDuration(config.CacheExpireTime) + exp, err := time.ParseDuration(config.CacheExpireTime) if err != nil { log.Error("Unexpected error with configured expiretime") return err } + CacheExpireTime = exp if config.CachePurgeTime == "" { // set default purgeTime @@ -80,26 +81,18 @@ func NewServer(config common.Config) error { config.CachePurgeTime = "15m" } - purgeTime, err := time.ParseDuration(config.CachePurgeTime) + pt, err := time.ParseDuration(config.CachePurgeTime) if err != nil { log.Error("Unexpected error with configured purgetime") return err } + CachePurgeTime = pt - log.Debugf("creating new optimizer cache with expire time: %s and purge time: %s", expireTime.String(), purgeTime.String()) - s.optimizerCache = cache.New(expireTime, purgeTime) - - // Create shared cost explorer sessions, cloudwatch sessions, and go-cache instances per account defined in the config - for name, c := range config.Accounts { - log.Debugf("creating new cost explorer service for account '%s' with key '%s' in region '%s' (org: %s)", name, c.Akid, c.Region, s.org) - s.costExplorerServices[name] = costexplorer.NewSession(c) - - log.Debugf("creating new cloudwatch service for account '%s' with key '%s' in region '%s' (org: %s)", name, c.Akid, c.Region, s.org) - s.cloudwatchServices[name] = cloudwatch.NewSession(c) + log.Debugf("creating new cost explorer result cache with expire time: %s and purge time: %s", CacheExpireTime, CachePurgeTime) + s.resultCache = cache.New(CacheExpireTime, CachePurgeTime) - log.Debugf("creating new result cache for account '%s' with expire time: %s and purge time: %s", name, expireTime.String(), purgeTime.String()) - s.resultCache[name] = cache.New(expireTime, purgeTime) - } + log.Debugf("creating new optimizer cache with expire time: %s and purge time: %s", CacheExpireTime, CachePurgeTime) + s.optimizerCache = cache.New(CacheExpireTime, CachePurgeTime) // Create a new session used for authentication and assuming cross account roles log.Debugf("Creating new session with key '%s' in region '%s'", config.Account.Akid, config.Account.Region) @@ -159,3 +152,11 @@ func (w LogWriter) Write(p []byte) (n int, err error) { } return } + +// if we have an entry for the account name, return the associated account number +func (s *server) mapAccountNumber(name string) string { + if a, ok := s.accountsMap[name]; ok { + return a + } + return name +} diff --git a/cloudwatch/cloudwatch.go b/cloudwatch/cloudwatch.go index 8441601..d139ecb 100644 --- a/cloudwatch/cloudwatch.go +++ b/cloudwatch/cloudwatch.go @@ -1,7 +1,6 @@ package cloudwatch import ( - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -12,17 +11,40 @@ import ( // Cloudwatch is a wrapper around the aws cloudwatch service with some default config info type Cloudwatch struct { + session *session.Session Service cloudwatchiface.CloudWatchAPI } -// NewSession creates a new cloudwatch session -func NewSession(account common.Account) Cloudwatch { - c := Cloudwatch{} - log.Infof("creating new aws session for costexplorer with key id %s in region %s", account.Akid, account.Region) - sess := session.Must(session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(account.Akid, account.Secret, ""), - Region: aws.String(account.Region), - })) - c.Service = cloudwatch.New(sess) - return c +type CloudwatchOption func(*Cloudwatch) + +func New(opts ...CloudwatchOption) *Cloudwatch { + client := Cloudwatch{} + + for _, opt := range opts { + opt(&client) + } + + if client.session != nil { + client.Service = cloudwatch.New(client.session) + } + + return &client +} + +func WithSession(sess *session.Session) CloudwatchOption { + return func(client *Cloudwatch) { + log.Debug("using aws session") + client.session = sess + } +} + +func WithCredentials(key, secret, token, region string) CloudwatchOption { + return func(client *Cloudwatch) { + log.Debugf("creating new session with key id %s in region %s", key, region) + sess := session.Must(session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(key, secret, token), + Region: aws.String(region), + })) + client.session = sess + } } diff --git a/cloudwatch/cloudwatch_test.go b/cloudwatch/cloudwatch_test.go index 196187a..4e166ad 100644 --- a/cloudwatch/cloudwatch_test.go +++ b/cloudwatch/cloudwatch_test.go @@ -4,7 +4,6 @@ import ( "reflect" "testing" - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface" ) @@ -23,8 +22,8 @@ func newmockCloudwatchClient(t *testing.T, err error) cloudwatchiface.CloudWatch } func TestNewSession(t *testing.T) { - e := NewSession(common.Account{}) - if to := reflect.TypeOf(e).String(); to != "cloudwatch.Cloudwatch" { + e := New() + if to := reflect.TypeOf(e).String(); to != "*cloudwatch.Cloudwatch" { t.Errorf("expected type to be 'cloudwatch.Cloudwatch', got %s", to) } } diff --git a/common/config.go b/common/config.go index 4e79bf3..a93958e 100644 --- a/common/config.go +++ b/common/config.go @@ -10,16 +10,17 @@ import ( // Config is representation of the configuration data type Config struct { - ListenAddress string - Accounts map[string]Account Account Account - Token string - LogLevel string - Version Version - Org string + Accounts map[string]Account + AccountsMap map[string]string CacheExpireTime string CachePurgeTime string ImageCache *S3Cache + ListenAddress string + LogLevel string + Org string + Token string + Version Version } // Account is the configuration for an individual account diff --git a/config/config.example.json b/config/config.example.json index 0decd94..6aa9c40 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -1,11 +1,11 @@ { "listenAddress": ":8080", - "accounts": { - "spinup": { - "region": "us-east-1", - "akid": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", - "secret": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy" - } + "account": { + "region": "us-east-1", + "akid": "xxxxxxxxxxxxxxxxxxxxxxxx", + "secret": "yyyyyyyyyyyyyyyyyyyyyyyyyyyyyy", + "externalId": "xxxxxxx-yyyy-zzzz-aaaa-bbbbbbbbb", + "role": "someRole" }, "imageCache": { "region": "us-east-1", @@ -15,6 +15,11 @@ "prefix": "costapi", "hashingToken": "xxxxxxxx-yyyy-zzzz-aaaa-bbbbbbbbbbb" }, + "accountsMap": { + "spinup": "1234567890", + "spinupsec": "0987654321", + "sandbox": "00000000000" + }, "token": "xxxxxxxx-yyyy-zzzz-aaaa-bbbbbbbbbbb", "logLevel": "debug", "org": "localdev", diff --git a/costexplorer/costexplorer.go b/costexplorer/costexplorer.go index 6f7b26b..d4e356b 100644 --- a/costexplorer/costexplorer.go +++ b/costexplorer/costexplorer.go @@ -1,7 +1,6 @@ package costexplorer import ( - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -12,17 +11,40 @@ import ( // CostExplorer is a wrapper around the aws costexplorer service with some default config info type CostExplorer struct { + session *session.Session Service costexploreriface.CostExplorerAPI } -// NewSession creates a new costexplorer session -func NewSession(account common.Account) CostExplorer { - c := CostExplorer{} - log.Infof("creating new aws session for costexplorer with key id %s in region %s", account.Akid, account.Region) - sess := session.Must(session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(account.Akid, account.Secret, ""), - Region: aws.String(account.Region), - })) - c.Service = costexplorer.New(sess) - return c +type CostExplorerOption func(*CostExplorer) + +func New(opts ...CostExplorerOption) *CostExplorer { + client := CostExplorer{} + + for _, opt := range opts { + opt(&client) + } + + if client.session != nil { + client.Service = costexplorer.New(client.session) + } + + return &client +} + +func WithSession(sess *session.Session) CostExplorerOption { + return func(client *CostExplorer) { + log.Debug("using aws session") + client.session = sess + } +} + +func WithCredentials(key, secret, token, region string) CostExplorerOption { + return func(client *CostExplorer) { + log.Debugf("creating new session with key id %s in region %s", key, region) + sess := session.Must(session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(key, secret, token), + Region: aws.String(region), + })) + client.session = sess + } } diff --git a/costexplorer/costexplorer_test.go b/costexplorer/costexplorer_test.go index 843a477..b590220 100644 --- a/costexplorer/costexplorer_test.go +++ b/costexplorer/costexplorer_test.go @@ -4,7 +4,6 @@ import ( "reflect" "testing" - "github.com/YaleSpinup/cost-api/common" "github.com/aws/aws-sdk-go/service/costexplorer/costexploreriface" ) @@ -23,8 +22,8 @@ func newmockCostExplorerClient(t *testing.T, err error) costexploreriface.CostEx } func TestNewSession(t *testing.T) { - e := NewSession(common.Account{}) - if to := reflect.TypeOf(e).String(); to != "costexplorer.CostExplorer" { - t.Errorf("expected type to be 'costexplorer.CostExplorer', got %s", to) + e := New() + if to := reflect.TypeOf(e).String(); to != "*costexplorer.CostExplorer" { + t.Errorf("expected type to be '*costexplorer.CostExplorer', got %s", to) } } diff --git a/docker/config.deco.json b/docker/config.deco.json index 201b8ab..c588ad7 100644 --- a/docker/config.deco.json +++ b/docker/config.deco.json @@ -1,17 +1,5 @@ { "listenAddress": ":8080", - "accounts": { - "spinup": { - "region": "us-east-1", - "akid": "{{ .spinup_akid }}", - "secret": "{{ .spinup_secret }}" - }, - "spinupsec": { - "region": "us-east-1", - "akid": "{{ .spinupsec_akid }}", - "secret": "{{ .spinupsec_secret }}" - } - }, "account": { "region": "us-east-1", "akid": "{{ .akid }}", @@ -19,6 +7,10 @@ "externalId": "{{ .external_id }}", "role": "{{ .cross_account_role }}" }, + "accountsMap": { + "spinup": "{{ .spinup_account_number }}", + "spinupsec": "{{ .spinupsec_account_number }}" + }, "imageCache": { "region": "us-east-1", "bucket": "{{ .imagecache_bucket }}", diff --git a/go.mod b/go.mod index d52701f..f09190c 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,36 @@ module github.com/YaleSpinup/cost-api -go 1.16 +go 1.17 require ( github.com/YaleSpinup/apierror v0.1.0 github.com/YaleSpinup/aws-go v0.1.0 - github.com/aws/aws-sdk-go v1.38.65 - github.com/felixge/httpsnoop v1.0.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect - github.com/google/uuid v1.2.0 + github.com/aws/aws-sdk-go v1.42.11 + github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 - github.com/kr/text v0.2.0 // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/common v0.29.0 // indirect github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.8.1 + golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/felixge/httpsnoop v1.0.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/stretchr/testify v1.7.0 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e - golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect + golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect ) diff --git a/go.sum b/go.sum index d17490c..4393064 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/aws/aws-sdk-go v1.38.50/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.65 h1:umGu5gjIOKxzhi34T0DIA1TWupUDjV2aAW5vK6154Gg= github.com/aws/aws-sdk-go v1.38.65/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.42.11 h1:5wfKuNcbch3IFZth5+j2Ud/+UOxCR0zfgLGPoiK1p4s= +github.com/aws/aws-sdk-go v1.42.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -52,6 +54,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -132,6 +136,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= @@ -195,11 +201,15 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -231,6 +241,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -293,6 +305,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -348,6 +362,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab h1:rfJ1bsoJQQIAoAxTxB7bme+vHrNkRw8CqfsYh9w54cw= +golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -480,6 +496,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=