Skip to content

Commit

Permalink
Add inventory endpoint and support for pulling details about resource…
Browse files Browse the repository at this point in the history
…s by spaceid (#26)
  • Loading branch information
fishnix authored Jul 26, 2021
1 parent e0ea54d commit 7c8c701
Show file tree
Hide file tree
Showing 11 changed files with 633 additions and 1 deletion.
56 changes: 56 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ GET /v1/cost/{account}/instances/{id}/metrics/graph.png?metric={metric1}[&metric
GET /v1/cost/{account}/instances/{id}/metrics/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300]
##################
GET /v1/inventory/{account}/spaces/{spaceid}
GET /v1/metrics/{account}/instances/{id}/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300]
GET /v1/metrics/{account}/clusters/{cluster}/services/{service}/graph?metric={metric1}[&metric={metric2}&start=-P1D&end=PT0H&period=300]
GET /v1/metrics/{account}/buckets/{bucket}/graph?metric={BucketSizeBytes|NumberOfObjects}
Expand Down Expand Up @@ -675,6 +677,60 @@ Empty response is usually a result of not enough data for a recommendation.
]
```

## Inventory Usage

The inventory endpoint returns resources belonging to a space by tag. It uses the resourcegroupstaggingapi and also parses the ARN to
determine the resource type information. Since the `resource` prefix is inconsistent, it shouldn't be relied upon for categorization, but the
`service` should be accurate.

### Request

GET /v1/inventory/{account}/spaces/{spaceid}

### Response

```json
[
{
"name": "spintst-000a16-TestFS",
"arn": "arn:aws:elasticfilesystem:us-east-1:1234567890:file-system/fs-aaaabbbb11",
"partition": "aws",
"service": "elasticfilesystem",
"region": "us-east-1",
"account_id": "1234567890",
"resource": "file-system/fs-aaaabbbb11"
},
{
"name": "",
"arn": "arn:aws:elasticloadbalancing:us-east-1:1234567890:targetgroup/testTargetGroup-HTTP80/abcdefg12",
"partition": "aws",
"service": "elasticloadbalancing",
"region": "us-east-1",
"account_id": "1234567890",
"resource": "targetgroup/testTargetGroup-HTTP80/abcdefg12"
},
{
"name": "spintst-000028",
"arn": "arn:aws:logs:us-east-1:1234567890:log-group:spintst-000028",
"partition": "aws",
"service": "logs",
"region": "us-east-1",
"account_id": "1234567890",
"resource": "log-group:spintst-000028"
},
{
"name": "spintst-000b67-webServiceTest",
"arn": "arn:aws:secretsmanager:us-east-1:1234567890:secret:spinup/sstst/spintst-000028/spintst-000b67-webServiceTest-api-cred-ibdIk7",
"partition": "aws",
"service": "secretsmanager",
"region": "us-east-1",
"account_id": "1234567890",
"resource": "secret:spinup/sstst/spintst-000028/spintst-000b67-webServiceTest-api-cred-ibdIk7"
}
]
```


## Metrics Usage

### Get cloudwatch metrics widgets URL from S3 for an instance ID
Expand Down
58 changes: 58 additions & 0 deletions api/handlers_inventory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package api

import (
"encoding/json"
"fmt"
"net/http"
"strconv"

"github.com/YaleSpinup/apierror"
"github.com/YaleSpinup/cost-api/resourcegroupstaggingapi"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
)

// SpaceInventoryGetHandler handles getting the inventory for resources with a spaceid
func (s *server) SpaceInventoryGetHandler(w http.ResponseWriter, r *http.Request) {
w = LogWriter{w}
vars := mux.Vars(r)
account := vars["account"]
spaceID := vars["space"]

role := fmt.Sprintf("arn:aws:iam::%s:role/%s", account, s.session.RoleName)
session, err := s.assumeRole(
r.Context(),
s.session.ExternalID,
role,
"",
"arn:aws:iam::aws:policy/AWSResourceGroupsReadOnlyAccess",
)
if err != nil {
msg := fmt.Sprintf("failed to assume role in account: %s", account)
handleError(w, apierror.New(apierror.ErrForbidden, msg, nil))
return
}

orch := newInventoryOrchestrator(
resourcegroupstaggingapi.New(resourcegroupstaggingapi.WithSession(session.Session)),
s.org,
)

out, err := orch.GetResourceInventory(r.Context(), account, spaceID)
if err != nil {
handleError(w, err)
return
}

j, err := json.Marshal(out)
if err != nil {
log.Errorf("cannot marshal response (%v) into JSON: %s", out, err)
w.WriteHeader(http.StatusInternalServerError)
return
}

w.Header().Set("X-Items", strconv.Itoa(len(out)))
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(j)
}
50 changes: 50 additions & 0 deletions api/orchestration_inventory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package api

import (
"context"

"github.com/YaleSpinup/apierror"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
log "github.com/sirupsen/logrus"
)

func (o *inventoryOrchestrator) GetResourceInventory(ctx context.Context, account, spaceid string) ([]*InventoryResponse, error) {
if spaceid == "" {
return nil, apierror.New(apierror.ErrBadRequest, "spaceid is required", nil)
}

list := []*InventoryResponse{}
input := resourcegroupstaggingapi.GetResourcesInput{
TagFilters: []*resourcegroupstaggingapi.TagFilter{
{
Key: aws.String("spinup:spaceid"),
Values: []*string{aws.String(spaceid)},
},
},
ResourcesPerPage: aws.Int64(100),
}

for {
out, err := o.client.ListResourcesWithTags(ctx, &input)
if err != nil {
return nil, err
}

for _, res := range out.ResourceTagMappingList {
list = append(list, toInventoryResponse(res))
}

log.Debugf("%+v:%s", out.PaginationToken, aws.StringValue(out.PaginationToken))

if aws.StringValue(out.PaginationToken) == "" {
break
}

input.PaginationToken = out.PaginationToken

log.Debugf("%d resources in list", len(list))
}

return list, nil
}
13 changes: 13 additions & 0 deletions api/orchestrators.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package api
import (
"github.com/YaleSpinup/cost-api/budgets"
"github.com/YaleSpinup/cost-api/computeoptimizer"
"github.com/YaleSpinup/cost-api/resourcegroupstaggingapi"
"github.com/YaleSpinup/cost-api/sns"
)

Expand Down Expand Up @@ -31,3 +32,15 @@ func newOptimizerOrchestrator(client *computeoptimizer.ComputeOptimizer, org str
org: org,
}
}

type inventoryOrchestrator struct {
client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI
org string
}

func newInventoryOrchestrator(client *resourcegroupstaggingapi.ResourceGroupsTaggingAPI, org string) *inventoryOrchestrator {
return &inventoryOrchestrator{
client: client,
org: org,
}
}
8 changes: 8 additions & 0 deletions api/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,14 @@ func (s *server) routes() {
metricsApi.HandleFunc("/{account}/buckets/{bucket}/graph", s.GetS3MetricsURLHandler).Queries("metric", "{metric:(?:BucketSizeBytes|NumberOfObjects)}").Methods(http.MethodGet)
// metrics endpoints for RDS services
metricsApi.HandleFunc("/{account}/rds/{type}/{id}/graph", s.GetRDSMetricsURLHandler).Methods(http.MethodGet)

inventoryApi := s.router.PathPrefix("/v1/inventory").Subrouter()
inventoryApi.HandleFunc("/ping", s.PingHandler).Methods(http.MethodGet)
inventoryApi.HandleFunc("/version", s.VersionHandler).Methods(http.MethodGet)
inventoryApi.Handle("/metrics", promhttp.Handler()).Methods(http.MethodGet)

// inventory endpoints for a space
inventoryApi.HandleFunc("/{account}/spaces/{space}", s.SpaceInventoryGetHandler).Methods(http.MethodGet)
}

// custom matcher for space queries
Expand Down
37 changes: 37 additions & 0 deletions api/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/service/budgets"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/sns"
log "github.com/sirupsen/logrus"
)

// BudgetCreateRequest is the request object to create a Budget
Expand Down Expand Up @@ -113,3 +115,38 @@ func toSnsTag(tags []*Tag) []*sns.Tag {

return snsTags
}

type InventoryResponse struct {
Name string `json:"name"`
ARN string `json:"arn"`
Partition string `json:"partition"`
Service string `json:"service"`
Region string `json:"region"`
AccountID string `json:"account_id"`
Resource string `json:"resource"`
}

func toInventoryResponse(i *resourcegroupstaggingapi.ResourceTagMapping) *InventoryResponse {
var name string
for _, tag := range i.Tags {
if aws.StringValue(tag.Key) == "Name" {
name = aws.StringValue(tag.Value)
}
}

resourceArn := aws.StringValue(i.ResourceARN)
a, err := arn.Parse(resourceArn)
if err != nil {
log.Warnf("failed to parse resource ARN: %s", resourceArn)
}

return &InventoryResponse{
Name: name,
ARN: aws.StringValue(i.ResourceARN),
Partition: a.Partition,
Service: a.Service,
Region: a.Region,
AccountID: a.AccountID,
Resource: a.Resource,
}
}
2 changes: 1 addition & 1 deletion k8s/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ ingress:
enabled: true
annotations: {}
rules:
- paths: ['/v1/cost', '/v1/metrics']
- paths: ['/v1/cost', '/v1/metrics', '/v1/inventory']

probePath: '/v1/cost/ping'
97 changes: 97 additions & 0 deletions resourcegroupstaggingapi/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package resourcegroupstaggingapi

import (
"github.com/YaleSpinup/apierror"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/pkg/errors"
)

func ErrCode(msg string, err error) error {
if aerr, ok := errors.Cause(err).(awserr.Error); ok {
switch aerr.Code() {
case

// ErrCodeInternalServiceException for service response error code
// "InternalServiceException".
//
// The request processing failed because of an unknown error, exception, or
// failure. You can retry the request.
resourcegroupstaggingapi.ErrCodeInternalServiceException:

return apierror.New(apierror.ErrInternalError, msg, err)
case

// ErrCodeConcurrentModificationException for service response error code
// "ConcurrentModificationException".
//
// The target of the operation is currently being modified by a different request.
// Try again later.
resourcegroupstaggingapi.ErrCodeConcurrentModificationException,

// ErrCodeThrottledException for service response error code
// "ThrottledException".
//
// The request was denied to limit the frequency of submitted requests.
resourcegroupstaggingapi.ErrCodeThrottledException:
return apierror.New(apierror.ErrConflict, msg, aerr)
case

// ErrCodeConstraintViolationException for service response error code
// "ConstraintViolationException".
//
// The request was denied because performing this operation violates a constraint.
//
// Some of the reasons in the following list might not apply to this specific
// operation.
//
// * You must meet the prerequisites for using tag policies. For information,
// see Prerequisites and Permissions for Using Tag Policies (http://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_tag-policies-prereqs.html)
// in the AWS Organizations User Guide.
//
// * You must enable the tag policies service principal (tagpolicies.tag.amazonaws.com)
// to integrate with AWS Organizations For information, see EnableAWSServiceAccess
// (http://docs.aws.amazon.com/organizations/latest/APIReference/API_EnableAWSServiceAccess.html).
//
// * You must have a tag policy attached to the organization root, an OU,
// or an account.
resourcegroupstaggingapi.ErrCodeConstraintViolationException,

// ErrCodeInvalidParameterException for service response error code
// "InvalidParameterException".
//
// This error indicates one of the following:
//
// * A parameter is missing.
//
// * A malformed string was supplied for the request parameter.
//
// * An out-of-range value was supplied for the request parameter.
//
// * The target ID is invalid, unsupported, or doesn't exist.
//
// * You can't access the Amazon S3 bucket for report storage. For more information,
// see Additional Requirements for Organization-wide Tag Compliance Reports
// (http://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_tag-policies-prereqs.html#bucket-policies-org-report)
// in the AWS Organizations User Guide.
resourcegroupstaggingapi.ErrCodeInvalidParameterException,

// ErrCodePaginationTokenExpiredException for service response error code
// "PaginationTokenExpiredException".
//
// A PaginationToken is valid for a maximum of 15 minutes. Your request was
// denied because the specified PaginationToken has expired.
resourcegroupstaggingapi.ErrCodePaginationTokenExpiredException:

return apierror.New(apierror.ErrBadRequest, msg, aerr)
case
"Not Found":
return apierror.New(apierror.ErrNotFound, msg, aerr)
default:
m := msg + ": " + aerr.Message()
return apierror.New(apierror.ErrBadRequest, m, aerr)
}
}

return apierror.New(apierror.ErrInternalError, msg, err)
}
Loading

0 comments on commit 7c8c701

Please sign in to comment.