diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 00000000..31dfc714 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: 2024 Comcast Cable Communications Management, LLC +# SPDX-License-Identifier: Apache-2.0 +--- +name: 'Auto Release' + +on: + schedule: # Run every day at 12:00 UTC + - cron: '0 12 * * *' + workflow_dispatch: + +jobs: + release: + uses: xmidt-org/shared-go/.github/workflows/auto-releaser.yml@b667178a63b2cf6d61739d53fa1e3f200adfb12e # v4.4.20 + secrets: inherit diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e7a8d1..f1cfdead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,14 @@ jobs: uses: xmidt-org/shared-go/.github/workflows/ci.yml@b667178a63b2cf6d61739d53fa1e3f200adfb12e # v4.4.20 with: release-type: program + release-arch-arm64: false release-docker: true release-docker-latest: true release-docker-major: true release-docker-minor: true + release-docker-extras: | + .release/docker + LICENSE + NOTICE + yaml-lint-skip: false secrets: inherit diff --git a/.gitignore b/.gitignore index 66dd7490..00958c1d 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ report.json # for releases .ignore/ +dist/ +.goreleaser.yml # vendor package vendor/ @@ -27,4 +29,4 @@ vendor/ deploy/docker-compose/ # misc -coverage.txt \ No newline at end of file +coverage.txt diff --git a/.golangci.yaml b/.golangci.yaml index 00c37323..60c10e4a 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -28,4 +28,3 @@ issues: linters: - dupl - funlen - diff --git a/.release/docker/argus_spruce.yaml b/.release/docker/argus_spruce.yaml index a14a4d09..f8a0aa5a 100644 --- a/.release/docker/argus_spruce.yaml +++ b/.release/docker/argus_spruce.yaml @@ -70,7 +70,7 @@ servers: X-Xmidt-Version: - development -store: +store: # dynamo is the configuration block to communicate with dynamoDB. dynamo: @@ -109,7 +109,7 @@ store: # # If the opTimeout is set to 0, it defaults to 10s. # # (Optional) defaults to 10s # opTimeout: 100ms - + # # username is the username to use when connecting to the database. # # (Optional) # username: "cassandra" @@ -142,7 +142,7 @@ store: # userInputValidation groups options around validating data on incoming requests. # (Optional) The default values are those listed above the fields below. userInputValidation: - # itemMaxTTL defines the limit for TTL values provided by users of the API. + # itemMaxTTL defines the limit for TTL values provided by users of the API. # refer to https://golang.org/pkg/time/#ParseDuration for valid strings. # (Optional) default: 24h (a day) itemMaxTTL: (( grab $ITEM_MAX_TTL || "24h" )) @@ -156,7 +156,7 @@ userInputValidation: ownerFormatRegex: (( grab $OWNER_FORMAT_REGEX || "^.{4,60}$" )) # itemDataMaxDepth is the max allowed depth of the Item JSON data field. - # If your DB supports up to N nested objects, itemDataMaxDepth should be set to + # If your DB supports up to N nested objects, itemDataMaxDepth should be set to # N-1. The value of itemDataMaxDepth must be > 0, otherwise the default value will # be used. # (Optional) default: 30 @@ -170,7 +170,7 @@ jwtValidator: Resolve: Template: (( grab $THEMIS_ENDPOINT || "http://themis:6500/keys/{keyID}" )) -# can we remove all of this section? +# can we remove all of this section? authx: inbound: # basic is a list of Basic Auth credentials intended to be used for local testing purposes. @@ -178,13 +178,13 @@ authx: basic: - (( grab $AUTH_HEADER || "dXNlcjpwYXNz" )) - + # accessLevel defines config around the injection of an attribute to bascule tokens # which application code can leverage to decide if a given request is allowed to execute some operation. - # Note that accessLevel differs from capabilityCheck in that it allows more complex access hierarchy. - # That is, while capabilityCheck verifies whether a user is allowed to use an API endpoint, accessLevel + # Note that accessLevel differs from capabilityCheck in that it allows more complex access hierarchy. + # That is, while capabilityCheck verifies whether a user is allowed to use an API endpoint, accessLevel # assigns a number to the user's request which application code can use for security purposes. - # An access level is defined as a non-negative number and the higher the number, the higher the access the + # An access level is defined as a non-negative number and the higher the number, the higher the access the # request has for the target application. # (Optional). If section is not provided, the lowest access level value of 0 will be assigned to the attribute. accessLevel: @@ -201,7 +201,7 @@ authx: # If this value is found in the list, the access level assigned to the request will be 1. Otherwise, it will be 0. # (Optional) defaults to 'xmidt:svc:admin' name: (( grab $ACCESS_LEVEL_CAPABILITY || "xmidt:svc:admin" )) - + # path is the list of nested keys to get to the claim which contains the capabilities. # For example, if your JWT payload looks like this: # ``` @@ -211,24 +211,24 @@ authx: # "my_company": { # "capabilities": ["capability0", "capability1"] # } - # } - # ``` + # } + # ``` # you'll want to set path to ["my_company", "capabilities"] # (Optional) default: ["capabilities"] - path: + path: - (( grab $ACCESS_LEVEL_CAPABILITIES_PATH || "capabilities" )) # # capabilityCheck provides the details needed for checking an incoming JWT's # # capabilities. If the type of check isn't provided, no checking is done. The - # # type can be "monitor" or "enforce". If "monitor" is provided, the capabilities - # # are checked but the request isn't rejected when there isn't a valid capability - # # for the request. Instead, a message is logged. When "enforce" is provided, a + # # type can be "monitor" or "enforce". If "monitor" is provided, the capabilities + # # are checked but the request isn't rejected when there isn't a valid capability + # # for the request. Instead, a message is logged. When "enforce" is provided, a # # request that doesn't have the needed capability is rejected. - + # # The capability is expected to have the format: - + # # {prefix}{endpoint}:{method} - + # # The prefix can be a regular expression. If it's empty, no capability check # # is done. The endpoint is a regular expression that should match the endpoint # # the request was sent to. The method is usually the method of the request, such as @@ -262,4 +262,3 @@ tracing: # endpoint is where trace information should be routed. Applies to zipkin and jaegar. endpoint: (( grab $TRACING_PROVIDER_ENDPOINT || "http://zipkin:9411/api/v2/spans" )) - diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 00000000..fcc23fd7 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2024 Comcast Cable Communications Management, LLC +# SPDX-License-Identifier: Apache-2.0 +--- + +extends: default + +ignore: + - .release/helm/argus/templates/argus.yaml + +rules: + braces: + level: warning + max-spaces-inside: 1 + brackets: + level: warning + max-spaces-inside: 1 + colons: + level: warning + max-spaces-after: -1 + commas: + level: warning + comments: disable + comments-indentation: disable + document-start: + present: true + empty-lines: + max: 2 + hyphens: + max-spaces-after: 1 + indentation: + level: error + indent-sequences: consistent + line-length: + level: warning + max: 90 + allow-non-breakable-words: true + allow-non-breakable-inline-mappings: true + truthy: disable diff --git a/Dockerfile b/Dockerfile index c142467f..f9011b1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,20 @@ -## SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC +## SPDX-FileCopyrightText: 2022 Comcast Cable Communications Management, LLC ## SPDX-License-Identifier: Apache-2.0 -FROM docker.io/library/golang:1.19-alpine as builder +FROM docker.io/library/golang:1.19-alpine AS builder WORKDIR /src -ARG VERSION -ARG GITCOMMIT -ARG BUILDTIME - RUN apk add --no-cache --no-progress \ ca-certificates \ - make \ - curl \ - git \ - openssh \ - gcc \ - libc-dev + curl # Download spruce here to eliminate the need for curl in the final image RUN mkdir -p /go/bin && \ - arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \ - curl -L -o /go/bin/spruce https://github.com/geofffranks/spruce/releases/download/v1.30.2/spruce-linux-${arch} && \ - chmod +x /go/bin/spruce && \ - sha1sum /go/bin/spruce + curl -L -o /go/bin/spruce https://github.com/geofffranks/spruce/releases/download/v1.29.0/spruce-linux-amd64 && \ + chmod +x /go/bin/spruce COPY . . -RUN make test release - ########################## # Build the final image. ########################## @@ -36,18 +23,17 @@ FROM alpine:latest # Copy over the standard things you'd expect. COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=builder /src/argus / -COPY --from=builder /src/.release/docker/entrypoint.sh / +COPY argus / +COPY .release/docker/entrypoint.sh / # Copy over spruce and the spruce template file used to make the actual configuration file. -COPY --from=builder /src/.release/docker/argus_spruce.yaml /tmp/argus_spruce.yaml -COPY --from=builder /go/bin/spruce /bin/ +COPY .release/docker/argus_spruce.yaml /tmp/argus_spruce.yaml +COPY --from=builder /go/bin/spruce /bin/ # Include compliance details about the container and what it contains. -COPY --from=builder /src/Dockerfile \ - /src/NOTICE \ - /src/LICENSE \ - /src/CHANGELOG.md / +COPY Dockerfile / +COPY NOTICE / +COPY LICENSE / # Make the location for the configuration file that will be used. RUN mkdir /etc/argus/ \ diff --git a/argus.yaml b/argus.yaml index 88aab638..092c1bba 100644 --- a/argus.yaml +++ b/argus.yaml @@ -1,5 +1,6 @@ ## SPDX-FileCopyrightText: 2021 Comcast Cable Communications Management, LLC ## SPDX-License-Identifier: Apache-2.0 +--- prometheus: defaultNamespace: xmidt defaultSubsystem: argus @@ -56,13 +57,13 @@ store: # endpoint is used to set a custom aws endpoint. # (Optional) endpoint: "http://localhost:8000" - + # table is the name of the table that is already configured with bucket and id as the key. table: "gifnoc" - + # region is where request should go to. region: "us-east-2" - + # maxRetires is the maximum times the application will retry the request to the db. # (Optional) default: 3 maxRetries: 3 @@ -70,13 +71,16 @@ store: # getAllLimit is the maximum number of items to get at a time. # (Optional) defaults to no limit getAllLimit: 50 - + # accessKey is the AWS accessKey to access dynamodb. accessKey: "accessKey" - + # secretKey is the AWS secretKey to go with the accessKey to access dynamodb. secretKey: "secretKey" + # If roleBasedAccess is enabled, accessKey and secretKey will be fetched using IAM temporary credentials + roleBasedAccess: false + #yugabyte: # # hosts is and array of address and port used to connect to the cluster. # hosts: @@ -87,7 +91,7 @@ store: # # If the opTimeout is set to 0, it defaults to 10s. # # (Optional) defaults to 10s # opTimeout: 100ms - + # # username is the username to use when connecting to the database. # # (Optional) # username: "cassandra" @@ -121,10 +125,10 @@ store: # userInputValidation groups options around validating data on incoming requests. # (Optional) The default values are those listed above the fields below. userInputValidation: - # itemMaxTTL defines the limit for TTL values provided by users of the API. + # itemMaxTTL defines the limit for TTL values provided by users of the API. # refer to https://golang.org/pkg/time/#ParseDuration for valid strings. # (Optional) default: 24h (a day) - itemMaxTTL: "24h" + itemMaxTTL: "24h" # bucketFormatRegex helps define the validity of a bucket through a regular expression. # (Optional) default: ^[0-9a-z][0-9a-z-]{1,61}[0-9a-z]$ @@ -135,7 +139,7 @@ userInputValidation: ownerFormatRegex: "^.{4,60}$" # itemDataMaxDepth is the max allowed depth of the Item JSON data field. - # If your DB supports up to N nested objects, itemDataMaxDepth should be set to + # If your DB supports up to N nested objects, itemDataMaxDepth should be set to # N-1. The value of itemDataMaxDepth must be > 0, otherwise the default value will # be used. # (Optional) default: 30 @@ -167,7 +171,7 @@ authx: basic: ["dXNlcjpwYXNz"] # bearer contains all the configuration needed for a JWT validator. - bearer: + bearer: key: factory: uri: "http://localhost:6500/keys/docker" @@ -176,10 +180,10 @@ authx: # accessLevel defines config around the injection of an attribute to bascule tokens # which application code can leverage to decide if a given request is allowed to execute some operation. - # Note that accessLevel differs from capabilityCheck in that it allows more complex access hierarchy. - # That is, while capabilityCheck verifies whether a user is allowed to use an API endpoint, accessLevel + # Note that accessLevel differs from capabilityCheck in that it allows more complex access hierarchy. + # That is, while capabilityCheck verifies whether a user is allowed to use an API endpoint, accessLevel # assigns a number to the user's request which application code can use for security purposes. - # An access level is defined as a non-negative number and the higher the number, the higher the access the + # An access level is defined as a non-negative number and the higher the number, the higher the access the # request has for the target application. # (Optional). If section is not provided, the lowest access level value of 0 will be assigned to the attribute. accessLevel: @@ -196,7 +200,7 @@ authx: # If this value is found in the list, the access level assigned to the request will be 1. Otherwise, it will be 0. # (Optional) defaults to 'xmidt:svc:admin' name: "xmidt:svc:admin" - + # path is the list of nested keys to get to the claim which contains the capabilities. # For example, if your JWT payload looks like this: # ``` @@ -206,8 +210,8 @@ authx: # "my_company": { # "capabilities": ["capability0", "capability1"] # } - # } - # ``` + # } + # ``` # you'll want to set path to ["my_company", "capabilities"] # (Optional) default: ["capabilities"] path: ["capabilities"] @@ -215,15 +219,15 @@ authx: # # capabilities provides the details needed for checking an incoming JWT's # # capabilities. If the type of check isn't provided, no checking is done. The - # # type can be "monitor" or "enforce". If "monitor" is provided, the capabilities - # # are checked but the request isn't rejected when there isn't a valid capability - # # for the request. Instead, a message is logged. When "enforce" is provided, a + # # type can be "monitor" or "enforce". If "monitor" is provided, the capabilities + # # are checked but the request isn't rejected when there isn't a valid capability + # # for the request. Instead, a message is logged. When "enforce" is provided, a # # request that doesn't have the needed capability is rejected. - + # # The capability is expected to have the format: - + # # {prefix}{endpoint}:{method} - + # # The prefix can be a regular expression. If it's empty, no capability check # # is done. The endpoint is a regular expression that should match the endpoint # # the request was sent to. The method is usually the method of the request, such as @@ -258,4 +262,3 @@ tracing: # endpoint is where trace information should be routed. Applies to otlp, zipkin, and jaegar. OTLP/gRPC uses port 4317 by default. # OTLP/HTTP uses port 4318 by default. # endpoint: "http://localhost:9411/api/v2/spans" - diff --git a/integtests/dynamodb/docker-compose.yml b/integtests/dynamodb/docker-compose.yml index 2cfebb13..bc99be8f 100644 --- a/integtests/dynamodb/docker-compose.yml +++ b/integtests/dynamodb/docker-compose.yml @@ -8,4 +8,4 @@ services: hostname: dynamodb-local container_name: dynamodb-local ports: - - "8042:8000" \ No newline at end of file + - "8042:8000" diff --git a/store/dynamodb/db.go b/store/dynamodb/db.go index 6349c71b..637690dc 100644 --- a/store/dynamodb/db.go +++ b/store/dynamodb/db.go @@ -4,12 +4,15 @@ package dynamodb import ( "errors" + "fmt" "net/http" + "os" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" "github.com/go-playground/validator/v10" "github.com/xmidt-org/argus/model" "github.com/xmidt-org/argus/store" @@ -69,6 +72,9 @@ type Config struct { // dual stack (IPv4 and IPv6). // (Optional) Defaults to False. DisableDualStack bool + + // If roleBasedAccess is enabled, accessKey and secretKey will be fetched using IAM temporary credentials + RoleBasedAccess bool } // dao adapts the underlying dynamodb data service to match @@ -89,16 +95,44 @@ func NewDynamoDB(config Config, measures metric.Measures) (store.S, error) { return nil, err } + var creds credentials.Value + if config.RoleBasedAccess { + awsRegion, err := getAwsRegionForRoleBasedAccess(config) + if err != nil { + return nil, err + } + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(awsRegion)}, + ) + if err != nil { + return nil, err + } + + value, err := sess.Config.Credentials.Get() + if err != nil { + return nil, err + } + + creds = credentials.Value{ + AccessKeyID: value.AccessKeyID, + SecretAccessKey: value.SecretAccessKey, + SessionToken: value.SessionToken, + } + } else { + creds = credentials.Value{ + AccessKeyID: config.AccessKey, + SecretAccessKey: config.SecretKey, + } + } + awsConfig := *aws.NewConfig(). WithEndpoint(config.Endpoint). WithUseDualStack(!config.DisableDualStack). WithMaxRetries(config.MaxRetries). WithCredentialsChainVerboseErrors(true). WithRegion(config.Region). - WithCredentials(credentials.NewStaticCredentialsFromCreds(credentials.Value{ - AccessKeyID: config.AccessKey, - SecretAccessKey: config.SecretKey, - })) + WithCredentials(credentials.NewStaticCredentialsFromCreds(creds)) svc, err := newService(awsConfig, "", config.Table, int64(config.GetAllLimit), &measures) if err != nil { @@ -111,6 +145,20 @@ func NewDynamoDB(config Config, measures metric.Measures) (store.S, error) { }, nil } +func getAwsRegionForRoleBasedAccess(config Config) (string, error) { + awsRegion := config.Region + + if len(awsRegion) == 0 { + awsRegion = os.Getenv("AWS_REGION") + } + + if len(awsRegion) == 0 { + return "", fmt.Errorf("%s", "Aws region is not provided") + } + + return awsRegion, nil +} + func (d dao) Push(key model.Key, item store.OwnableItem) error { _, err := d.s.Push(key, item) return sanitizeError(err)