diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1b17104 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM --platform=$BUILDPLATFORM golang:1.22.0 as builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /token-refresher + +COPY . . + +RUN go vet ./... && \ + go test -v -race ./... && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o token-refresher + +FROM alpine + +WORKDIR /token-refresher + +RUN addgroup token-refresher \ + && adduser -u 1000 -S -g 1000 token-refresher --ingroup token-refresher \ + && chown -R token-refresher:token-refresher /token-refresher + +USER token-refresher + +COPY --from=builder /token-refresher/token-refresher /usr/local/bin/token-refresher + +ENTRYPOINT ["token-refresher"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..756cccf --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# Overview + +`service-account-token-refresher` is a light-weight sidecar designed to ensure the continuous validity of the service account token projected by kubelet. It takes on the role of renewing the token when kubelet doesn't, addressing a known issue in Kubernetes where it ceases to refresh the token as a pod enters termination phase. + +For more details on the issue, visit: [Kubelet stops rotating service account tokens when pod is terminating, breaking preStop hooks](https://github.com/kubernetes/kubernetes/issues/116481) + +This issue particularly affects pods that require a significant amount of time, potentially hours or days, to shut down gracefully after Kubernetes sends a termination signal. + +![Working](assets/token-refresher.png) + +# Deployment Instructions + +1. Build and push the Docker image using the provided Dockerfile to your preferred container registry. +2. Deploy the tool to your cluster using the sample Kubernetes manifest found here: [examples/token-refresher.yaml](./examples/token-refresher.yaml) + +# How It Works + +The token refresher is designed to operate as a sidecar container alongside your main application that depends on the service account token. It generates a new token at a custom location and updates the main container to use this new path. The refresher goes through several key phases: + +1. **Initialization** + + Initially, the refresher checks for the existence of a custom token. If it's missing, it sets up a symlink to the default projected token. + +2. **Monitoring** + + Then it enters a passive state where it periodically checks if the current token is expiring soon while waiting for a termination signal from Kubernetes. + + If it receives a shutdown signal (either from Kubernetes or the application) or it detects that the token is about to expire, it transitions to the active state. + +3. **Refreshing** + + In the active state, the refresher begins to regularly request a new token from the Kubernetes API server before the current one expires. It includes robust error handling to manage potential API server issues. This process continues until the application signals the refresher to stop. + +# Usage + +```sh +$ token-refresher --help +A sidecar which starts auto-refreshing the service account token when the default one is close to expiry or container receives a shutdown signal. + +Usage: + token-refresher [flags] + +Flags: + --default_token_file string path to default service account token file (default "/var/run/secrets/eks.amazonaws.com/serviceaccount/token") + --expiration_duration duration token expiry duration (default 2h0m0s) + -h, --help help for token-refresher + --kubeconfig string (optional) absolute path to the kubeconfig file (default "/home/token-refresher/.kube/config") + --max_attempts int max retries on token refresh failure (default 3) + -n, --namespace string current namespace + --refresh_interval duration token refresh interval (default 1h0m0s) + -s, --service_account string name of service account to issue token for + --sleep duration sleep duration between retries (default 20s) + --token_audience strings comma separated token audience (default [sts.amazonaws.com]) + --token_file string path to self-managed service account token file (default "/var/run/secrets/token-refresher/token") +``` + +# Backstory + +While moving a microservice to Kubernetes, we encountered a scenario where the service required over 24 hours to fully drain. We set up a PreStop hook and extended the `terminationGracePeriodSeconds` to accommodate this. However, we soon faced `ExpiredTokenException` errors. + +Investigation led us to a bug in Kubernetes, still unresolved as of March 2024, detailed here: [Kubelet stops rotating service account tokens when pod is terminating, breaking preStop hooks](https://github.com/kubernetes/kubernetes/issues/116481). + +We attempted a workaround by extending the token expiration using the `eks.amazonaws.com/token-expiration` annotation, but it couldn't exceed 24 hours as discussed [here](https://github.com/aws/amazon-eks-pod-identity-webhook#amazon-eks-pod-identity-webhook). We then looked at the cluster level `service-account-max-token-expiration` flag, only to be blocked by an open feature request that prevented us from adjusting it ourselves: [Allow user to modify the kube-apiserver flag --service-account-max-token-expiration](https://github.com/aws/containers-roadmap/issues/1836). + +We also considered using long-lived tokens, but they were incompatible due to a hardcoded issuer in the tokens, which was not accepted as per the error: `An error occurred (InvalidIdentityToken) when calling the AssumeRoleWithWebIdentity operation: Issuer must be a valid URL`. We needed the issuer to match the cluster's OIDC HTTP URL. + +After exhausting all other available options, we decided to build our own token-refresher. It began as a simple shell script to fetch new tokens from the API server, but as complexity grew with retries and error handling, and with the need for better testing, we developed this Go-based service. + +During testing, we encountered another hiccup where the refresher would start after the main container, causing errors due to the missing token. To resolve this, we added an init container to create the necessary symlink from the custom token to the default one at startup. + +This refresher has proven to be very effective for us, and we hope it will be beneficial to you as well! diff --git a/assets/token-refresher.png b/assets/token-refresher.png new file mode 100644 index 0000000..309e6d4 Binary files /dev/null and b/assets/token-refresher.png differ diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..245b506 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/SumoLogic-Labs/service-account-token-refresher/pkg/signals" + tokenrefresher "github.com/SumoLogic-Labs/service-account-token-refresher/pkg/token-refresher" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "k8s.io/client-go/util/homedir" +) + +type config struct { + tokenrefresher.TokenRefresher `mapstructure:",squash"` +} + +var conf *config + +var rootCmd = &cobra.Command{ + Use: "token-refresher", + Short: "Automatic token refresher for terminating pods", + Long: `A sidecar which starts auto-refreshing the service account token when the default one is close to expiry or container receives a shutdown signal.`, + Run: func(cmd *cobra.Command, args []string) { + stopCh := signals.SignalShutdown() + refresher := conf.TokenRefresher + if err := refresher.Run(stopCh); err != nil { + fmt.Printf("unable to run: %s", err.Error()) + os.Exit(2) + } + fmt.Println("Exiting") + }, +} + +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig) + + // The flag names must match those from conf.TokenRefresher + rootCmd.Flags().StringP("namespace", "n", "", "current namespace") + rootCmd.Flags().StringP("service_account", "s", "", "name of service account to issue token for") + rootCmd.Flags().String("default_token_file", "/var/run/secrets/eks.amazonaws.com/serviceaccount/token", "path to default service account token file") + rootCmd.Flags().String("token_file", "/var/run/secrets/token-refresher/token", "path to self-managed service account token file") + rootCmd.Flags().StringSlice("token_audience", []string{"sts.amazonaws.com"}, "comma separated token audience") + rootCmd.Flags().Duration("expiration_duration", time.Hour*2, "token expiry duration") + rootCmd.Flags().Duration("refresh_interval", time.Hour*1, "token refresh interval") + rootCmd.Flags().Int("max_attempts", 3, "max retries on token refresh failure") + rootCmd.Flags().Duration("sleep", time.Second*20, "sleep duration between retries") + + if home := homedir.HomeDir(); home != "" { + rootCmd.Flags().String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + } else { + rootCmd.Flags().String("kubeconfig", "", "absolute path to the kubeconfig file") + } + + viper.BindPFlags(rootCmd.LocalFlags()) +} + +func initConfig() { + viper.AutomaticEnv() // read in upper-cased env vars corresponding to above CLI flags + conf = new(config) + err := viper.Unmarshal(conf) + if err != nil { + fmt.Printf("unable to decode into config struct, %v", err) + } +} diff --git a/examples/token-refresher.yaml b/examples/token-refresher.yaml new file mode 100644 index 0000000..a3edc9b --- /dev/null +++ b/examples/token-refresher.yaml @@ -0,0 +1,110 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: app + namespace: app +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: token-refresher + namespace: app +rules: +- apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: token-refresher + namespace: app +subjects: +- kind: ServiceAccount + name: app + namespace: app +roleRef: + kind: ClusterRole + name: token-refresher + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Pod +metadata: + name: long-draining-pod + namespace: app +spec: + containers: + - image: service-account-token-refresher:latest # update this + imagePullPolicy: Always + name: token-refresher + env: + - name: DEFAULT_TOKEN_FILE + value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + - name: TOKEN_FILE + value: /var/run/secrets/token-refresher/token + - name: EXPIRATION_DURATION + value: 10m + - name: REFRESH_INTERVAL + value: 1m + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: SERVICE_ACCOUNT + value: app + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/eks.amazonaws.com/serviceaccount/token + volumeMounts: + - mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + name: aws-iam-token + readOnly: true + - mountPath: /var/run/secrets/token-refresher + name: token-refresher + - name: long-draining-app + image: alpine + command: + - sh + - -c + - | + for i in `seq 1 10` + do + # prints the token's expiry + echo "Now: $(date)" + EXPIRY=$(awk -F . '{if (length($2) % 4 == 3) print $2"="; else if (length($2) % 4 == 2) print $2"=="; else print $2; }' $AWS_WEB_IDENTITY_TOKEN_FILE | tr -- '-_' '+/' | base64 -d | awk -F , '{print $2}' | awk -F : '{print "@"$2}' | xargs date -d) + echo "Exp: $EXPIRY" + echo + sleep 20s + done + lifecycle: + preStop: + exec: + command: + - sh + - -c + - # custom draining logic here + sleep 30s && + touch /var/run/secrets/token-refresher/shutdown + env: + - name: AWS_WEB_IDENTITY_TOKEN_FILE + value: /var/run/secrets/token-refresher/token + volumeMounts: + - mountPath: /var/run/secrets/eks.amazonaws.com/serviceaccount + name: aws-iam-token + readOnly: true + - name: token-refresher + mountPath: /var/run/secrets/token-refresher + readOnly: false + terminationGracePeriodSeconds: 180 + serviceAccountName: app + volumes: + - name: token-refresher + emptyDir: {} + - name: aws-iam-token + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: sts.amazonaws.com + path: token diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ed8c98d --- /dev/null +++ b/go.mod @@ -0,0 +1,67 @@ +module github.com/SumoLogic-Labs/service-account-token-refresher + +go 1.22.0 + +require ( + github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.18.2 + k8s.io/api v0.29.2 + k8s.io/apimachinery v0.29.2 + k8s.io/client-go v0.29.2 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.3 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240102154912-e7106e64919e // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c5f6e26 --- /dev/null +++ b/go.sum @@ -0,0 +1,209 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.3 h1:yagOQz/38xJmcNeZJtrUcKjkHRltIaIFXKWeG1SkWGE= +github.com/emicklei/go-restful/v3 v3.11.3/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7c40f5c --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/SumoLogic-Labs/service-account-token-refresher/cmd" + +func main() { + cmd.Execute() +} diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go new file mode 100644 index 0000000..feb8f85 --- /dev/null +++ b/pkg/retry/retry.go @@ -0,0 +1,33 @@ +package retry + +import ( + "fmt" + "time" +) + +type Retryer struct { + MaxAttempts int `mapstructure:"max_attempts"` + Sleep time.Duration `mapstructure:"sleep"` +} + +func (r Retryer) Do(f func() (error, bool)) error { + return Retry(r.MaxAttempts, r.Sleep, f) +} + +func Retry(attempts int, sleep time.Duration, f func() (error, bool)) error { + if err, isRetryable := f(); err != nil { + if !isRetryable { + return err + } + + if attempts = attempts - 1; attempts > 0 { + fmt.Printf("Error: %s, Sleeping for %v before retrying\n", err.Error(), sleep) + time.Sleep(sleep) + fmt.Printf("Retrying with remaining attempts: %v\n", attempts) + return Retry(attempts, sleep, f) + } + return err + } + + return nil +} diff --git a/pkg/signals/shutdown.go b/pkg/signals/shutdown.go new file mode 100644 index 0000000..dcf06d8 --- /dev/null +++ b/pkg/signals/shutdown.go @@ -0,0 +1,26 @@ +package signals + +import ( + "os" + "os/signal" + "syscall" +) + +// onlyOneHandler ensures at most 1 shutdown handler is registered +var onlyOneHandler = make(chan struct{}) + +// SignalShutdown returns a stop channel which is closed on receiving an interrupt signal, +// giving the application a chance to shutdown gracefully. After the first signal, +// it forwards any further signals directly to the application causing it to force-quit. +func SignalShutdown() <-chan struct{} { + close(onlyOneHandler) // panics when called more than once + stopCh := make(chan struct{}) + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + signal.Reset() + close(stopCh) + }() + return stopCh +} diff --git a/pkg/ticker/ticker.go b/pkg/ticker/ticker.go new file mode 100644 index 0000000..77bf2d6 --- /dev/null +++ b/pkg/ticker/ticker.go @@ -0,0 +1,18 @@ +package ticker + +import "time" + +// NewTicker wraps around time.NewTicker but adds an extra tick at the start as soon as it starts +func NewTicker(repeat time.Duration) *time.Ticker { + ticker := time.NewTicker(repeat) + oc := ticker.C + nc := make(chan time.Time, 1) + go func() { + nc <- time.Now() + for tm := range oc { + nc <- tm + } + }() + ticker.C = nc + return ticker +} diff --git a/pkg/token-refresher/token-refresher.go b/pkg/token-refresher/token-refresher.go new file mode 100644 index 0000000..a177c26 --- /dev/null +++ b/pkg/token-refresher/token-refresher.go @@ -0,0 +1,173 @@ +package tokenrefresher + +import ( + "fmt" + "os" + "path" + "time" + + "github.com/SumoLogic-Labs/service-account-token-refresher/pkg/retry" + "github.com/SumoLogic-Labs/service-account-token-refresher/pkg/ticker" + + v1 "k8s.io/api/authentication/v1" + "k8s.io/client-go/kubernetes" +) + +// ShutdownFile indicates to token refresher that it can exit gracefuly now and cleanup the file on exit. +// Must be in the same directory as TokenFile. Content does not matter. +const ShutdownFile = "shutdown" + +type TokenRefresher struct { + Namespace string `mapstructure:"namespace"` + ServiceAccount string `mapstructure:"service_account"` + KubeConfig string `mapstructure:"kubeconfig"` + DefaultTokenFile string `mapstructure:"default_token_file"` + TokenFile string `mapstructure:"token_file"` + TokenAudience []string `mapstructure:"token_audience"` + ExpirationDuration time.Duration `mapstructure:"expiration_duration"` + RefreshInterval time.Duration `mapstructure:"refresh_interval"` + Retryer retry.Retryer `mapstructure:",squash"` + + minExpiryDuration time.Duration + shutdownFile string +} + +func (r TokenRefresher) Run(stopCh <-chan struct{}) error { + client, err := r.Init() + if err != nil { + return fmt.Errorf("unable to initialize: %w", err) + } + r.waitForTrigger(stopCh) + r.refreshLoop(client) + return nil +} + +func (r *TokenRefresher) Init() (kubernetes.Interface, error) { + r.minExpiryDuration = r.RefreshInterval + r.RefreshInterval/2 + r.shutdownFile = path.Join(path.Dir(r.TokenFile), ShutdownFile) + fmt.Printf("Running TokenRefresher with config: %+v\n", *r) + err := r.ensureTarget() + if err != nil { + return nil, err + } + return createKubeClient(r.KubeConfig) +} + +func (r TokenRefresher) ensureTarget() error { + _, err := os.Stat(r.DefaultTokenFile) + if err != nil { + return fmt.Errorf("unable to access default token at %s: %w", r.DefaultTokenFile, err) + } + _, err = os.Stat(r.TokenFile) + if err == nil { + fmt.Printf("Target already exists: %s\n", r.TokenFile) + return nil + } + err = os.Symlink(r.DefaultTokenFile, r.TokenFile) + if err != nil { + return fmt.Errorf("unable to symlink %s -> %s: %w", r.TokenFile, r.DefaultTokenFile, err) + } + fmt.Printf("Created link: %s -> %s\n", r.TokenFile, r.DefaultTokenFile) + return nil +} + +// waitForTrigger blocks until it either receives a shutdown signal or detects an invalid token +// token-refresher spends most of its time here - waiting for the trigger +func (r TokenRefresher) waitForTrigger(stopCh <-chan struct{}) { + fmt.Println("Waiting for shutdown signal and monitoring token expiry") + ch := r.monitorToken(stopCh) + for { + select { + case <-stopCh: + fmt.Println("Shutdown signal received - Ignoring") + return + case msg := <-ch: + fmt.Println(msg) + return + } + } +} + +func (r TokenRefresher) monitorToken(stopCh <-chan struct{}) <-chan string { + ticker := ticker.NewTicker(r.RefreshInterval) + ch := make(chan string) + go func() { + defer ticker.Stop() + for { + select { + case <-ticker.C: + if !readTokenAndValidate(r.TokenFile, r.minExpiryDuration) { + ch <- "Invalid/expired token detected" + return + } + if r.shouldShutdown() { + ch <- "Shutdown file detected while monitoring token" + return + } + case <-stopCh: + return + } + } + }() + return ch +} + +func (r TokenRefresher) refreshLoop(client kubernetes.Interface) { + fmt.Println("Starting refresh loop") + fmt.Printf("Will refresh every %v\n", r.RefreshInterval) + ticker := ticker.NewTicker(r.RefreshInterval) + defer ticker.Stop() + for range ticker.C { + // We could setup a file-watcher for faster shutdown instead of waiting for an entire loop interval + if r.shouldShutdown() { + if err := os.Remove(r.shutdownFile); err != nil { + fmt.Printf("unable to remove shutdown file: %s\n", err.Error()) + } + break + } + + err := r.Retryer.Do(func() (error, bool) { + return r.refresh(client), true + }) + if err != nil { + fmt.Printf("unable to refresh token: %s\n", err.Error()) + continue + } + fmt.Println("Refreshed token") + } +} + +func (r TokenRefresher) refresh(client kubernetes.Interface) error { + token, err := r.createToken(client) + if err != nil { + return err + } + if !isTokenValid(token, r.minExpiryDuration) { + return fmt.Errorf("invalid token from server") + } + return safeWrite(r.TokenFile, token) +} + +func (r TokenRefresher) createToken(client kubernetes.Interface) (string, error) { + expSec := r.ExpirationDuration.Milliseconds() / 1000 + req := &v1.TokenRequest{ + Spec: v1.TokenRequestSpec{ + Audiences: r.TokenAudience, + ExpirationSeconds: &expSec, + }, + } + resp, err := createToken(client, r.Namespace, r.ServiceAccount, req) + if err != nil { + return "", fmt.Errorf("unable to create token: %w", err) + } + return resp.Status.Token, nil +} + +func (r TokenRefresher) shouldShutdown() bool { + _, err := os.Stat(r.shutdownFile) + if err != nil { + return false + } + fmt.Printf("Shutdown file detected at %s\n", r.shutdownFile) + return true +} diff --git a/pkg/token-refresher/token-refresher_test.go b/pkg/token-refresher/token-refresher_test.go new file mode 100644 index 0000000..e3f7e8a --- /dev/null +++ b/pkg/token-refresher/token-refresher_test.go @@ -0,0 +1,278 @@ +package tokenrefresher + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + authv1 "k8s.io/api/authentication/v1" + "k8s.io/apimachinery/pkg/runtime" + testclient "k8s.io/client-go/kubernetes/fake" + k8stesing "k8s.io/client-go/testing" +) + +func TestTokenRefresher_ensureTarget(t *testing.T) { + t.Run("ensureTarget() should fail if default token does not exist", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + + err := r.ensureTarget() + if err == nil { + t.Error("ensureTarget() not failing if default token does not exist") + } + }) + + t.Run("ensureTarget() should create a symlink to default token", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + defaultToken := "default_token_contents" + safeWrite(r.DefaultTokenFile, defaultToken) + + err := r.ensureTarget() + if err != nil { + t.Errorf("ensureTarget() failed: %s", err.Error()) + } + + buf, err := os.ReadFile(r.TokenFile) + if err != nil { + t.Errorf("cannot read token file: %s", err.Error()) + } + if string(buf) != defaultToken { + t.Errorf("expected %s, got %s", defaultToken, string(buf)) + } + }) + + t.Run("ensureTarget() should not overwrite existing token", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + token := "new_token_contents" + safeWrite(r.TokenFile, token) + defaultToken := "default_token_contents" + safeWrite(r.DefaultTokenFile, defaultToken) + + err := r.ensureTarget() + if err != nil { + t.Errorf("ensureTarget() failed: %s", err.Error()) + } + + buf, err := os.ReadFile(r.TokenFile) + if err != nil { + t.Errorf("cannot read token file: %s", err.Error()) + } + if string(buf) != token { + t.Errorf("expected %s, got %s", defaultToken, string(buf)) + } + }) +} + +func TestTokenRefresher_waitForTrigger(t *testing.T) { + t.Run("waitForTrigger() should return when signalled", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + safeWrite(r.TokenFile, getTokenWithExpiry(time.Hour*2)) + stopCh := make(chan struct{}) + retCh := make(chan struct{}) + + go func() { + r.waitForTrigger(stopCh) + close(retCh) + }() + + select { + case <-retCh: + t.Fatalf("waitForTrigger() returned pre-maturely without being signalled") + case <-time.After(r.RefreshInterval * 2): + } + close(stopCh) + select { + case <-retCh: + case <-time.After(r.RefreshInterval * 2): + t.Errorf("waitForTrigger() did not return even after being signalled") + } + }) + + t.Run("waitForTrigger() should return when token is invalidated", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + safeWrite(r.TokenFile, getTokenWithExpiry(time.Hour*2)) + stopCh := make(chan struct{}) + retCh := make(chan struct{}) + + go func() { + r.waitForTrigger(stopCh) + close(retCh) + }() + + select { + case <-retCh: + t.Fatalf("waitForTrigger() returned pre-maturely without being signalled") + case <-time.After(r.RefreshInterval * 2): + } + safeWrite(r.TokenFile, getTokenWithExpiry(time.Minute)) + select { + case <-retCh: + case <-time.After(r.RefreshInterval * 2): + t.Errorf("waitForTrigger() did not return even after token is invalidated") + } + }) + + t.Run("waitForTrigger() should return immediately if token is invalid", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + stopCh := make(chan struct{}) + retCh := make(chan struct{}) + + go func() { + r.waitForTrigger(stopCh) + close(retCh) + }() + + select { + case <-retCh: + case <-time.After(r.RefreshInterval * 2): + t.Errorf("waitForTrigger() did not return for an invalid token") + } + }) + + t.Run("waitForTrigger() should return when shutdown file is detected", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + stopCh := make(chan struct{}) + retCh := make(chan struct{}) + safeWrite(r.TokenFile, getTokenWithExpiry(time.Hour*2)) + safeWrite(r.shutdownFile, "") + + go func() { + r.waitForTrigger(stopCh) + close(retCh) + }() + + select { + case <-retCh: + case <-time.After(r.RefreshInterval * 2): + t.Errorf("waitForTrigger() did not return on creation of shutdown file") + } + // The shutdown file should still exist + if !r.shouldShutdown() { + t.Errorf("waitForTrigger() deleted the shutdown file after consuming the signal") + } + }) +} + +func TestTokenRefresher_refresh(t *testing.T) { + t.Run("refresh() should write a valid token to file", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + safeWrite(r.TokenFile, "") + c := getFakeClient(r, false) + + err := r.refresh(c) + if err != nil { + t.Fatalf("refresh() did not create a valid token: %s", err.Error()) + } + + if !readTokenAndValidate(r.TokenFile, r.minExpiryDuration) { + t.Fatalf("refresh() created an invalid token file") + } + }) + + t.Run("refresh() should fail and skip updating token in case of errors", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + want := "this_string_should_not_be_overwritten" + safeWrite(r.TokenFile, want) + c := getFakeClient(r, true) + + err := r.refresh(c) + if err == nil { + t.Error("refresh() did not fail on error") + } + + got, err := os.ReadFile(r.TokenFile) + if err != nil { + t.Errorf("unable to read token file: %s", err.Error()) + } + if want != string(got) { + t.Errorf("want: %s, got %s", want, string(got)) + } + }) +} + +func TestTokenRefresher_refreshLoop(t *testing.T) { + t.Run("refreshLoop() should exit when shutdown file exists", func(t *testing.T) { + r, cleanup := setup() + defer cleanup() + safeWrite(r.TokenFile, "") + c := getFakeClient(r, false) + retCh := make(chan struct{}) + + go func() { + r.refreshLoop(c) + close(retCh) + }() + + select { + case <-retCh: + t.Fatalf("refreshLoop() returned pre-maturely without being shutdown") + case <-time.After(r.RefreshInterval * 2): + } + safeWrite(r.shutdownFile, "") + select { + case <-retCh: + case <-time.After(r.RefreshInterval * 2): + t.Errorf("refreshLoop() did not return even after shutdown file was created") + } + // The shutdown file should be cleaned up + if r.shouldShutdown() { + t.Errorf("refreshLoop() did not delete the shutdown file on exit") + } + }) +} + +func setup() (*TokenRefresher, func()) { + testDir, err := os.MkdirTemp(os.TempDir(), "token-refresher-test-tmp-*") + if err != nil { + panic(err.Error()) + } + r := &TokenRefresher{ + DefaultTokenFile: path.Join(testDir, "default_token"), + TokenFile: path.Join(testDir, "token"), + ExpirationDuration: time.Hour * 2, // used to test if refresh() is sending this correctly to apiserver + RefreshInterval: time.Millisecond * 200, + Namespace: "test-ns", + ServiceAccount: "test-sa", + + minExpiryDuration: time.Minute * 90, + shutdownFile: path.Join(testDir, ShutdownFile), + } + cleanup := func() { + os.RemoveAll(testDir) + } + return r, cleanup +} + +func getFakeClient(r *TokenRefresher, wantErr bool) *testclient.Clientset { + ns, sa, exp := r.Namespace, r.ServiceAccount, r.ExpirationDuration + c := testclient.NewSimpleClientset() + c.PrependReactor("create", "serviceaccounts", func(action k8stesing.Action) (bool, runtime.Object, error) { + if wantErr { + return true, nil, fmt.Errorf("apiserver overloaded, could not create token") + } + act := action.(k8stesing.CreateActionImpl) + ret := act.GetObject().DeepCopyObject().(*authv1.TokenRequest) + if act.GetNamespace() != ns { + return true, nil, fmt.Errorf("want ns: %s, got %s", ns, act.GetNamespace()) + } + if act.Name != sa { + return true, nil, fmt.Errorf("want sa: %s, got %s", sa, act.Name) + } + if *ret.Spec.ExpirationSeconds != int64(exp.Seconds()) { + return true, nil, fmt.Errorf("want exp: %v, got %v", exp.Seconds(), *ret.Spec.ExpirationSeconds) + } + ret.Status.Token = getTokenWithExpiry(time.Duration(*ret.Spec.ExpirationSeconds) * time.Second) + return true, ret, nil + }) + return c +} diff --git a/pkg/token-refresher/token.go b/pkg/token-refresher/token.go new file mode 100644 index 0000000..6636038 --- /dev/null +++ b/pkg/token-refresher/token.go @@ -0,0 +1,110 @@ +package tokenrefresher + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path" + "strings" + "time" + + v1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func createKubeClient(kubeconfig string) (*kubernetes.Clientset, error) { + var config *rest.Config + config, err := rest.InClusterConfig() + if err != nil { + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("no in-cluster config or kubeconfig found") + } + } + return kubernetes.NewForConfig(config) +} + +func createToken(client kubernetes.Interface, ns, sa string, req *v1.TokenRequest) (*v1.TokenRequest, error) { + return client.CoreV1(). + ServiceAccounts(ns). + CreateToken(context.TODO(), sa, req, metav1.CreateOptions{}) +} + +func readTokenAndValidate(tokenFile string, minExp time.Duration) bool { + b, err := os.ReadFile(tokenFile) + if err != nil { + fmt.Printf("unable to read file %s: %s\n", tokenFile, err.Error()) + return false + } + return isTokenValid(string(b), minExp) +} + +// isTokenValid checks if the `exp` key in the claims of the jwt is valid for at least the given duration +func isTokenValid(token string, minExp time.Duration) bool { + parts := strings.Split(token, ".") + if len(parts) != 3 { + fmt.Println("invalid token") + return false + } + data, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + fmt.Printf("unable to decode token: %s\n", err.Error()) + return false + } + var claims map[string]interface{} + err = json.Unmarshal(data, &claims) + if err != nil { + fmt.Printf("unable to decode json: %s\n", err.Error()) + return false + } + exp, ok := claims["exp"].(float64) + if !ok { + fmt.Printf("exp not a number: %v", claims["exp"]) + return false + } + expiresAt := time.Unix(int64(exp), 0) + expiresIn := time.Until(expiresAt) + if expiresIn < 0 { + fmt.Printf("token has expired at %v (%v ago)\n", expiresAt, -expiresIn) + return false + } + if expiresIn < minExp { + fmt.Printf("token too old, expires at %v (in %v)\n", expiresAt, expiresIn) + return false + } + // fmt.Printf("token is valid, expires at %v (in %v)\n", expiresAt, expiresIn) + return true +} + +// safeWrite first writes to a temp file and then switches it with the target file atomically by renaming +func safeWrite(filename, data string) error { + tmpFilename, err := writeTemp(filename, data) + if err != nil { + return err + } + // Cleanup temp file in case rename fails + defer os.Remove(tmpFilename) + err = os.Rename(tmpFilename, filename) + if err != nil { + return fmt.Errorf("unable to rename %s to %s: %w", tmpFilename, filename, err) + } + return nil +} + +func writeTemp(filename, data string) (string, error) { + f, err := os.CreateTemp(path.Dir(filename), path.Base(filename)) + if err != nil { + return "", fmt.Errorf("unable to create file: %w", err) + } + defer f.Close() + _, err = f.WriteString(data) + if err != nil { + return f.Name(), fmt.Errorf("unable to write to file %s: %w", f.Name(), err) + } + return f.Name(), nil +} diff --git a/pkg/token-refresher/token_test.go b/pkg/token-refresher/token_test.go new file mode 100644 index 0000000..53091fb --- /dev/null +++ b/pkg/token-refresher/token_test.go @@ -0,0 +1,71 @@ +package tokenrefresher + +import ( + "encoding/base64" + "fmt" + "testing" + "time" +) + +const JwtFmt = ".%s." + +func Test_isTokenValid(t *testing.T) { + type args struct { + token string + } + tests := []struct { + name string + args args + want bool + }{ + { + "Reject empty token", + args{""}, + false, + }, + { + "Reject invalid token", + args{"thisisnotatoken"}, + false, + }, + { + "Reject ill-formated token", + args{fmt.Sprintf(JwtFmt, "notb64string")}, + false, + }, + { + "Reject token about to expire", + args{getTokenWithExpiry(time.Hour)}, + false, + }, + { + "Reject token that has expired", + args{getTokenWithExpiry(-time.Second)}, + false, + }, + { + "Accept token with enough expiry", + args{getTokenWithExpiry(time.Hour * 2)}, + true, + }, + { + "Accept token with enough expiry", + args{getTokenWithExpiry(time.Hour * 48)}, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isTokenValid(tt.args.token, time.Minute*90); got != tt.want { + t.Errorf("isTokenValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func getTokenWithExpiry(exp time.Duration) string { + expiresAt := time.Now().Add(exp) + data := fmt.Sprintf(`{"exp":%v}`, expiresAt.Unix()) + claims := base64.RawURLEncoding.EncodeToString([]byte(data)) + return fmt.Sprintf(JwtFmt, claims) +}