Skip to content

Commit

Permalink
feat: support AWS key manager (#72)
Browse files Browse the repository at this point in the history
* feat:use aws to manage keys

* feat:improve use aws to manage keys

* feat:improve logs use aws to manage keys

* add debug information to facilitate testing

* fix ci error

* fix ci error, add github.com/aws/aws-sdk-go-v2/config

* fix  bather/proposer

* fix  bather/proposer key_flag_name

* fix  aws_key_region aws_key_id

* fix  re-define aws_key format

* fix  re-define aws_key format

* fix  add log

* feat:Use AWS to manage keys

* feat:Use AWS to manage keys

* Optimize unreasonable specifications

* Optimize unreasonable specifications

* update return by switch default

* using `context` instead of `golang.org/x/net/context`

* using `context` instead of `golang.org/x/net/context`

* Modify unresaonable variable naming and syntax erros in logs

* Update op-aws-sdk/key_manager.go

Co-authored-by: bendanzhentan <136774549+bendanzhentan@users.noreply.github.com>

* Modify code accroding to bendanzhentan reviewed

---------

Co-authored-by: eric <eric.l@nodereal.io>
Co-authored-by: Owen <103096885+owen-reorg@users.noreply.github.com>
Co-authored-by: bendanzhentan <136774549+bendanzhentan@users.noreply.github.com>
  • Loading branch information
4 people authored Nov 6, 2023
1 parent b87483c commit afa3cac
Show file tree
Hide file tree
Showing 9 changed files with 144 additions and 5 deletions.
13 changes: 13 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ require (
github.com/DataDog/zstd v1.5.2 // indirect
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
github.com/allegro/bigcache v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.21.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.44 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.42 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 // indirect
github.com/aws/smithy-go v1.15.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
Expand Down
29 changes: 29 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,32 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/aws/aws-sdk-go-v2 v1.21.1 h1:wjHYshtPpYOZm+/mu3NhVgRRc0baM6LJZOmxPZ5Cwzs=
github.com/aws/aws-sdk-go-v2 v1.21.1/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
github.com/aws/aws-sdk-go-v2/config v1.18.44 h1:U10NQ3OxiY0dGGozmVIENIDnCT0W432PWxk2VO8wGnY=
github.com/aws/aws-sdk-go-v2/config v1.18.44/go.mod h1:pHxnQBldd0heEdJmolLBk78D1Bf69YnKLY3LOpFImlU=
github.com/aws/aws-sdk-go-v2/credentials v1.13.42 h1:KMkjpZqcMOwtRHChVlHdNxTUUAC6NC/b58mRZDIdcRg=
github.com/aws/aws-sdk-go-v2/credentials v1.13.42/go.mod h1:7ltKclhvEB8305sBhrpls24HGxORl6qgnQqSJ314Uw8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12 h1:3j5lrl9kVQrJ1BU4O0z7MQ8sa+UXdiLuo4j0V+odNI8=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.12/go.mod h1:JbFpcHDBdsex1zpIKuVRorZSQiZEyc3MykNCcjgz174=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 h1:817VqVe6wvwE46xXy6YF5RywvjOX6U2zRQQ6IbQFK0s=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42/go.mod h1:oDfgXoBBmj+kXnqxDDnIDnC56QBosglKp8ftRCTxR+0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 h1:7ZApaXzWbo8slc+W5TynuUlB4z66g44h7uqa3/d/BsY=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36/go.mod h1:rwr4WnmFi3RJO0M4dxbJtgi9BPLMpVBMX1nUte5ha9U=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44 h1:quOJOqlbSfeJTboXLjYXM1M9T52LBXqLoTPlmsKLpBo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.44/go.mod h1:LNy+P1+1LiRcCsVYr/4zG5n8zWFL0xsvZkOybjbftm8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 h1:YXlm7LxwNlauqb2OrinWlcvtsflTzP8GaMvYfQBhoT4=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36/go.mod h1:ou9ffqJ9hKOVZmjlC6kQ6oROAyG1M4yBKzR+9BKbDwk=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4 h1:LUtjmUxYPkiFkiVyvLmHVcuthVPnEKd0hEprTOVRTS0=
github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.21.4/go.mod h1:Bph0xA97xjEciochtR3JKrgGHt1psILMtFgu3KAbiBE=
github.com/aws/aws-sdk-go-v2/service/sso v1.15.1 h1:ZN3bxw9OYC5D6umLw6f57rNJfGfhg1DIAAcKpzyUTOE=
github.com/aws/aws-sdk-go-v2/service/sso v1.15.1/go.mod h1:PieckvBoT5HtyB9AsJRrYZFY2Z+EyfVM/9zG6gbV8DQ=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2 h1:fSCCJuT5i6ht8TqGdZc5Q5K9pz/atrf7qH4iK5C9XzU=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.2/go.mod h1:5eNtr+vNc5vVd92q7SJ+U/HszsIdhZBEyi9dkMRKsp8=
github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7HROOpeSMu1xDA/I8=
github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI=
github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
Expand Down Expand Up @@ -306,6 +332,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
Expand Down Expand Up @@ -412,6 +439,8 @@ github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZl
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down
83 changes: 83 additions & 0 deletions op-aws-sdk/key_manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package op_aws_sdk

import (
"context"
"encoding/json"
"errors"
"os"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli"
)

const (
OP_NODE_P2P_SEQUENCER_KEY = "OP_NODE_P2P_SEQUENCER_KEY"
OP_BATCHER_SIGN_KEY = "OP_BATCHER_SIGN_KEY"
OP_PROPOSER_SIGN_KEY = "OP_PROPOSER_SIGN_KEY"
AWS_KEY_JSON_NAME = "pk"
)

func KeyManager(context context.Context, ctx *cli.Context, keyType string) error {
secretName := ""
awsRegion := ""
flagName := ""
log.Info("Key manager ", "keyType", keyType)
switch keyType {
case OP_NODE_P2P_SEQUENCER_KEY:
secretName = "OP_NODE_AWS_P2P_SECRET_NAME"
awsRegion = "OP_NODE_AWS_P2P_SECRET_REGION"
flagName = "p2p.sequencer.key"
case OP_BATCHER_SIGN_KEY:
secretName = "OP_BATCHER_AWS_SECRET_NAME"
awsRegion = "OP_BATCHER_AWS_SECRET_REGION"
flagName = "private-key"
case OP_PROPOSER_SIGN_KEY:
secretName = "OP_PROPOSER_AWS_SECRET_NAME"
awsRegion = "OP_PROPOSER_AWS_SECRET_REGION"
flagName = "private-key"
default:
log.Error("Key manager ", "unknown keyType", keyType)
panic("Key manager unknown key type")
}
return load(context, ctx, awsRegion, secretName, flagName)
}
func load(context context.Context, ctx *cli.Context, awsRegion string, secretName string, flagName string) error {
name := os.Getenv(secretName)
region := os.Getenv(awsRegion)
if name != "" && region != "" {
loadKeyConfig, err := config.LoadDefaultConfig(context, config.WithRegion(region))
if err != nil {
log.Error("Key manager load key config from aws", "error", err)
return err
}
secretManager := secretsmanager.NewFromConfig(loadKeyConfig)
input := &secretsmanager.GetSecretValueInput{
SecretId: aws.String(name),
VersionStage: aws.String("AWSCURRENT"),
}
result, err := secretManager.GetSecretValue(context, input)
if err != nil {
log.Error("Key manager key value from aws", "error", err)
return err
}
resultMap := make(map[string]string)
secretBytes := []byte(*result.SecretString)
err = json.Unmarshal(secretBytes, &resultMap)
if err != nil {
return err
}
key, ok := resultMap[AWS_KEY_JSON_NAME]
if !ok {
log.Error("Key manager load key does not exist")
return errors.New("Key manager load key does not exist")
}
log.Info("Key manager load key is success")
ctx.Set(flagName, key)
} else {
log.Info("Key manager is skipped")
}
return nil
}
1 change: 1 addition & 0 deletions op-batcher/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY ./op-bindings /app/op-bindings
COPY ./op-node /app/op-node
COPY ./op-service /app/op-service
COPY ./op-signer /app/op-signer
COPY ./op-aws-sdk /app/op-aws-sdk
COPY ./go.mod /app/go.mod
COPY ./go.sum /app/go.sum

Expand Down
8 changes: 6 additions & 2 deletions op-batcher/cmd/main.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package main

import (
"context"
"fmt"
"os"

"github.com/urfave/cli"

op_aws_sdk "github.com/ethereum-optimism/optimism/op-aws-sdk"
"github.com/ethereum-optimism/optimism/op-batcher/batcher"
"github.com/ethereum-optimism/optimism/op-batcher/cmd/doc"
"github.com/ethereum-optimism/optimism/op-batcher/flags"
oplog "github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli"
)

var (
Expand Down Expand Up @@ -46,6 +47,9 @@ func main() {
// This is done to capture the Version of the batcher.
func curryMain(version string) func(ctx *cli.Context) error {
return func(ctx *cli.Context) error {
if err := op_aws_sdk.KeyManager(context.Background(), ctx, op_aws_sdk.OP_BATCHER_SIGN_KEY); err != nil {
return err
}
return batcher.Main(version, ctx)
}
}
1 change: 1 addition & 0 deletions op-node/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ COPY ./op-node /app/op-node
COPY ./op-chain-ops /app/op-chain-ops
COPY ./op-service /app/op-service
COPY ./op-bindings /app/op-bindings
COPY ./op-aws-sdk /app/op-aws-sdk
COPY ./go.mod /app/go.mod
COPY ./go.sum /app/go.sum
COPY ./.git /app/.git
Expand Down
5 changes: 4 additions & 1 deletion op-node/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/ethereum/go-ethereum/log"

op_aws_sdk "github.com/ethereum-optimism/optimism/op-aws-sdk"
opnode "github.com/ethereum-optimism/optimism/op-node"
"github.com/ethereum-optimism/optimism/op-node/cmd/genesis"
"github.com/ethereum-optimism/optimism/op-node/cmd/p2p"
Expand Down Expand Up @@ -90,7 +91,9 @@ func RollupNodeMain(ctx *cli.Context) error {
log := oplog.NewLogger(logCfg)
opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, log)
m := metrics.NewMetrics("default")

if err := op_aws_sdk.KeyManager(context.Background(), ctx, op_aws_sdk.OP_NODE_P2P_SEQUENCER_KEY); err != nil {
return err
}
cfg, err := opnode.NewConfig(ctx, log)
if err != nil {
log.Error("Unable to create the rollup node config", "error", err)
Expand Down
1 change: 1 addition & 0 deletions op-proposer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ COPY ./op-bindings /app/op-bindings
COPY ./op-node /app/op-node
COPY ./op-service /app/op-service
COPY ./op-signer /app/op-signer
COPY ./op-aws-sdk /app/op-aws-sdk
COPY ./go.mod /app/go.mod
COPY ./go.sum /app/go.sum
COPY ./.git /app/.git
Expand Down
8 changes: 6 additions & 2 deletions op-proposer/cmd/main.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package main

import (
"context"
"fmt"
"os"

"github.com/urfave/cli"

op_aws_sdk "github.com/ethereum-optimism/optimism/op-aws-sdk"
"github.com/ethereum-optimism/optimism/op-proposer/cmd/doc"
"github.com/ethereum-optimism/optimism/op-proposer/flags"
"github.com/ethereum-optimism/optimism/op-proposer/proposer"
oplog "github.com/ethereum-optimism/optimism/op-service/log"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli"
)

var (
Expand Down Expand Up @@ -46,6 +47,9 @@ func main() {
// This is done to capture the Version of the proposer.
func curryMain(version string) func(ctx *cli.Context) error {
return func(ctx *cli.Context) error {
if err := op_aws_sdk.KeyManager(context.Background(), ctx, op_aws_sdk.OP_PROPOSER_SIGN_KEY); err != nil {
return err
}
return proposer.Main(version, ctx)
}
}

0 comments on commit afa3cac

Please sign in to comment.