Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

command line tools for redacting keyring from snapshots #24023

Merged
merged 2 commits into from
Sep 20, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/24023.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:improvement
cli: Added redaction options to operator snapshot commands
```
5 changes: 5 additions & 0 deletions command/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -849,6 +849,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
Meta: meta,
}, nil
},
"operator snapshot redact": func() (cli.Command, error) {
return &OperatorSnapshotRedactCommand{
Meta: meta,
}, nil
},

"plan": func() (cli.Command, error) {
return &JobPlanCommand{
Expand Down
95 changes: 95 additions & 0 deletions command/operator_snapshot_redact.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package command

import (
"fmt"
"io"
"os"
"strings"

"github.com/hashicorp/nomad/helper/raftutil"
"github.com/posener/complete"
)

type OperatorSnapshotRedactCommand struct {
Meta
}

func (c *OperatorSnapshotRedactCommand) Help() string {
helpText := `
Usage: nomad operator snapshot redact [options] <file>

Removes key material from an existing snapshot file created by the operator
snapshot save command, when using the AEAD keyring provider. When using a KMS
keyring provider, no cleartext key material is stored in snapshots and this
command is not necessary. Note that this command requires loading the entire
snapshot into memory locally and overwrites the existing snapshot.

This is useful for situations where you need to transmit a snapshot without
exposing key material.

General Options:

` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace)

return strings.TrimSpace(helpText)
}

func (c *OperatorSnapshotRedactCommand) AutocompleteFlags() complete.Flags {
return complete.Flags{}
}

func (c *OperatorSnapshotRedactCommand) AutocompleteArgs() complete.Predictor {
return complete.PredictFiles("*")
}

func (c *OperatorSnapshotRedactCommand) Synopsis() string {
return "Redacts an existing snapshot of Nomad server state"
}

func (c *OperatorSnapshotRedactCommand) Name() string { return "operator snapshot redact" }

func (c *OperatorSnapshotRedactCommand) Run(args []string) int {
if len(args) != 1 {
c.Ui.Error("This command takes one argument: <file>")
c.Ui.Error(commandErrorText(c))
return 1
}

path := args[0]
f, err := os.Open(path)
if err != nil {
c.Ui.Error(fmt.Sprintf("Error opening snapshot file: %s", err))
return 1
}
defer f.Close()

tmpFile, err := os.Create(path + ".tmp")
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to create temporary file: %v", err))
return 1
}

_, err = io.Copy(tmpFile, f)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to copy snapshot to temporary file: %v", err))
return 1
}

err = raftutil.RedactSnapshot(tmpFile)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to redact snapshot: %v", err))
return 1
}

tgross marked this conversation as resolved.
Show resolved Hide resolved
err = os.Rename(tmpFile.Name(), path)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to finalize snapshot file: %v", err))
return 1
}

c.Ui.Output("Snapshot redacted")
return 0
}
24 changes: 21 additions & 3 deletions command/operator_snapshot_save.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/hashicorp/nomad/api"
"github.com/hashicorp/nomad/helper/raftutil"
"github.com/posener/complete"
)

Expand Down Expand Up @@ -48,8 +49,14 @@ General Options:

Snapshot Save Options:

-stale=[true|false]
The -stale argument defaults to "false" which means the leader provides the
-redact
The -redact option will locally edit the snapshot to remove any cleartext key
material from the root keyring. Only the AEAD keyring provider has cleartext
key material in Raft. Note that this operation requires loading the snapshot
into memory locally.

-stale
The -stale option defaults to "false" which means the leader provides the
result. If the cluster is in an outage state without a leader, you may need
to set -stale to "true" to get the configuration from a non-leader server.
`
Expand All @@ -74,12 +81,14 @@ func (c *OperatorSnapshotSaveCommand) Synopsis() string {
func (c *OperatorSnapshotSaveCommand) Name() string { return "operator snapshot save" }

func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
var stale bool
var stale, redact bool

flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
flags.Usage = func() { c.Ui.Output(c.Help()) }

flags.BoolVar(&stale, "stale", false, "")
flags.BoolVar(&redact, "redact", false, "")

if err := flags.Parse(args); err != nil {
c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
return 1
Expand Down Expand Up @@ -141,6 +150,15 @@ func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
return 1
}

if redact {
c.Ui.Info("Redacting key material from snapshot")
err := raftutil.RedactSnapshot(tmpFile)
if err != nil {
c.Ui.Error(fmt.Sprintf("Could not redact snapshot: %v", err))
return 1
}
}

err = os.Rename(tmpFile.Name(), filename)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to finalize snapshot file: %v", err))
Expand Down
2 changes: 1 addition & 1 deletion command/operator_snapshot_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func (c *OperatorSnapshotStateCommand) Run(args []string) int {
}
defer f.Close()

state, meta, err := raftutil.RestoreFromArchive(f, filter)
_, state, meta, err := raftutil.RestoreFromArchive(f, filter)
if err != nil {
c.Ui.Error(fmt.Sprintf("Failed to read archive file: %s", err))
return 1
Expand Down
26 changes: 26 additions & 0 deletions helper/raftutil/fsm.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/hashicorp/go-memdb"
"github.com/hashicorp/nomad/nomad"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/raft"
raftboltdb "github.com/hashicorp/raft-boltdb/v2"
)
Expand Down Expand Up @@ -209,6 +210,7 @@ func StateAsMap(store *state.StateStore) map[string][]interface{} {
"Jobs": toArray(store.Jobs(nil, state.SortDefault)),
"Nodes": toArray(store.Nodes(nil)),
"PeriodicLaunches": toArray(store.PeriodicLaunches(nil)),
"RootKeys": rootKeyMeta(store),
"SITokenAccessors": toArray(store.SITokenAccessors(nil)),
"ScalingEvents": toArray(store.ScalingEvents(nil)),
"ScalingPolicies": toArray(store.ScalingPolicies(nil)),
Expand Down Expand Up @@ -265,3 +267,27 @@ func toArray(iter memdb.ResultIterator, err error) []interface{} {

return r
}

// rootKeyMeta allows displaying keys without their key material
func rootKeyMeta(store *state.StateStore) []any {

iter, err := store.RootKeys(nil)
if err != nil {
return []any{err}
}

keyMeta := []any{}
for {
raw := iter.Next()
if raw == nil {
break
}
k := raw.(*structs.RootKey)
if k == nil {
break
}
keyMeta = append(keyMeta, k.Meta())
}

return keyMeta
}
70 changes: 63 additions & 7 deletions helper/raftutil/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,22 @@ package raftutil
import (
"fmt"
"io"
"os"

"github.com/hashicorp/go-hclog"
"github.com/hashicorp/raft"

"github.com/hashicorp/nomad/helper/snapshot"
"github.com/hashicorp/nomad/nomad"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/raft"
)

func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (*state.StateStore, *raft.SnapshotMeta, error) {
func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (raft.FSM, *state.StateStore, *raft.SnapshotMeta, error) {
logger := hclog.L()

fsm, err := dummyFSM(logger)
if err != nil {
return nil, nil, fmt.Errorf("failed to create FSM: %w", err)
return nil, nil, nil, fmt.Errorf("failed to create FSM: %w", err)
}

// r is closed by RestoreFiltered, w is closed by CopySnapshot
Expand All @@ -40,13 +41,68 @@ func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (*state.Stat

err = fsm.RestoreWithFilter(r, filter)
if err != nil {
return nil, nil, fmt.Errorf("failed to restore from snapshot: %w", err)
return nil, nil, nil, fmt.Errorf("failed to restore from snapshot: %w", err)
}

select {
case err := <-errCh:
return nil, nil, err
return nil, nil, nil, err
case meta := <-metaCh:
return fsm.State(), meta, nil
return fsm, fsm.State(), meta, nil
}
}

func RedactSnapshot(srcFile *os.File) error {
srcFile.Seek(0, 0)
fsm, store, meta, err := RestoreFromArchive(srcFile, nil)
if err != nil {
return fmt.Errorf("Failed to load snapshot from archive: %w", err)
}

iter, err := store.RootKeys(nil)
if err != nil {
return fmt.Errorf("Failed to query for root keys: %v", err)
}

for {
raw := iter.Next()
if raw == nil {
break
}
rootKey := raw.(*structs.RootKey)
if rootKey == nil {
break
}
if len(rootKey.WrappedKeys) > 0 {
rootKey.KeyID = rootKey.KeyID + " [REDACTED]"
rootKey.WrappedKeys = nil
}
msg, err := structs.Encode(structs.WrappedRootKeysUpsertRequestType,
&structs.KeyringUpsertWrappedRootKeyRequest{
WrappedRootKeys: rootKey,
})
if err != nil {
return fmt.Errorf("Could not re-encode redacted key: %v", err)
}

fsm.Apply(&raft.Log{
Type: raft.LogCommand,
Data: msg,
})
}

snap, err := snapshot.NewFromFSM(hclog.Default(), fsm, meta)
if err != nil {
return fmt.Errorf("Failed to create redacted snapshot: %v", err)
}

srcFile.Truncate(0)
srcFile.Seek(0, 0)

_, err = io.Copy(srcFile, snap)
if err != nil {
return fmt.Errorf("Failed to copy snapshot to temporary file: %v", err)
}

return srcFile.Sync()
}
43 changes: 43 additions & 0 deletions helper/snapshot/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,49 @@ func New(logger hclog.Logger, r *raft.Raft) (*Snapshot, error) {
if err != nil {
return nil, fmt.Errorf("failed to open snapshot: %v:", err)
}

return writeSnapshot(logger, metadata, snap)
}

// NewFromFSM takes a state snapshot of the given FSM (for when we don't have a
// Raft instance setup) into a temporary file and returns an object that gives
// access to the file as an io.Reader. You must arrange to call Close() on the
// returned object or else you will leak a temporary file.
func NewFromFSM(logger hclog.Logger, fsm raft.FSM, meta *raft.SnapshotMeta) (*Snapshot, error) {
_, trans := raft.NewInmemTransport("")
snapshotStore := raft.NewInmemSnapshotStore()

fsmSnap, err := fsm.Snapshot()
if err != nil {
return nil, err
}

sink, err := snapshotStore.Create(meta.Version, meta.Index, meta.Term,
meta.Configuration, meta.ConfigurationIndex, trans)
if err != nil {
return nil, err
}
err = fsmSnap.Persist(sink)
if err != nil {
return nil, err
}

err = sink.Close()
if err != nil {
return nil, err
}

snapshotID := sink.ID()
metadata, snap, err := snapshotStore.Open(snapshotID)
if err != nil {
return nil, err
}

return writeSnapshot(logger, metadata, snap)
}

func writeSnapshot(logger hclog.Logger, metadata *raft.SnapshotMeta, snap io.ReadCloser) (*Snapshot, error) {

defer func() {
if err := snap.Close(); err != nil {
logger.Error("Failed to close Raft snapshot", "error", err)
Expand Down
Loading
Loading