command line tools for redacting keyring from snapshots

In #23977 we moved the keyring into Raft, which can expose key material in Raft
snapshots when using the less-secure AEAD keyring instead of KMS. This changeset
adds tools for redacting this material from snapshots:

* The `operator snapshot state` command gains the ability to display key
  metadata (only), which respects the `-filter` option.
* The `operator snapshot save` command gains a `-redact` option that removes key
  material from the snapshot after it's downloaded.
* A new `operator snapshot redact` command allows removing key material from an
  existing snapshot.

command/commands.go

@@ -849,6 +849,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"operator snapshot redact": func() (cli.Command, error) {
+			return &OperatorSnapshotRedactCommand{
+				Meta: meta,
+			}, nil
+		},
 
 		"plan": func() (cli.Command, error) {
 			return &JobPlanCommand{

command/operator_snapshot_redact.go

@@ -0,0 +1,94 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package command
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/posener/complete"
+)
+
+type OperatorSnapshotRedactCommand struct {
+	Meta
+}
+
+func (c *OperatorSnapshotRedactCommand) Help() string {
+	helpText := `
+Usage: nomad operator snapshot redact [options] <file>
+
+  Removes key material from an existing snapshot file created by the operator
+  snapshot save command, when using the AEAD keyring provider. When using a KMS
+  keyring provider, no cleartext key material is stored in snapshots and this
+  command is not neccessary. Note that this command requires loading the entire
+  snapshot into memory locally and overwrites the existing snapshot.
+
+  This is useful for situations where you need to transmit a snapshot without
+  exposing key material.
+
+General Options:
+
+  ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace)
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *OperatorSnapshotRedactCommand) AutocompleteFlags() complete.Flags {
+	return complete.Flags{}
+}
+
+func (c *OperatorSnapshotRedactCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictFiles("*")
+}
+
+func (c *OperatorSnapshotRedactCommand) Synopsis() string {
+	return "Redacts an existing snapshot of Nomad server state"
+}
+
+func (c *OperatorSnapshotRedactCommand) Name() string { return "operator snapshot redact" }
+
+func (c *OperatorSnapshotRedactCommand) Run(args []string) int {
+	if len(args) != 1 {
+		c.Ui.Error("This command takes one argument: <file>")
+		c.Ui.Error(commandErrorText(c))
+		return 1
+	}
+
+	path := args[0]
+	f, err := os.Open(path)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error opening snapshot file: %s", err))
+		return 1
+	}
+	defer f.Close()
+
+	tmpFile, err := os.Create(path + ".tmp")
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to create temporary file: %v", err))
+		return 1
+	}
+
+	_, err = io.Copy(tmpFile, f)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to copy snapshot to temporary file: %v", err))
+		return 1
+	}
+
+	err = redactSnapshot(tmpFile)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to redact snapshot: %v", err))
+		return 1
+	}
+
+	err = os.Rename(tmpFile.Name(), path)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to finalize snapshot file: %v", err))
+		return 1
+	}
+
+	c.Ui.Output("Snapshot redacted")
+	return 0
+}

command/operator_snapshot_save.go

@@ -10,7 +10,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/helper/raftutil"
+	"github.com/hashicorp/nomad/helper/snapshot"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/raft"
 	"github.com/posener/complete"
 )
 
@@ -48,8 +53,14 @@ General Options:
 
 Snapshot Save Options:
 
-  -stale=[true|false]
-    The -stale argument defaults to "false" which means the leader provides the
+  -redact
+    The -redact option will locally edit the snapshot to remove any cleartext key
+    material from the root keyring. Only the AEAD keyring provider has cleartext
+    key material in Raft. Note that this operation requires loading the snapshot
+    into memory locally.
+
+  -stale
+    The -stale option defaults to "false" which means the leader provides the
     result. If the cluster is in an outage state without a leader, you may need
     to set -stale to "true" to get the configuration from a non-leader server.
 `
@@ -74,12 +85,14 @@ func (c *OperatorSnapshotSaveCommand) Synopsis() string {
 func (c *OperatorSnapshotSaveCommand) Name() string { return "operator snapshot save" }
 
 func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
-	var stale bool
+	var stale, redact bool
 
 	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
 
 	flags.BoolVar(&stale, "stale", false, "")
+	flags.BoolVar(&redact, "redact", false, "")
+
 	if err := flags.Parse(args); err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
 		return 1
@@ -141,6 +154,15 @@ func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
 		return 1
 	}
 
+	if redact {
+		c.Ui.Info("Redacting key material from snapshot")
+		err := redactSnapshot(tmpFile)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Could not redact snapshot: %v", err))
+			return 1
+		}
+	}
+
 	err = os.Rename(tmpFile.Name(), filename)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to finalize snapshot file: %v", err))
@@ -150,3 +172,57 @@ func (c *OperatorSnapshotSaveCommand) Run(args []string) int {
 	c.Ui.Output(fmt.Sprintf("State file written to %v", filename))
 	return 0
 }
+
+func redactSnapshot(srcFile *os.File) error {
+	srcFile.Seek(0, 0)
+	fsm, store, meta, err := raftutil.RestoreFromArchive(srcFile, nil)
+	if err != nil {
+		return fmt.Errorf("Failed to load snapshot from archive: %w", err)
+	}
+
+	iter, err := store.RootKeys(nil)
+	if err != nil {
+		return fmt.Errorf("Failed to query for root keys: %v", err)
+	}
+
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+		rootKey := raw.(*structs.RootKey)
+		if rootKey == nil {
+			break
+		}
+		if len(rootKey.WrappedKeys) > 0 {
+			rootKey.KeyID = rootKey.KeyID + " [REDACTED]"
+			rootKey.WrappedKeys = nil
+		}
+		msg, err := structs.Encode(structs.WrappedRootKeysUpsertRequestType,
+			&structs.KeyringUpsertWrappedRootKeyRequest{
+				WrappedRootKeys: rootKey,
+			})
+		if err != nil {
+			return fmt.Errorf("Could not re-encode redacted key: %v", err)
+		}
+
+		fsm.Apply(&raft.Log{
+			Type: raft.LogCommand,
+			Data: msg,
+		})
+	}
+
+	snap, err := snapshot.NewFromFSM(hclog.Default(), fsm, meta)
+	if err != nil {
+		return fmt.Errorf("Failed to create redacted snapshot: %v", err)
+	}
+
+	srcFile.Truncate(0)
+	srcFile.Seek(0, 0)
+
+	_, err = io.Copy(srcFile, snap)
+	if err != nil {
+		return fmt.Errorf("Failed to copy snapshot to temporary file: %v", err)
+	}
+	return nil
+}

command/operator_snapshot_state.go

@@ -85,7 +85,7 @@ func (c *OperatorSnapshotStateCommand) Run(args []string) int {
 	}
 	defer f.Close()
 
-	state, meta, err := raftutil.RestoreFromArchive(f, filter)
+	_, state, meta, err := raftutil.RestoreFromArchive(f, filter)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Failed to read archive file: %s", err))
 		return 1

helper/raftutil/fsm.go

@@ -14,6 +14,7 @@ import (
 	"github.com/hashicorp/go-memdb"
 	"github.com/hashicorp/nomad/nomad"
 	"github.com/hashicorp/nomad/nomad/state"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/raft"
 	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
 )
@@ -209,6 +210,7 @@ func StateAsMap(store *state.StateStore) map[string][]interface{} {
 		"Jobs":             toArray(store.Jobs(nil, state.SortDefault)),
 		"Nodes":            toArray(store.Nodes(nil)),
 		"PeriodicLaunches": toArray(store.PeriodicLaunches(nil)),
+		"RootKeys":         rootKeyMeta(store),
 		"SITokenAccessors": toArray(store.SITokenAccessors(nil)),
 		"ScalingEvents":    toArray(store.ScalingEvents(nil)),
 		"ScalingPolicies":  toArray(store.ScalingPolicies(nil)),
@@ -265,3 +267,27 @@ func toArray(iter memdb.ResultIterator, err error) []interface{} {
 
 	return r
 }
+
+// rootKeyMeta allows displaying keys without their key material
+func rootKeyMeta(store *state.StateStore) []any {
+
+	iter, err := store.RootKeys(nil)
+	if err != nil {
+		return []any{err}
+	}
+
+	keyMeta := []any{}
+	for {
+		raw := iter.Next()
+		if raw == nil {
+			break
+		}
+		k := raw.(*structs.RootKey)
+		if k == nil {
+			break
+		}
+		keyMeta = append(keyMeta, k.Meta())
+	}
+
+	return keyMeta
+}

helper/raftutil/snapshot.go

@@ -8,19 +8,18 @@ import (
 	"io"
 
 	"github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/raft"
-
 	"github.com/hashicorp/nomad/helper/snapshot"
 	"github.com/hashicorp/nomad/nomad"
 	"github.com/hashicorp/nomad/nomad/state"
+	"github.com/hashicorp/raft"
 )
 
-func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (*state.StateStore, *raft.SnapshotMeta, error) {
+func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (raft.FSM, *state.StateStore, *raft.SnapshotMeta, error) {
 	logger := hclog.L()
 
 	fsm, err := dummyFSM(logger)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create FSM: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to create FSM: %w", err)
 	}
 
 	// r is closed by RestoreFiltered, w is closed by CopySnapshot
@@ -40,13 +39,13 @@ func RestoreFromArchive(archive io.Reader, filter *nomad.FSMFilter) (*state.Stat
 
 	err = fsm.RestoreWithFilter(r, filter)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to restore from snapshot: %w", err)
+		return nil, nil, nil, fmt.Errorf("failed to restore from snapshot: %w", err)
 	}
 
 	select {
 	case err := <-errCh:
-		return nil, nil, err
+		return nil, nil, nil, err
 	case meta := <-metaCh:
-		return fsm.State(), meta, nil
+		return fsm, fsm.State(), meta, nil
 	}
 }

helper/snapshot/snapshot.go

@@ -43,6 +43,49 @@ func New(logger hclog.Logger, r *raft.Raft) (*Snapshot, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to open snapshot: %v:", err)
 	}
+
+	return writeSnapshot(logger, metadata, snap)
+}
+
+// NewFromFSM takes a state snapshot of the given FSM (for when we don't have a
+// Raft instance setup) into a temporary file and returns an object that gives
+// access to the file as an io.Reader. You must arrange to call Close() on the
+// returned object or else you will leak a temporary file.
+func NewFromFSM(logger hclog.Logger, fsm raft.FSM, meta *raft.SnapshotMeta) (*Snapshot, error) {
+	_, trans := raft.NewInmemTransport("")
+	snapshotStore := raft.NewInmemSnapshotStore()
+
+	fsmSnap, err := fsm.Snapshot()
+	if err != nil {
+		return nil, err
+	}
+
+	sink, err := snapshotStore.Create(meta.Version, meta.Index, meta.Term,
+		meta.Configuration, meta.ConfigurationIndex, trans)
+	if err != nil {
+		return nil, err
+	}
+	err = fsmSnap.Persist(sink)
+	if err != nil {
+		return nil, err
+	}
+
+	err = sink.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	snapshotID := sink.ID()
+	metadata, snap, err := snapshotStore.Open(snapshotID)
+	if err != nil {
+		return nil, err
+	}
+
+	return writeSnapshot(logger, metadata, snap)
+}
+
+func writeSnapshot(logger hclog.Logger, metadata *raft.SnapshotMeta, snap io.ReadCloser) (*Snapshot, error) {
+
 	defer func() {
 		if err := snap.Close(); err != nil {
 			logger.Error("Failed to close Raft snapshot", "error", err)