diff --git a/app/client/cli/node.go b/app/client/cli/node.go index 96a8674c6..1e8245200 100644 --- a/app/client/cli/node.go +++ b/app/client/cli/node.go @@ -1,12 +1,22 @@ package cli -import "github.com/spf13/cobra" +import ( + "fmt" + + "github.com/pokt-network/pocket/app/client/cli/flags" + "github.com/pokt-network/pocket/rpc" + "github.com/spf13/cobra" +) func init() { nodeCmd := NewNodeCommand() rootCmd.AddCommand(nodeCmd) } +var ( + dir string +) + func NewNodeCommand() *cobra.Command { cmd := &cobra.Command{ Use: "Node", @@ -14,5 +24,37 @@ func NewNodeCommand() *cobra.Command { Aliases: []string{"node", "n"}, } + cmd.AddCommand(nodeSaveCommands()...) + return cmd } + +func nodeSaveCommands() []*cobra.Command { + cmds := []*cobra.Command{ + { + Use: "Save", + Short: "save a backup of node databases in the provided directory", + Example: "node save --dir /dir/path/here/", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpc.NewClientWithResponses(flags.RemoteCLIURL) + if err != nil { + return err + } + resp, err := client.PostV1NodeBackup(cmd.Context(), rpc.NodeBackup{ + Dir: &dir, + }) + if err != nil { + return err + } + var dest []byte + _, err = resp.Body.Read(dest) + if err != nil { + return err + } + fmt.Printf("%s", dest) + return nil + }, + }, + } + return cmds +} diff --git a/app/docs/CHANGELOG.md b/app/docs/CHANGELOG.md index 1b2098eb3..f48881158 100644 --- a/app/docs/CHANGELOG.md +++ b/app/docs/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.0.0.9] - 2023-07-28 + +- Adds the `Save` sub-command CLI +- Adds `/v1/node/backup` to the OpenAPI spec + ## [0.0.0.8] - 2023-06-06 - Adds `query nodeRoles` sub-command the client CLI diff --git a/e2e/tests/node.feature b/e2e/tests/node.feature new file mode 100644 index 000000000..d04c02b82 --- /dev/null +++ b/e2e/tests/node.feature @@ -0,0 +1,7 @@ +Feature: Node Namespace + + Scenario: User Wants Help Using The Node Command + Given the user has a validator + When the user runs the command "Node help" + Then the user should be able to see standard output containing "Available Commands" + And the validator should have exited without error diff --git a/internal/testutil/trees.go b/internal/testutil/trees.go new file mode 100644 index 000000000..e67eb2cda --- /dev/null +++ b/internal/testutil/trees.go @@ -0,0 +1,124 @@ +package testutil + +import ( + "log" + "testing" + + "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/persistence" + "github.com/pokt-network/pocket/persistence/trees" + "github.com/pokt-network/pocket/runtime" + "github.com/pokt-network/pocket/runtime/configs" + "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/runtime/test_artifacts/keygen" + "github.com/pokt-network/pocket/shared/messaging" + "github.com/pokt-network/pocket/shared/modules" + + "github.com/stretchr/testify/require" +) + +var ( + testSchema = "test_schema" + + genesisStateNumValidators = 5 + genesisStateNumServicers = 1 + genesisStateNumApplications = 1 +) + +// creates a new tree store with a tmp directory for nodestore persistence +// and then starts the tree store and returns its pointer. +func NewTestTreeStoreSubmodule(t *testing.T, bus modules.Bus) modules.TreeStoreModule { + t.Helper() + + tmpDir := t.TempDir() + ts, err := trees.Create( + bus, + trees.WithTreeStoreDirectory(tmpDir), + trees.WithLogger(logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName))) + require.NoError(t, err) + + err = ts.Start() + require.NoError(t, err) + + t.Cleanup(func() { + err := ts.Stop() + require.NoError(t, err) + }) + + return ts +} + +func SeedTestTreeStoreSubmodule(t *testing.T, mod modules.TreeStoreModule) modules.TreeStoreModule { + // TODO insert transaction data into postgres + // TODO trigger an update with a pgx connection + return mod +} + +func NewTestPersistenceModule(t *testing.T, databaseUrl string) modules.PersistenceModule { + teardownDeterministicKeygen := keygen.GetInstance().SetSeed(42) + defer teardownDeterministicKeygen() + + cfg := &configs.Config{ + Persistence: &configs.PersistenceConfig{ + PostgresUrl: databaseUrl, + NodeSchema: testSchema, + BlockStorePath: ":memory:", + TxIndexerPath: ":memory:", + TreesStoreDir: ":memory:", + MaxConnsCount: 5, + MinConnsCount: 1, + MaxConnLifetime: "5m", + MaxConnIdleTime: "1m", + HealthCheckPeriod: "30s", + }, + } + + genesisState, _ := test_artifacts.NewGenesisState( + genesisStateNumValidators, + genesisStateNumServicers, + genesisStateNumApplications, + genesisStateNumServicers, + ) + + runtimeMgr := runtime.NewManager(cfg, genesisState) + bus, err := runtime.CreateBus(runtimeMgr) + require.NoError(t, err) + + persistenceMod, err := persistence.Create(bus) + require.NoError(t, err) + + return persistenceMod.(modules.PersistenceModule) +} + +func NewTestPostgresContext(t testing.TB, pmod modules.PersistenceModule, height int64) *persistence.PostgresContext { + rwCtx, err := pmod.NewRWContext(height) + if err != nil { + log.Fatalf("Error creating new context: %v\n", err) + } + + postgresCtx, ok := rwCtx.(*persistence.PostgresContext) + if !ok { + log.Fatalf("Error casting RW context to Postgres context") + } + + // TECHDEBT: This should not be part of `NewTestPostgresContext`. It causes unnecessary resets + // if we call `NewTestPostgresContext` more than once in a single test. + t.Cleanup(func() { + resetStateToGenesis(pmod) + }) + + return postgresCtx +} + +// This is necessary for unit tests that are dependant on a baseline genesis state +func resetStateToGenesis(pmod modules.PersistenceModule) { + if err := pmod.ReleaseWriteContext(); err != nil { + log.Fatalf("Error releasing write context: %v\n", err) + } + if err := pmod.HandleDebugMessage(&messaging.DebugMessage{ + Action: messaging.DebugMessageAction_DEBUG_PERSISTENCE_RESET_TO_GENESIS, + Message: nil, + }); err != nil { + log.Fatalf("Error clearing state: %v\n", err) + } +} diff --git a/persistence/blockstore/block_store.go b/persistence/blockstore/block_store.go index 9bc4aa5a4..9eafdaac9 100644 --- a/persistence/blockstore/block_store.go +++ b/persistence/blockstore/block_store.go @@ -4,6 +4,7 @@ package blockstore import ( "fmt" + "path/filepath" "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/shared/codec" @@ -11,6 +12,9 @@ import ( "github.com/pokt-network/pocket/shared/utils" ) +// backupName is the name of the archive file that is created when Backup is called for a BlockStore +const backupName = "blockstore.bak" + // BlockStore is a key-value store that maps block heights to serialized // block structures. // * It manages the atomic state transitions for applying a Unit of Work. @@ -93,8 +97,8 @@ func (bs *blockStore) Stop() error { return bs.kv.Stop() } -func (bs *blockStore) Backup(path string) error { - return bs.kv.Backup(path) +func (bs *blockStore) Backup(dir string) error { + return bs.kv.Backup(filepath.Join(dir, backupName)) } /////////////// diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index f4ea132d2..b25a35a22 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -26,7 +26,8 @@ type KVStore interface { Exists(key []byte) (bool, error) ClearAll() error - Backup(filepath string) error + // Backup takes a directory and makes a backup of the KVStore in that directory. + Backup(dir string) error } const ( diff --git a/persistence/trees/atomic_test.go b/persistence/trees/atomic_test.go index a02f2afd9..902281b7d 100644 --- a/persistence/trees/atomic_test.go +++ b/persistence/trees/atomic_test.go @@ -112,72 +112,55 @@ func TestTreeStore_SaveAndLoad(t *testing.T) { t.Parallel() t.Run("should save a backup in a directory", func(t *testing.T) { ts := newTestTreeStore(t) - tmpdir := t.TempDir() + backupDir := t.TempDir() // assert that the directory is empty before backup - ok, err := isEmpty(tmpdir) + ok, err := isEmpty(backupDir) require.NoError(t, err) require.True(t, ok) // Trigger a backup - require.NoError(t, ts.Backup(tmpdir)) + require.NoError(t, ts.Backup(backupDir)) // assert that the directory is not empty after Backup has returned - ok, err = isEmpty(tmpdir) + ok, err = isEmpty(backupDir) require.NoError(t, err) require.False(t, ok) }) t.Run("should load a backup and maintain TreeStore hash integrity", func(t *testing.T) { - ctrl := gomock.NewController(t) - tmpDir := t.TempDir() - - mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) - mockBus := mock_modules.NewMockBus(ctrl) - mockPersistenceMod := mock_modules.NewMockPersistenceModule(ctrl) - - mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) - mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) - - ts := &treeStore{ - logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), - treeStoreDir: tmpDir, - } - require.NoError(t, ts.Start()) - require.NotNil(t, ts.rootTree.tree) - - for _, treeName := range stateTreeNames { - err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar")) - require.NoError(t, err) - } + // create a new tree store and save it's initial hash + ts := newTestTreeStore(t) + hash1 := ts.getStateHash() - err := ts.Commit() + // make a temp directory for the backup and assert it's empty + backupDir := t.TempDir() + empty, err := isEmpty(backupDir) require.NoError(t, err) + require.True(t, empty) - hash1 := ts.getStateHash() - require.NotEmpty(t, hash1) + // make a backup + err = ts.Backup(backupDir) + require.NoError(t, err) - w, err := ts.save() + // assert directory is not empty after backup + empty2, err := isEmpty(backupDir) require.NoError(t, err) - require.NotNil(t, w) - require.NotNil(t, w.rootHash) - require.NotNil(t, w.merkleRoots) + require.False(t, empty2) - // Stop the first tree store so that it's databases are no longer used + // stop the first tree store so that it's databases are released require.NoError(t, ts.Stop()) // declare a second TreeStore with no trees then load the first worldstate into it ts2 := &treeStore{ - logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), - treeStoreDir: tmpDir, + logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), } - // Load sets a tree store to the provided worldstate - err = ts2.Load(w) + // call load with the backup directory + err = ts2.Load(backupDir) require.NoError(t, err) + // assert that hash is unchanged from save and load hash2 := ts2.getStateHash() - - // Assert that hash is unchanged from save and load - require.Equal(t, hash1, hash2) + require.Equal(t, hash1, hash2, "failed to maintain hash integrity") }) } diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 8c5c936bb..ed2701391 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -15,10 +15,11 @@ package trees import ( "crypto/sha256" "encoding/hex" - "errors" + "encoding/json" "fmt" "hash" "log" + "os" "path/filepath" "github.com/jackc/pgx/v5" @@ -117,6 +118,16 @@ type worldState struct { merkleRoots map[string][]byte } +// worldStateJson holds exported members for proper JSON marshaling and unmarshaling. +// It contains the root hash of the merkle roots as a byte slice and a map of the MerkleRoots +// where each key is the name of the file in the same directory that corresponds to the baderDB +// backup file for that tree. That tree's hash is the value of that object for checking the integrity +// of each file and tree. +type worldStateJson struct { + RootHash []byte + MerkleRoots map[string][]byte // +} + // GetTree returns the root hash and nodeStore for the matching tree stored in the TreeStore. // This enables the caller to import the SMT without changing the one stored unless they call // `Commit()` to write to the nodestore. @@ -298,7 +309,8 @@ func (t *treeStore) getStateHash() string { } // Convert the array to a slice and return it // REF: https://stackoverflow.com/questions/28886616/convert-array-to-slice-in-go - hexHash := hex.EncodeToString(t.rootTree.tree.Root()) + root := t.rootTree.tree.Root() + hexHash := hex.EncodeToString(root) t.logger.Info().Msgf("#️⃣ calculated state hash: %s", hexHash) return hexHash } @@ -329,35 +341,51 @@ func (t *treeStore) Rollback() error { return ErrFailedRollback } -// Load sets the TreeStore trees to the values provided in the worldstate -func (t *treeStore) Load(w *worldState) error { - t.merkleTrees = make(map[string]*stateTree) +// Load sets the TreeStore merkle and root trees to the values provided in the worldstate +func (t *treeStore) Load(dir string) error { + // look for a worldstate.json file to hydrate + data, err := readFile(filepath.Join(dir, "worldstate.json")) + if err != nil { + return err + } - // import root tree - rootTreePath := fmt.Sprintf("%s/%s_nodes", t.treeStoreDir, RootTreeName) - nodeStore, err := kvstore.NewKVStore(rootTreePath) + // assign tree store directory to dir if a valid worldstate.json exists + t.treeStoreDir = dir + + // hydrate a worldstate from the json object + var w *worldStateJson + err = json.Unmarshal(data, &w) + if err != nil { + return err + } + + t.logger.Info().Msgf("🌏 worldstate detected, beginning import at %s", dir) + + // create a new root tree and node store + nodeStore, err := kvstore.NewKVStore(fmt.Sprintf("%s/%s_nodes", t.treeStoreDir, RootTreeName)) if err != nil { return err } t.rootTree = &stateTree{ name: RootTreeName, - tree: smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, w.rootHash), + tree: smt.NewSparseMerkleTree(nodeStore, smtTreeHasher), nodeStore: nodeStore, } - // import merkle trees - for treeName, treeRootHash := range w.merkleRoots { - treePath := fmt.Sprintf("%s/%s_nodes", w.treeStoreDir, treeName) + // import merkle trees with the proper hash + t.merkleTrees = make(map[string]*stateTree) + for treeName, treeRootHash := range w.MerkleRoots { + treePath := fmt.Sprintf("%s/%s_nodes", dir, treeName) nodeStore, err := kvstore.NewKVStore(treePath) if err != nil { return err } - t.merkleTrees[treeName] = &stateTree{ name: treeName, - nodeStore: nodeStore, tree: smt.ImportSparseMerkleTree(nodeStore, smtTreeHasher, treeRootHash), + nodeStore: nodeStore, } + t.logger.Info().Msgf("🌳 %s initialized at %s", treeName, hex.EncodeToString(w.MerkleRoots[treeName])) } return nil @@ -404,15 +432,34 @@ func (t *treeStore) save() (*worldState, error) { // Backup creates a new backup of each tree in the tree store to the provided directory. // Each tree is backed up in an eponymous file in the provided backupDir. func (t *treeStore) Backup(backupDir string) error { - errs := []error{} + // save all current branches + if err := t.Commit(); err != nil { + return err + } + + w := &worldStateJson{ + RootHash: []byte(t.getStateHash()), + MerkleRoots: make(map[string][]byte), + } + for _, st := range t.merkleTrees { - treePath := filepath.Join(backupDir, st.name) + treePath := fmt.Sprintf("%s/%s_nodes.bak", backupDir, st.name) if err := st.nodeStore.Backup(treePath); err != nil { t.logger.Err(err).Msgf("failed to backup %s tree: %+v", st.name, err) - errs = append(errs, err) + return err } + w.MerkleRoots[st.name] = st.tree.Root() } - return errors.Join(errs...) + + worldstatePath := filepath.Join(backupDir, "worldstate.json") + err := writeFile(worldstatePath, w) + if err != nil { + return err + } + + t.logger.Info().Msgf("💾 backup created at %s", backupDir) + + return nil } //////////////////////// @@ -560,3 +607,46 @@ func getTransactions(txi indexer.TxIndexer, height uint64) ([]*coreTypes.Indexed } return indexedTxs, nil } + +func readFile(filePath string) ([]byte, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + // Use os.Stat to get file size and read the content into a byte slice + stat, err := file.Stat() + if err != nil { + return nil, err + } + + data := make([]byte, stat.Size()) + _, err = file.Read(data) + if err != nil { + return nil, err + } + + return data, nil +} + +func writeFile(filePath string, data interface{}) error { + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + // Use the json.MarshalIndent function to encode data into JSON format with indentation + jsonData, err := json.MarshalIndent(data, "", " ") + if err != nil { + return err + } + + _, err = file.Write(jsonData) + if err != nil { + return err + } + + return nil +} diff --git a/rpc/handlers_node.go b/rpc/handlers_node.go new file mode 100644 index 000000000..da878ad09 --- /dev/null +++ b/rpc/handlers_node.go @@ -0,0 +1,39 @@ +package rpc + +import ( + "fmt" + + "github.com/labstack/echo/v4" +) + +// PostV1NodeBackup triggers a backup of the TreeStore, the BlockStore, the PostgreSQL database. +// TECHDEBT: Run each backup process in a goroutine to as elapsed time will become significant +// with the current waterfall approach when even a moderate amount of data resides in each store. +func (s *rpcServer) PostV1NodeBackup(ctx echo.Context) error { + dir := ctx.Param("dir") + if dir == "" { + // TECHDEBT: Wire this up to a config param with a sane default + // cfg := s.GetBus().GetRuntimeMgr().GetConfig() + return fmt.Errorf("must specify a target backup directory") + } + + s.logger.Info().Msgf("creating backup in %s", dir) + + // backup the TreeStore + if err := s.GetBus().GetTreeStore().Backup(dir); err != nil { + return err + } + + // backup the BlockStore + if err := s.GetBus().GetPersistenceModule().GetBlockStore().Backup(dir); err != nil { + return err + } + + // TECHDEBT: backup Postgres + // if err := s.GetBus().GetPersistenceModule().Backup(dir); err != nil { + // return err + // } + + s.logger.Info().Msgf("backup created in %s", dir) + return nil +} diff --git a/rpc/handlers_test.go b/rpc/handlers_test.go new file mode 100644 index 000000000..96383ee47 --- /dev/null +++ b/rpc/handlers_test.go @@ -0,0 +1,104 @@ +package rpc + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/pokt-network/pocket/internal/testutil" + "github.com/pokt-network/pocket/logger" + "github.com/pokt-network/pocket/runtime/test_artifacts" + "github.com/pokt-network/pocket/shared/modules" + "github.com/stretchr/testify/require" + + "github.com/labstack/echo/v4" +) + +func Test_RPCPostV1NodeBackup(t *testing.T) { + type testCase struct { + name string + setup func(t *testing.T, e echo.Context) *rpcServer + assert func(t *testing.T, tt testCase, e echo.Context, s *rpcServer) + wantErr bool + } + + // NB: testDir is used and cleared by each test case + var testDir = t.TempDir() + + tests := []testCase{ + { + name: "should create a backup in the specified directory", + setup: func(t *testing.T, e echo.Context) *rpcServer { + _, _, url := test_artifacts.SetupPostgresDocker() + pmod := testutil.NewTestPersistenceModule(t, url) + + s := &rpcServer{ + logger: *logger.Global.CreateLoggerForModule(modules.RPCModuleName), + } + + s.SetBus(pmod.GetBus()) + + e.SetParamNames("dir") + e.SetParamValues(testDir) + + return s + }, + wantErr: false, + assert: func(t *testing.T, tt testCase, e echo.Context, s *rpcServer) { + f, err := os.Open(testDir) + require.NoError(t, err) + dirs, err := f.ReadDir(-1) + require.NoError(t, err) + // assert that we wrote the expected 12 files into this directory + require.True(t, len(dirs) == 12) + + // assert worldstate.json was written + _, err = os.Open(filepath.Join(testDir, "worldstate.json")) + require.NoError(t, err) + + // assert blockstore was written + _, err = os.Open(filepath.Join(testDir, "blockstore.bak")) + require.NoError(t, err) + + // cleanup the directory after each test + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(testDir)) + }) + }, + }, + { + name: "should error if no directory specified", + setup: func(t *testing.T, e echo.Context) *rpcServer { + _, _, url := test_artifacts.SetupPostgresDocker() + pmod := testutil.NewTestPersistenceModule(t, url) + + s := &rpcServer{ + logger: *logger.Global.CreateLoggerForModule(modules.RPCModuleName), + } + + s.SetBus(pmod.GetBus()) + + return s + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/v1/node/backup", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + s := tt.setup(t, c) + + if err := s.PostV1NodeBackup(c); (err != nil) != tt.wantErr { + t.Errorf("rpcServer.PostV1NodeBackup() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.assert != nil { + tt.assert(t, tt, c, s) + } + }) + } +} diff --git a/rpc/v1/openapi.yaml b/rpc/v1/openapi.yaml index c068b4238..c616875c6 100644 --- a/rpc/v1/openapi.yaml +++ b/rpc/v1/openapi.yaml @@ -951,6 +951,38 @@ paths: node_roles: - "validator" - "servicer" + /v1/node/backup: + post: + tags: + - node + summary: Creates a backup of all node databases in the specified directory + requestBody: + description: Request backup creation in the specified dir + content: + application/json: + schema: + $ref: "#/components/schemas/NodeBackup" + example: + dir: /path/to/backup/dir/ + required: true + responses: + "200": + description: Returns account data at the specified height + content: + application/json: + schema: + $ref: "#/components/schemas/Account" + "400": + description: Bad request + content: + text/plain: + example: "TODO" + "500": + description: An error occurred while retrieving the account data at the specified height + content: + text/plain: + example: "TODO" + externalDocs: description: Find out more about Pocket Network @@ -1875,7 +1907,12 @@ components: type: array items: $ref: "#/components/schemas/PartialSignature" - + NodeBackup: + type: object + properties: + dir: + type: string + securitySchemes: {} links: {} callbacks: {} diff --git a/shared/modules/persistence_module.go b/shared/modules/persistence_module.go index 38c7681ba..3bb60cc8a 100644 --- a/shared/modules/persistence_module.go +++ b/shared/modules/persistence_module.go @@ -276,3 +276,9 @@ type PersistenceLocalContext interface { // for the application associated with the session GetSessionTokensUsed(*coreTypes.Session) (*big.Int, error) } + +// Archivable defines a single function interface for creating a reusable archive of a module. +type Archivable interface { + // Backup creates an archive for the module in the given directory and returns an error if anything went wrong. + Backup(dir string) error +} diff --git a/shared/modules/treestore_module.go b/shared/modules/treestore_module.go index bddc2451c..06629a4cf 100644 --- a/shared/modules/treestore_module.go +++ b/shared/modules/treestore_module.go @@ -23,6 +23,7 @@ type TreeStoreModule interface { treeStoreFactory AtomicStore + Archivable // Update returns the computed state hash for a given height. // * Height is passed through to the Update function and is used to query the TxIndexer for transactions