[wip] load from directory test almost working

app/client/cli/node.go

@@ -3,6 +3,8 @@ package cli
 import (
 	"fmt"
 
+	"github.com/pokt-network/pocket/app/client/cli/flags"
+	"github.com/pokt-network/pocket/rpc"
 	"github.com/spf13/cobra"
 )
 
@@ -11,6 +13,10 @@ func init() {
 	rootCmd.AddCommand(nodeCmd)
 }
 
+var (
+	dir string
+)
+
 func NewNodeCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "Node",
@@ -26,11 +32,28 @@ func NewNodeCommand() *cobra.Command {
 func nodeSaveCommands() []*cobra.Command {
 	cmds := []*cobra.Command{
 		{
-			Use: "Save",
+			Use:     "Save",
+			Short:   "save a backup of node databases in the provided directory",
+			Example: "node save --dir /dir/path/here/",
 			RunE: func(cmd *cobra.Command, args []string) error {
-				return fmt.Errorf("not impl")
+				client, err := rpc.NewClientWithResponses(flags.RemoteCLIURL)
+				if err != nil {
+					return err
+				}
+				resp, err := client.PostV1NodeBackup(cmd.Context(), rpc.NodeBackup{
+					Dir: &dir,
+				})
+				if err != nil {
+					return err
+				}
+				var dest []byte
+				_, err = resp.Body.Read(dest)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("%s", dest)
+				return nil
 			},
-			Short: "save a backup of world state",
 		},
 	}
 	return cmds

persistence/trees/atomic_test.go

@@ -2,9 +2,11 @@ package trees
 
 import (
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/pokt-network/pocket/logger"
@@ -13,7 +15,6 @@ import (
 	mockModules "github.com/pokt-network/pocket/shared/modules/mocks"
 
 	"github.com/golang/mock/gomock"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 )
 
@@ -105,52 +106,29 @@ func TestTreeStore_AtomicUpdatesWithSuccessfulRollback(t *testing.T) {
 	require.Nil(t, v)
 }
 
-func TestTreeStore_SaveAndLoad(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	tmpDir := t.TempDir()
-
-	mockTxIndexer := mock_types.NewMockTxIndexer(ctrl)
-	mockBus := mockModules.NewMockBus(ctrl)
-	mockPersistenceMod := mockModules.NewMockPersistenceModule(ctrl)
-
-	mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod)
-	mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer)
-
-	ts := &treeStore{
-		logger:       &zerolog.Logger{},
-		treeStoreDir: tmpDir,
-	}
-	require.NoError(t, ts.Start())
-	require.NotNil(t, ts.rootTree.tree)
-
-	for _, treeName := range stateTreeNames {
-		err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar"))
-		require.NoError(t, err)
-	}
+func TestTreeStore_LoadBackup(t *testing.T) {
+	ts := newTestTreeStore(t)
+	tmpdir := t.TempDir()
 
-	err := ts.Commit()
+	// assert that the directory is empty before backup
+	ok, err := isEmpty(tmpdir)
 	require.NoError(t, err)
+	require.True(t, ok)
 
+	// set the first hash for comparison
 	hash1 := ts.getStateHash()
-	require.NotEmpty(t, hash1)
 
-	w, err := ts.save()
-	require.NoError(t, err)
-	require.NotNil(t, w)
-	require.NotNil(t, w.rootHash)
-	require.NotNil(t, w.merkleRoots)
-
-	// Stop the first tree store so that it's databases are no longer used
+	// Trigger a backup, then shut down the treeStore.
+	require.NoError(t, ts.Backup(tmpdir))
 	require.NoError(t, ts.Stop())
 
 	// declare a second TreeStore with no trees then load the first worldstate into it
 	ts2 := &treeStore{
-		logger:       logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
-		treeStoreDir: tmpDir,
+		logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
 	}
 
 	// Load sets a tree store to the provided worldstate
-	err = ts2.Load(w)
+	err = ts2.Load(tmpdir)
 	require.NoError(t, err)
 
 	hash2 := ts2.getStateHash()
@@ -174,6 +152,20 @@ func TestTreeStore_SaveBackup(t *testing.T) {
 	ok, err = isEmpty(tmpdir)
 	require.NoError(t, err)
 	require.False(t, ok)
+
+	// assert on the worldstate.json file
+	data, err := readFile(filepath.Join(tmpdir, "worldstate.json"))
+	require.NoError(t, err)
+
+	var w *worldState
+	err = json.Unmarshal(data, &w)
+	require.NoError(t, err)
+
+	require.NotEmpty(t, w.merkleRoots)
+	require.Equal(t, len(w.merkleRoots), len(ts.merkleTrees))
+	for _, v := range w.merkleRoots {
+		require.NotEmpty(t, v)
+	}
 }
 
 // creates a new tree store with a tmp directory for nodestore persistence
@@ -190,7 +182,7 @@ func newTestTreeStore(t *testing.T) *treeStore {
 	mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer)
 
 	ts := &treeStore{
-		logger:       &zerolog.Logger{},
+		logger:       logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName),
 		treeStoreDir: tmpDir,
 	}
 	require.NoError(t, ts.Start())

persistence/trees/trees.go

@@ -15,8 +15,10 @@ package trees
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"hash"
+	"io/ioutil"
 	"log"
 	"path/filepath"
 
@@ -30,7 +32,6 @@ import (
 	"github.com/pokt-network/pocket/shared/modules"
 	"github.com/pokt-network/pocket/shared/modules/base_modules"
 	"github.com/pokt-network/smt"
-	"go.uber.org/multierr"
 )
 
 // smtTreeHasher sets the hasher used by the tree SMT trees
@@ -339,11 +340,25 @@ func (t *treeStore) Rollback() error {
 }
 
 // Load sets the TreeStore merkle and root trees to the values provided in the worldstate
-func (t *treeStore) Load(w *worldState) error {
+func (t *treeStore) Load(dir string) error {
+	// Look for a worldstate.json file to hydrate
+	data, err := readFile(filepath.Join(dir, "worldstate.json"))
+	if err != nil {
+		return err
+	}
+
+	// Hydrate a worldstate from the json object
+	var w *worldState
+	err = json.Unmarshal(data, &w)
+	if err != nil {
+		return err
+	}
+
 	t.merkleTrees = make(map[string]*stateTree)
 
-	// import root tree
-	nodeStore, err := kvstore.NewKVStore(fmt.Sprintf("%s/%s_nodes", t.treeStoreDir, RootTreeName))
+	// import root tree from worldState
+	path := formattedTreePath(dir, RootTreeName)
+	nodeStore, err := kvstore.NewKVStore(path)
 	if err != nil {
 		return err
 	}
@@ -353,13 +368,13 @@ func (t *treeStore) Load(w *worldState) error {
 		nodeStore: nodeStore,
 	}
 
-	// import merkle trees
+	// import merkle tree roots trees from worldState
 	for treeName, treeRootHash := range w.merkleRoots {
-		nodeStore, err := kvstore.NewKVStore(fmt.Sprintf("%s/%s_nodes", w.treeStoreDir, treeName))
+		path := formattedTreePath(dir, treeName)
+		nodeStore, err := kvstore.NewKVStore(path)
 		if err != nil {
 			return err
 		}
-
 		t.merkleTrees[treeName] = &stateTree{
 			name:      treeName,
 			nodeStore: nodeStore,
@@ -412,15 +427,31 @@ func (t *treeStore) save() (*worldState, error) {
 // Backup creates a new backup of each tree in the tree store to the provided directory.
 // Each tree is backed up in an eponymous file in the provided backupDir.
 func (t *treeStore) Backup(backupDir string) error {
-	errs := []error{}
+	// save all current branches
+	if err := t.Commit(); err != nil {
+		return err
+	}
+
+	w := &worldState{
+		rootHash:     []byte(t.getStateHash()),
+		merkleRoots:  make(map[string][]byte),
+		treeStoreDir: backupDir, // TODO IN THIS COMMIT make sure this is the proper formatting
+	}
+
 	for _, st := range t.merkleTrees {
-		treePath := filepath.Join(backupDir, st.name)
-		if err := st.nodeStore.Backup(treePath); err != nil {
+		if err := st.nodeStore.Backup(formattedTreePath(backupDir, st.name)); err != nil {
 			t.logger.Err(err).Msgf("failed to backup %s tree: %+v", st.name, err)
-			errs = append(errs, err)
+			return err
 		}
+		w.merkleRoots[st.name] = st.tree.Root()
+	}
+
+	err := writeFile(filepath.Join(backupDir, "worldstate.json"), w)
+	if err != nil {
+		return err
 	}
-	return multierr.Combine(errs...)
+
+	return nil
 }
 
 ////////////////////////
@@ -568,3 +599,30 @@ func getTransactions(txi indexer.TxIndexer, height uint64) ([]*coreTypes.Indexed
 	}
 	return indexedTxs, nil
 }
+
+func readFile(filePath string) ([]byte, error) {
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func writeFile(filePath string, data interface{}) error {
+	jsonData, err := json.MarshalIndent(data, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(filePath, jsonData, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// defines a standard naming scheme for each tree's storage location
+func formattedTreePath(dir, treeName string) string {
+	return fmt.Sprintf("%s/%s_nodes", dir, treeName)
+}

rpc/handlers_node.go

@@ -1,11 +1,22 @@
 package rpc
 
 import (
-	"fmt"
-
 	"github.com/labstack/echo/v4"
 )
 
 func (s *rpcServer) PostV1NodeBackup(ctx echo.Context) error {
-	return fmt.Errorf("not impl")
+	store := s.GetBus().GetPersistenceModule().GetBus().GetTreeStore()
+	rw, err := s.GetBus().GetPersistenceModule().NewRWContext(0)
+	if err != nil {
+		return err
+	}
+	if err := rw.SetSavePoint(); err != nil {
+		return err
+	}
+	if err := store.Backup(ctx.Param("dir")); err != nil {
+		return err
+	}
+	rw.Release()
+	s.logger.Info().Msgf("backup created in %s", ctx.Param("dir"))
+	return nil
 }

rpc/v1/openapi.yaml

@@ -953,39 +953,35 @@ paths:
                   - "servicer"
   /v1/node/backup:
     post:
-      summary: Create a backup of all node databases
-      description: Create a backup at the specified location
+      tags:
+        - node
+      summary: Creates a backup of all node databases in the specified directory
       requestBody:
-        required: true
+        description: Request backup creation in the specified dir 
         content:
-          multipart/form-data:
+          application/json:
             schema:
-              type: object
-              properties:
-                file:
-                  type: string
-                  format: binary
+              $ref: "#/components/schemas/NodeBackup"
+            example:
+              dir: /path/to/backup/dir/
+        required: true
       responses:
-        '200':
-          description: OK
+        "200":
+          description: Returns account data at the specified height
           content:
             application/json:
               schema:
-                type: object
-                properties:
-                  location:
-                    type: string
-                    description: The location of the saved file
-        '400':
-          description: Bad Request
+                $ref: "#/components/schemas/Account"
+        "400":
+          description: Bad request
           content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  error:
-                    type: string
-                    description: Description of the error
+            text/plain:
+              example: "TODO"
+        "500":
+          description: An error occurred while retrieving the account data at the specified height
+          content:
+            text/plain:
+              example: "TODO"
 
 
 externalDocs:
@@ -1911,7 +1907,12 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/PartialSignature"
-
+    NodeBackup: 
+      type: object
+      properties:
+        dir:
+          type: string
+
   securitySchemes: {}
   links: {}
   callbacks: {}