Skip to content

Commit 948f08b

Browse files
ccecce
authored
config: move crash and stateproof DB defaults to hot dir (algorand#5817)
1 parent 5d047a4 commit 948f08b

File tree

5 files changed

+170
-21
lines changed

5 files changed

+170
-21
lines changed

‎config/config_test.go

Lines changed: 117 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -675,6 +675,12 @@ func TestEnsureAbsDir(t *testing.T) {
675675
require.Equal(t, testDirectory+"/myGenesisID", t2Abs)
676676
}
677677

678+
type tLogger struct{ t *testing.T }
679+
680+
func (l tLogger) Infof(fmts string, args ...interface{}) {
681+
l.t.Logf(fmts, args...)
682+
}
683+
678684
// TestEnsureAndResolveGenesisDirs confirms that paths provided in the config are resolved to absolute paths and are created if relevant
679685
func TestEnsureAndResolveGenesisDirs(t *testing.T) {
680686
partitiontest.PartitionTest(t)
@@ -689,7 +695,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) {
689695
cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof")
690696
cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
691697

692-
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
698+
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
693699
require.NoError(t, err)
694700

695701
// confirm that the paths are absolute, and contain the genesisID
@@ -711,7 +717,7 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
711717

712718
cfg := GetDefaultLocal()
713719
testDirectory := t.TempDir()
714-
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
720+
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
715721
require.NoError(t, err)
716722
// confirm that if only the root is specified, it is used for all directories
717723
require.Equal(t, testDirectory+"/myGenesisID", paths.TrackerGenesisDir)
@@ -731,21 +737,125 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
731737
cold := filepath.Join(testDirectory, "cold")
732738
cfg.HotDataDir = hot
733739
cfg.ColdDataDir = cold
734-
paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
740+
paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
735741
require.NoError(t, err)
736742
// confirm that if hot/cold are specified, hot/cold are used for appropriate directories
737743
require.Equal(t, hot+"/myGenesisID", paths.TrackerGenesisDir)
738744
require.DirExists(t, paths.TrackerGenesisDir)
739745
require.Equal(t, cold+"/myGenesisID", paths.BlockGenesisDir)
740746
require.DirExists(t, paths.BlockGenesisDir)
741-
require.Equal(t, cold+"/myGenesisID", paths.CrashGenesisDir)
747+
require.Equal(t, hot+"/myGenesisID", paths.CrashGenesisDir)
742748
require.DirExists(t, paths.CrashGenesisDir)
743-
require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir)
749+
require.Equal(t, hot+"/myGenesisID", paths.StateproofGenesisDir)
744750
require.DirExists(t, paths.StateproofGenesisDir)
745751
require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir)
746752
require.DirExists(t, paths.CatchpointGenesisDir)
747753
}
748754

755+
func TestEnsureAndResolveGenesisDirs_migrate(t *testing.T) {
756+
partitiontest.PartitionTest(t)
757+
758+
cfg := GetDefaultLocal()
759+
testDirectory := t.TempDir()
760+
cfg.HotDataDir = filepath.Join(testDirectory, "hot")
761+
cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
762+
coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
763+
hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
764+
err := os.MkdirAll(coldDir, 0755)
765+
require.NoError(t, err)
766+
// put a crash.sqlite file in the ColdDataDir
767+
err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
768+
require.NoError(t, err)
769+
err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
770+
require.NoError(t, err)
771+
// put a stateproof.sqlite file in the ColdDataDir
772+
err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
773+
require.NoError(t, err)
774+
err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
775+
require.NoError(t, err)
776+
// Resolve
777+
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
778+
require.NoError(t, err)
779+
// Confirm that crash.sqlite was moved to HotDataDir
780+
require.DirExists(t, paths.CrashGenesisDir)
781+
require.Equal(t, hotDir, paths.CrashGenesisDir)
782+
require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite"))
783+
require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
784+
require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
785+
require.FileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
786+
// Confirm that stateproof.sqlite was moved to HotDataDir
787+
require.DirExists(t, paths.StateproofGenesisDir)
788+
require.Equal(t, hotDir, paths.StateproofGenesisDir)
789+
require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
790+
require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
791+
require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
792+
require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
793+
}
794+
795+
func TestEnsureAndResolveGenesisDirs_migrateCrashFail(t *testing.T) {
796+
partitiontest.PartitionTest(t)
797+
798+
cfg := GetDefaultLocal()
799+
testDirectory := t.TempDir()
800+
cfg.HotDataDir = filepath.Join(testDirectory, "hot")
801+
cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
802+
coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
803+
hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
804+
err := os.MkdirAll(coldDir, 0755)
805+
require.NoError(t, err)
806+
err = os.MkdirAll(hotDir, 0755)
807+
require.NoError(t, err)
808+
// put a crash.sqlite file in the ColdDataDir
809+
err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
810+
require.NoError(t, err)
811+
err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
812+
require.NoError(t, err)
813+
// also put a crash.sqlite file in the HotDataDir
814+
err = os.WriteFile(filepath.Join(hotDir, "crash.sqlite"), []byte("test"), 0644)
815+
require.NoError(t, err)
816+
// Resolve
817+
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
818+
require.Error(t, err)
819+
require.Empty(t, paths)
820+
// Confirm that crash.sqlite was not moved to HotDataDir
821+
require.FileExists(t, filepath.Join(coldDir, "crash.sqlite"))
822+
require.FileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
823+
require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
824+
require.NoFileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
825+
}
826+
827+
func TestEnsureAndResolveGenesisDirs_migrateSPFail(t *testing.T) {
828+
partitiontest.PartitionTest(t)
829+
830+
cfg := GetDefaultLocal()
831+
testDirectory := t.TempDir()
832+
cfg.HotDataDir = filepath.Join(testDirectory, "hot")
833+
cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
834+
coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
835+
hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
836+
err := os.MkdirAll(coldDir, 0755)
837+
require.NoError(t, err)
838+
err = os.MkdirAll(hotDir, 0755)
839+
require.NoError(t, err)
840+
// put a stateproof.sqlite file in the ColdDataDir
841+
err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
842+
require.NoError(t, err)
843+
err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
844+
require.NoError(t, err)
845+
// also put a stateproof.sqlite-wal file in the HotDataDir
846+
err = os.WriteFile(filepath.Join(hotDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
847+
require.NoError(t, err)
848+
// Resolve
849+
paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
850+
require.Error(t, err)
851+
require.Empty(t, paths)
852+
// Confirm that stateproof.sqlite was not moved to HotDataDir
853+
require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
854+
require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
855+
require.NoFileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
856+
require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
857+
}
858+
749859
// TestEnsureAndResolveGenesisDirsError confirms that if a path can't be created, an error is returned
750860
func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
751861
partitiontest.PartitionTest(t)
@@ -761,15 +871,15 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
761871
cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
762872

763873
// first try an error with an empty root dir
764-
paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID")
874+
paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID", tLogger{t: t})
765875
require.Empty(t, paths)
766876
require.Error(t, err)
767877
require.Contains(t, err.Error(), "rootDir is required")
768878

769879
require.NoError(t, os.Chmod(testDirectory, 0200))
770880

771881
// now try an error with a root dir that can't be written to
772-
paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
882+
paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
773883
require.Empty(t, paths)
774884
require.Error(t, err)
775885
require.Contains(t, err.Error(), "permission denied")

‎config/localTemplate.go

Lines changed: 48 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,13 @@ type Local struct {
111111
// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
112112
// If not specified, the node will use the ColdDataDir.
113113
CatchpointDir string `version[31]:""`
114-
// StateproofDir is an optional directory to store stateproof data.
114+
// StateproofDir is an optional directory to persist state about observed and issued state proof messages.
115115
// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
116-
// If not specified, the node will use the ColdDataDir.
116+
// If not specified, the node will use the HotDataDir.
117117
StateproofDir string `version[31]:""`
118-
// CrashDBDir is an optional directory to store the crash database.
118+
// CrashDBDir is an optional directory to persist agreement's consensus participation state.
119119
// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
120-
// If not specified, the node will use the ColdDataDir.
120+
// If not specified, the node will use the HotDataDir
121121
CrashDBDir string `version[31]:""`
122122

123123
// LogFileDir is an optional directory to store the log, node.log
@@ -785,9 +785,13 @@ func (cfg *Local) ResolveLogPaths(rootDir string) (liveLog, archive string) {
785785
return liveLog, archive
786786
}
787787

788+
type logger interface {
789+
Infof(format string, args ...interface{})
790+
}
791+
788792
// EnsureAndResolveGenesisDirs will resolve the supplied config paths to absolute paths, and will create the genesis directories of each
789793
// returns a ResolvedGenesisDirs struct with the resolved paths for use during runtime
790-
func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (ResolvedGenesisDirs, error) {
794+
func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string, logger logger) (ResolvedGenesisDirs, error) {
791795
var resolved ResolvedGenesisDirs
792796
var err error
793797
if rootDir != "" {
@@ -843,27 +847,62 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv
843847
} else {
844848
resolved.CatchpointGenesisDir = resolved.ColdGenesisDir
845849
}
846-
// if StateproofDir is not set, use ColdDataDir
850+
// if StateproofDir is not set, use HotDataDir
847851
if cfg.StateproofDir != "" {
848852
resolved.StateproofGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofDir, genesisID)
849853
if err != nil {
850854
return ResolvedGenesisDirs{}, err
851855
}
852856
} else {
853-
resolved.StateproofGenesisDir = resolved.ColdGenesisDir
857+
resolved.StateproofGenesisDir = resolved.HotGenesisDir
858+
// if separate HotDataDir and ColdDataDir was configured, but StateproofDir was not configured
859+
if resolved.ColdGenesisDir != resolved.HotGenesisDir {
860+
// move existing stateproof DB files from ColdDataDir to HotDataDir
861+
moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, StateProofFileName, StateProofFileName+"-shm", StateProofFileName+"-wal")
862+
if moveErr != nil {
863+
return ResolvedGenesisDirs{}, fmt.Errorf("error moving stateproof DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
864+
}
865+
}
854866
}
855-
// if CrashDBDir is not set, use ColdDataDir
867+
// if CrashDBDir is not set, use HotDataDir
856868
if cfg.CrashDBDir != "" {
857869
resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID)
858870
if err != nil {
859871
return ResolvedGenesisDirs{}, err
860872
}
861873
} else {
862-
resolved.CrashGenesisDir = resolved.ColdGenesisDir
874+
resolved.CrashGenesisDir = resolved.HotGenesisDir
875+
// if separate HotDataDir and ColdDataDir was configured, but CrashDBDir was not configured
876+
if resolved.ColdGenesisDir != resolved.HotGenesisDir {
877+
// move existing crash DB files from ColdDataDir to HotDataDir
878+
moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, CrashFilename, CrashFilename+"-shm", CrashFilename+"-wal")
879+
if moveErr != nil {
880+
return ResolvedGenesisDirs{}, fmt.Errorf("error moving crash DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
881+
}
882+
}
863883
}
864884
return resolved, nil
865885
}
866886

887+
func moveDirIfExists(logger logger, srcdir, dstdir string, files ...string) error {
888+
// first, check if any files already exist in dstdir, and quit if so
889+
for _, file := range files {
890+
if _, err := os.Stat(filepath.Join(dstdir, file)); err == nil {
891+
return fmt.Errorf("destination file %s already exists, not overwriting", filepath.Join(dstdir, file))
892+
}
893+
}
894+
// then, check if any files exist in srcdir, and move them to dstdir
895+
for _, file := range files {
896+
if _, err := os.Stat(filepath.Join(srcdir, file)); err == nil {
897+
if err := os.Rename(filepath.Join(srcdir, file), filepath.Join(dstdir, file)); err != nil {
898+
return fmt.Errorf("failed to move file %s from %s to %s: %v", file, srcdir, dstdir, err)
899+
}
900+
logger.Infof("Moved DB file %s from ColdDataDir %s to HotDataDir %s", file, srcdir, dstdir)
901+
}
902+
}
903+
return nil
904+
}
905+
867906
// AdjustConnectionLimits updates RestConnectionsSoftLimit, RestConnectionsHardLimit, IncomingConnectionsLimit
868907
// if requiredFDs greater than maxFDs
869908
func (cfg *Local) AdjustConnectionLimits(requiredFDs, maxFDs uint64) bool {

‎node/follower_node.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func MakeFollower(log logging.Logger, rootDir string, cfg config.Local, phoneboo
8383
node.genesisHash = genesis.Hash()
8484
node.devMode = genesis.DevMode
8585
var err error
86-
node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID())
86+
node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log)
8787
if err != nil {
8888
return nil, err
8989
}

‎node/node.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ func MakeFull(log logging.Logger, rootDir string, cfg config.Local, phonebookAdd
183183
node.devMode = genesis.DevMode
184184
node.config = cfg
185185
var err error
186-
node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID())
186+
node.genesisDirs, err = cfg.EnsureAndResolveGenesisDirs(rootDir, genesis.ID(), log)
187187
if err != nil {
188188
return nil, err
189189
}

‎node/node_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -598,7 +598,7 @@ func TestConfiguredDataDirs(t *testing.T) {
598598
require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "ledger.tracker.sqlite"))
599599

600600
// confirm the stateproof db in the genesis dir of hot data dir
601-
require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "stateproof.sqlite"))
601+
require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "stateproof.sqlite"))
602602

603603
// confirm cold data dir exists and contains a genesis dir
604604
require.DirExists(t, filepath.Join(testDirCold, genesis.ID()))
@@ -609,8 +609,8 @@ func TestConfiguredDataDirs(t *testing.T) {
609609
// confirm the partregistry is in the genesis dir of cold data dir
610610
require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "partregistry.sqlite"))
611611

612-
// confirm the partregistry is in the genesis dir of cold data dir
613-
require.FileExists(t, filepath.Join(testDirCold, genesis.ID(), "crash.sqlite"))
612+
// confirm the agreement crash DB is in the genesis dir of hot data dir
613+
require.FileExists(t, filepath.Join(testDirHot, genesis.ID(), "crash.sqlite"))
614614
}
615615

616616
// TestConfiguredResourcePaths tests to see that when TrackerDbFilePath, BlockDbFilePath, StateproofDir, and CrashFilePath are set, underlying resources are created in the correct locations

0 commit comments

Comments
 (0)