go-algorand 3.20.0-beta Release PR

README.md

@@ -96,7 +96,7 @@ Please refer to our [CONTRIBUTING](CONTRIBUTING.md) document.
 
 ## Project Layout
 
-`go-algorand` is split into various subsystems containing varius packages.
+`go-algorand` is split into various subsystems containing various packages.
 
 ### Core
 

cmd/algocfg/profileCommand.go

@@ -92,6 +92,7 @@ func init() {
 	rootCmd.AddCommand(profileCmd)
 	profileCmd.AddCommand(setProfileCmd)
 	setProfileCmd.Flags().BoolVarP(&forceUpdate, "yes", "y", false, "Force updates to be written")
+	profileCmd.AddCommand(printProfileCmd)
 	profileCmd.AddCommand(listProfileCmd)
 }
 
@@ -133,6 +134,23 @@ var listProfileCmd = &cobra.Command{
 	},
 }
 
+var printProfileCmd = &cobra.Command{
+	Use:   "print",
+	Short: "Print config.json to stdout.",
+	Args:  cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		cfg, err := getConfigForArg(args[0])
+		if err != nil {
+			reportErrorf("%v", err)
+		}
+		err = codecs.WriteNonDefaultValues(os.Stdout, cfg, config.GetDefaultLocal(), nil)
+		if err != nil {
+			reportErrorf("Error writing config file to stdout: %s", err)
+		}
+		fmt.Fprintf(os.Stdout, "\n")
+	},
+}
+
 var setProfileCmd = &cobra.Command{
 	Use:   "set",
 	Short: "Set config.json file from a profile.",
@@ -157,7 +175,7 @@ var setProfileCmd = &cobra.Command{
 					return
 				}
 			}
-			err = codecs.SaveNonDefaultValuesToFile(file, cfg, config.GetDefaultLocal(), nil, true)
+			err = codecs.SaveNonDefaultValuesToFile(file, cfg, config.GetDefaultLocal(), nil)
 			if err != nil {
 				reportErrorf("Error saving updated config file '%s' - %s", file, err)
 			}

cmd/algocfg/resetCommand.go

@@ -63,7 +63,7 @@ var resetCmd = &cobra.Command{
 			}
 
 			file := filepath.Join(dataDir, config.ConfigFilename)
-			err = codecs.SaveNonDefaultValuesToFile(file, cfg, defaults, nil, true)
+			err = codecs.SaveNonDefaultValuesToFile(file, cfg, defaults, nil)
 			if err != nil {
 				reportWarnf("Error saving updated config file '%s' - %s", file, err)
 				anyError = true

cmd/algocfg/setCommand.go

@@ -66,7 +66,7 @@ var setCmd = &cobra.Command{
 			}
 
 			file := filepath.Join(dataDir, config.ConfigFilename)
-			err = codecs.SaveNonDefaultValuesToFile(file, cfg, config.GetDefaultLocal(), nil, true)
+			err = codecs.SaveNonDefaultValuesToFile(file, cfg, config.GetDefaultLocal(), nil)
 			if err != nil {
 				reportWarnf("Error saving updated config file '%s' - %s", file, err)
 				anyError = true

config/config_test.go

@@ -675,6 +675,12 @@ func TestEnsureAbsDir(t *testing.T) {
 	require.Equal(t, testDirectory+"/myGenesisID", t2Abs)
 }
 
+type tLogger struct{ t *testing.T }
+
+func (l tLogger) Infof(fmts string, args ...interface{}) {
+	l.t.Logf(fmts, args...)
+}
+
 // TestEnsureAndResolveGenesisDirs confirms that paths provided in the config are resolved to absolute paths and are created if relevant
 func TestEnsureAndResolveGenesisDirs(t *testing.T) {
 	partitiontest.PartitionTest(t)
@@ -689,7 +695,7 @@ func TestEnsureAndResolveGenesisDirs(t *testing.T) {
 	cfg.StateproofDir = filepath.Join(testDirectory, "/RELATIVEPATHS/../RELATIVE/../custom_stateproof")
 	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
 
-	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 
 	// confirm that the paths are absolute, and contain the genesisID
@@ -711,7 +717,7 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
 
 	cfg := GetDefaultLocal()
 	testDirectory := t.TempDir()
-	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 	// confirm that if only the root is specified, it is used for all directories
 	require.Equal(t, testDirectory+"/myGenesisID", paths.TrackerGenesisDir)
@@ -731,21 +737,125 @@ func TestEnsureAndResolveGenesisDirs_hierarchy(t *testing.T) {
 	cold := filepath.Join(testDirectory, "cold")
 	cfg.HotDataDir = hot
 	cfg.ColdDataDir = cold
-	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.NoError(t, err)
 	// confirm that if hot/cold are specified, hot/cold are used for appropriate directories
 	require.Equal(t, hot+"/myGenesisID", paths.TrackerGenesisDir)
 	require.DirExists(t, paths.TrackerGenesisDir)
 	require.Equal(t, cold+"/myGenesisID", paths.BlockGenesisDir)
 	require.DirExists(t, paths.BlockGenesisDir)
-	require.Equal(t, cold+"/myGenesisID", paths.CrashGenesisDir)
+	require.Equal(t, hot+"/myGenesisID", paths.CrashGenesisDir)
 	require.DirExists(t, paths.CrashGenesisDir)
-	require.Equal(t, cold+"/myGenesisID", paths.StateproofGenesisDir)
+	require.Equal(t, hot+"/myGenesisID", paths.StateproofGenesisDir)
 	require.DirExists(t, paths.StateproofGenesisDir)
 	require.Equal(t, cold+"/myGenesisID", paths.CatchpointGenesisDir)
 	require.DirExists(t, paths.CatchpointGenesisDir)
 }
 
+func TestEnsureAndResolveGenesisDirs_migrate(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	// put a crash.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// put a stateproof.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.NoError(t, err)
+	// Confirm that crash.sqlite was moved to HotDataDir
+	require.DirExists(t, paths.CrashGenesisDir)
+	require.Equal(t, hotDir, paths.CrashGenesisDir)
+	require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite"))
+	require.NoFileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
+	// Confirm that stateproof.sqlite was moved to HotDataDir
+	require.DirExists(t, paths.StateproofGenesisDir)
+	require.Equal(t, hotDir, paths.StateproofGenesisDir)
+	require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
+	require.NoFileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
+}
+
+func TestEnsureAndResolveGenesisDirs_migrateCrashFail(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	err = os.MkdirAll(hotDir, 0755)
+	require.NoError(t, err)
+	// put a crash.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "crash.sqlite-shm"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// also put a crash.sqlite file in the HotDataDir
+	err = os.WriteFile(filepath.Join(hotDir, "crash.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.Error(t, err)
+	require.Empty(t, paths)
+	// Confirm that crash.sqlite was not moved to HotDataDir
+	require.FileExists(t, filepath.Join(coldDir, "crash.sqlite"))
+	require.FileExists(t, filepath.Join(coldDir, "crash.sqlite-shm"))
+	require.FileExists(t, filepath.Join(hotDir, "crash.sqlite"))
+	require.NoFileExists(t, filepath.Join(hotDir, "crash.sqlite-shm"))
+}
+
+func TestEnsureAndResolveGenesisDirs_migrateSPFail(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
+	cfg := GetDefaultLocal()
+	testDirectory := t.TempDir()
+	cfg.HotDataDir = filepath.Join(testDirectory, "hot")
+	cfg.ColdDataDir = filepath.Join(testDirectory, "cold")
+	coldDir := filepath.Join(cfg.ColdDataDir, "myGenesisID")
+	hotDir := filepath.Join(cfg.HotDataDir, "myGenesisID")
+	err := os.MkdirAll(coldDir, 0755)
+	require.NoError(t, err)
+	err = os.MkdirAll(hotDir, 0755)
+	require.NoError(t, err)
+	// put a stateproof.sqlite file in the ColdDataDir
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite"), []byte("test"), 0644)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(coldDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// also put a stateproof.sqlite-wal file in the HotDataDir
+	err = os.WriteFile(filepath.Join(hotDir, "stateproof.sqlite-wal"), []byte("test"), 0644)
+	require.NoError(t, err)
+	// Resolve
+	paths, err := cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
+	require.Error(t, err)
+	require.Empty(t, paths)
+	// Confirm that stateproof.sqlite was not moved to HotDataDir
+	require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(coldDir, "stateproof.sqlite-wal"))
+	require.NoFileExists(t, filepath.Join(hotDir, "stateproof.sqlite"))
+	require.FileExists(t, filepath.Join(hotDir, "stateproof.sqlite-wal"))
+}
+
 // TestEnsureAndResolveGenesisDirsError confirms that if a path can't be created, an error is returned
 func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
 	partitiontest.PartitionTest(t)
@@ -761,15 +871,15 @@ func TestEnsureAndResolveGenesisDirsError(t *testing.T) {
 	cfg.CatchpointDir = filepath.Join(testDirectory, "custom_catchpoint")
 
 	// first try an error with an empty root dir
-	paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID")
+	paths, err := cfg.EnsureAndResolveGenesisDirs("", "myGenesisID", tLogger{t: t})
 	require.Empty(t, paths)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "rootDir is required")
 
 	require.NoError(t, os.Chmod(testDirectory, 0200))
 
 	// now try an error with a root dir that can't be written to
-	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID")
+	paths, err = cfg.EnsureAndResolveGenesisDirs(testDirectory, "myGenesisID", tLogger{t: t})
 	require.Empty(t, paths)
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "permission denied")

config/localTemplate.go

@@ -111,13 +111,13 @@ type Local struct {
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
 	// If not specified, the node will use the ColdDataDir.
 	CatchpointDir string `version[31]:""`
-	// StateproofDir is an optional directory to store stateproof data.
+	// StateproofDir is an optional directory to persist state about observed and issued state proof messages.
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
-	// If not specified, the node will use the ColdDataDir.
+	// If not specified, the node will use the HotDataDir.
 	StateproofDir string `version[31]:""`
-	// CrashDBDir is an optional directory to store the crash database.
+	// CrashDBDir is an optional directory to persist agreement's consensus participation state.
 	// For isolation, the node will create a subdirectory in this location, named by the genesis-id of the network.
-	// If not specified, the node will use the ColdDataDir.
+	// If not specified, the node will use the HotDataDir
 	CrashDBDir string `version[31]:""`
 
 	// LogFileDir is an optional directory to store the log, node.log
@@ -666,7 +666,7 @@ func (cfg Local) SaveAllToDisk(root string) error {
 func (cfg Local) SaveToFile(filename string) error {
 	var alwaysInclude []string
 	alwaysInclude = append(alwaysInclude, "Version")
-	return codecs.SaveNonDefaultValuesToFile(filename, cfg, defaultLocal, alwaysInclude, true)
+	return codecs.SaveNonDefaultValuesToFile(filename, cfg, defaultLocal, alwaysInclude)
 }
 
 // DNSSecuritySRVEnforced returns true if SRV response verification enforced
@@ -785,9 +785,13 @@ func (cfg *Local) ResolveLogPaths(rootDir string) (liveLog, archive string) {
 	return liveLog, archive
 }
 
+type logger interface {
+	Infof(format string, args ...interface{})
+}
+
 // EnsureAndResolveGenesisDirs will resolve the supplied config paths to absolute paths, and will create the genesis directories of each
 // returns a ResolvedGenesisDirs struct with the resolved paths for use during runtime
-func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (ResolvedGenesisDirs, error) {
+func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string, logger logger) (ResolvedGenesisDirs, error) {
 	var resolved ResolvedGenesisDirs
 	var err error
 	if rootDir != "" {
@@ -843,27 +847,62 @@ func (cfg *Local) EnsureAndResolveGenesisDirs(rootDir, genesisID string) (Resolv
 	} else {
 		resolved.CatchpointGenesisDir = resolved.ColdGenesisDir
 	}
-	// if StateproofDir is not set, use ColdDataDir
+	// if StateproofDir is not set, use HotDataDir
 	if cfg.StateproofDir != "" {
 		resolved.StateproofGenesisDir, err = ensureAbsGenesisDir(cfg.StateproofDir, genesisID)
 		if err != nil {
 			return ResolvedGenesisDirs{}, err
 		}
 	} else {
-		resolved.StateproofGenesisDir = resolved.ColdGenesisDir
+		resolved.StateproofGenesisDir = resolved.HotGenesisDir
+		// if separate HotDataDir and ColdDataDir was configured, but StateproofDir was not configured
+		if resolved.ColdGenesisDir != resolved.HotGenesisDir {
+			// move existing stateproof DB files from ColdDataDir to HotDataDir
+			moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, StateProofFileName, StateProofFileName+"-shm", StateProofFileName+"-wal")
+			if moveErr != nil {
+				return ResolvedGenesisDirs{}, fmt.Errorf("error moving stateproof DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
+			}
+		}
 	}
-	// if CrashDBDir is not set, use ColdDataDir
+	// if CrashDBDir is not set, use HotDataDir
 	if cfg.CrashDBDir != "" {
 		resolved.CrashGenesisDir, err = ensureAbsGenesisDir(cfg.CrashDBDir, genesisID)
 		if err != nil {
 			return ResolvedGenesisDirs{}, err
 		}
 	} else {
-		resolved.CrashGenesisDir = resolved.ColdGenesisDir
+		resolved.CrashGenesisDir = resolved.HotGenesisDir
+		// if separate HotDataDir and ColdDataDir was configured, but CrashDBDir was not configured
+		if resolved.ColdGenesisDir != resolved.HotGenesisDir {
+			// move existing crash DB files from ColdDataDir to HotDataDir
+			moveErr := moveDirIfExists(logger, resolved.ColdGenesisDir, resolved.HotGenesisDir, CrashFilename, CrashFilename+"-shm", CrashFilename+"-wal")
+			if moveErr != nil {
+				return ResolvedGenesisDirs{}, fmt.Errorf("error moving crash DB files from ColdDataDir %s to HotDataDir %s: %v", resolved.ColdGenesisDir, resolved.HotGenesisDir, moveErr)
+			}
+		}
 	}
 	return resolved, nil
 }
 
+func moveDirIfExists(logger logger, srcdir, dstdir string, files ...string) error {
+	// first, check if any files already exist in dstdir, and quit if so
+	for _, file := range files {
+		if _, err := os.Stat(filepath.Join(dstdir, file)); err == nil {
+			return fmt.Errorf("destination file %s already exists, not overwriting", filepath.Join(dstdir, file))
+		}
+	}
+	// then, check if any files exist in srcdir, and move them to dstdir
+	for _, file := range files {
+		if _, err := os.Stat(filepath.Join(srcdir, file)); err == nil {
+			if err := os.Rename(filepath.Join(srcdir, file), filepath.Join(dstdir, file)); err != nil {
+				return fmt.Errorf("failed to move file %s from %s to %s: %v", file, srcdir, dstdir, err)
+			}
+			logger.Infof("Moved DB file %s from ColdDataDir %s to HotDataDir %s", file, srcdir, dstdir)
+		}
+	}
+	return nil
+}
+
 // AdjustConnectionLimits updates RestConnectionsSoftLimit, RestConnectionsHardLimit, IncomingConnectionsLimit
 // if requiredFDs greater than maxFDs
 func (cfg *Local) AdjustConnectionLimits(requiredFDs, maxFDs uint64) bool {

data/transactions/logic/README.md

@@ -341,7 +341,7 @@ An application transaction must indicate the action to be taken following the ex
 
 Most operations work with only one type of argument, uint64 or bytes, and fail if the wrong type value is on the stack.
 
-Many instructions accept values to designate Accounts, Assets, or Applications. Beginning with v4, these values may be given as an _offset_ in the corresponding Txn fields (Txn.Accounts, Txn.ForeignAssets, Txn.ForeignApps) _or_ as the value itself (a byte-array address for Accounts, or a uint64 ID). The values, however, must still be present in the Txn fields. Before v4, most opcodes required the use of an offset, except for reading account local values of assets or applications, which accepted the IDs directly and did not require the ID to be present in they corresponding _Foreign_ array. (Note that beginning with v4, those IDs _are_ required to be present in their corresponding _Foreign_ array.) See individual opcodes for details. In the case of account offsets or application offsets, 0 is specially defined to Txn.Sender or the ID of the current application, respectively.
+Many instructions accept values to designate Accounts, Assets, or Applications. Beginning with v4, these values may be given as an _offset_ in the corresponding Txn fields (Txn.Accounts, Txn.ForeignAssets, Txn.ForeignApps) _or_ as the value itself (a byte-array address for Accounts, or a uint64 ID). The values, however, must still be present in the Txn fields. Before v4, most opcodes required the use of an offset, except for reading account local values of assets or applications, which accepted the IDs directly and did not require the ID to be present in the corresponding _Foreign_ array. (Note that beginning with v4, those IDs _are_ required to be present in their corresponding _Foreign_ array.) See individual opcodes for details. In the case of account offsets or application offsets, 0 is specially defined to Txn.Sender or the ID of the current application, respectively.
 
 This summary is supplemented by more detail in the [opcodes document](TEAL_opcodes.md).
 
@@ -775,7 +775,7 @@ are sure to be _available_.
 
 The following opcodes allow for "inner transactions". Inner
 transactions allow stateful applications to have many of the effects
-of a true top-level transaction, programatically.  However, they are
+of a true top-level transaction, programmatically.  However, they are
 different in significant ways.  The most important differences are
 that they are not signed, duplicates are not rejected, and they do not
 appear in the block in the usual away. Instead, their effects are
@@ -786,7 +786,7 @@ account that has been rekeyed to that hash.
 
 In v5, inner transactions may perform `pay`, `axfer`, `acfg`, and
 `afrz` effects.  After executing an inner transaction with
-`itxn_submit`, the effects of the transaction are visible begining
+`itxn_submit`, the effects of the transaction are visible beginning
 with the next instruction with, for example, `balance` and
 `min_balance` checks. In v6, inner transactions may also perform
 `keyreg` and `appl` effects. Inner `appl` calls fail if they attempt
@@ -806,7 +806,7 @@ setting is used when `itxn_submit` executes. For this purpose `Type`
 and `TypeEnum` are considered to be the same field. When using
 `itxn_field` to set an array field (`ApplicationArgs` `Accounts`,
 `Assets`, or `Applications`) each use adds an element to the end of
-the the array, rather than setting the entire array at once.
+the array, rather than setting the entire array at once.
 
 `itxn_field` fails immediately for unsupported fields, unsupported
 transaction types, or improperly typed values for a particular