Converts between experimental and non-experimental flags in downgrade & upgrade robustness & e2e test

tests/framework/e2e/cluster.go

@@ -617,6 +617,16 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		}
 	}
 
+	var (
+		binaryVersion *semver.Version
+		err           error
+	)
+	if execPath != "" {
+		binaryVersion, err = GetVersionFromBinary(execPath)
+		if err != nil {
+			tb.Logf("Failed to get binary version from %s: %v", execPath, err)
+		}
+	}
 	defaultValues := values(*embed.NewConfig())
 	overrideValues := values(cfg.ServerConfig)
 	for flag, value := range overrideValues {
@@ -626,7 +636,7 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		if strings.HasSuffix(flag, "snapshot-catchup-entries") && !CouldSetSnapshotCatchupEntries(execPath) {
 			continue
 		}
-		args = append(args, fmt.Sprintf("--%s=%s", flag, value))
+		args = append(args, convertFlag(flag, value, binaryVersion))
 	}
 	envVars := map[string]string{}
 	maps.Copy(envVars, cfg.EnvVars)

tests/framework/e2e/config.go

@@ -14,6 +14,15 @@
 
 package e2e
 
+import (
+	"fmt"
+	"strings"
+
+	"github.com/coreos/go-semver/semver"
+
+	"go.etcd.io/etcd/api/v3/version"
+)
+
 type ClusterVersion string
 
 const (
@@ -33,3 +42,60 @@ func (cv ClusterVersion) String() string {
 type ClusterContext struct {
 	Version ClusterVersion
 }
+
+var experimentalFlags = map[string]struct{}{
+	"compact-hash-check-time":              {},
+	"corrupt-check-time":                   {},
+	"compaction-batch-limit":               {},
+	"watch-progress-notify-interval":       {},
+	"warning-apply-duration":               {},
+	"bootstrap-defrag-threshold-megabytes": {},
+	"memory-mlock":                         {},
+	"snapshot-catchup-entries":             {},
+	"compaction-sleep-interval":            {},
+	"downgrade-check-time":                 {},
+	"peer-skip-client-san-verification":    {},
+	"enable-distributed-tracing":           {},
+	"distributed-tracing-address":          {},
+	"distributed-tracing-service-name":     {},
+	"distributed-tracing-instance-id":      {},
+	"distributed-tracing-sampling-rate":    {},
+}
+
+// convertFlag converts between experimental and non-experimental flags.
+// All experimental flags have been removed in version 3.7, but versions prior
+// to 3.6 only support experimental flags. The robustness tests use the same
+// code from the main branch to test all previously supported releases, so we
+// need to convert flags accordingly based on the binary version.
+func convertFlag(name, value string, ver *semver.Version) string {
+	// For versions >= 3.6, use the normal (non-experimental) flag.
+	if ver == nil || !ver.LessThan(version.V3_6) {
+		return fmt.Sprintf("--%s=%s", name, value)
+	}
+
+	// For versions < 3.6, use the experimental flag if it exists in `experimentalFlags`
+	if _, ok := experimentalFlags[name]; ok {
+		return fmt.Sprintf("--experimental-%s=%s", name, value)
+	}
+
+	return fmt.Sprintf("--%s=%s", name, value)
+}
+
+func convertFlags(args []string, ver *semver.Version) []string {
+	var retArgs []string
+
+	for _, arg := range args {
+		kv := strings.Split(arg, "=")
+		if len(kv) != 2 {
+			retArgs = append(retArgs, arg)
+			continue
+		}
+
+		name := strings.TrimPrefix(kv[0], "--")
+		name = strings.TrimPrefix(name, "experimental")
+
+		retArgs = append(retArgs, convertFlag(name, kv[1], ver))
+	}
+
+	return retArgs
+}

tests/framework/e2e/downgrade.go

@@ -147,6 +147,11 @@ func DowngradeUpgradeMembersByID(t *testing.T, lg *zap.Logger, clus *EtcdProcess
 		newExecPath = BinPath.EtcdLastRelease
 	}
 
+	binaryVersion, err := GetVersionFromBinary(newExecPath)
+	if err != nil {
+		return fmt.Errorf("failed to get binary version from %s: %w", newExecPath, err)
+	}
+
 	g := new(errgroup.Group)
 	for _, memberID := range membersToChange {
 		member := clus.Procs[memberID]
@@ -157,6 +162,11 @@ func DowngradeUpgradeMembersByID(t *testing.T, lg *zap.Logger, clus *EtcdProcess
 		if err := member.Stop(); err != nil {
 			return err
 		}
+
+		// When we downgrade or upgrade a member, we need to re-generate the flags, to convert some non-experimental
+		// flags to experimental flags, or vice verse.
+		member.Config().Args = convertFlags(member.Config().Args, binaryVersion)
+
 		member.Config().ExecPath = newExecPath
 		lg.Info("Restarting member", zap.String("member", member.Config().Name))
 		// We shouldn't block on waiting for the member to be ready,