Add experiment system + events experiment (#18682)

changelog/18682.txt

@@ -0,0 +1,4 @@
+```release-note:improvement
+core: Add experiments system and `events.beta1` experiment.
+```
+

command/commands.go

@@ -87,6 +87,12 @@ const (
 	// EnvVaultLogLevel is used to specify the log level applied to logging
 	// Supported log levels: Trace, Debug, Error, Warn, Info
 	EnvVaultLogLevel = "VAULT_LOG_LEVEL"
+	// EnvVaultExperiments defines the experiments to enable for a server as a
+	// comma separated list. See experiments.ValidExperiments() for the list of
+	// valid experiments. Not mutable or persisted in storage, only read and
+	// logged at startup _per node_. This was initially introduced for the events
+	// system being developed over multiple release cycles.
+	EnvVaultExperiments = "VAULT_EXPERIMENTS"
 
 	// DisableSSCTokens is an env var used to disable index bearing
 	// token functionality

command/server.go

@@ -36,6 +36,7 @@ import (
 	"github.com/hashicorp/vault/command/server"
 	"github.com/hashicorp/vault/helper/builtinplugins"
 	"github.com/hashicorp/vault/helper/constants"
+	"github.com/hashicorp/vault/helper/experiments"
 	loghelper "github.com/hashicorp/vault/helper/logging"
 	"github.com/hashicorp/vault/helper/metricsutil"
 	"github.com/hashicorp/vault/helper/namespace"
@@ -116,6 +117,7 @@ type ServerCommand struct {
 
 	flagConfigs            []string
 	flagRecovery           bool
+	flagExperiments        []string
 	flagDev                bool
 	flagDevTLS             bool
 	flagDevTLSCertDir      string
@@ -204,6 +206,17 @@ func (c *ServerCommand) Flags() *FlagSets {
 			"Using a recovery operation token, \"sys/raw\" API can be used to manipulate the storage.",
 	})
 
+	f.StringSliceVar(&StringSliceVar{
+		Name:       "experiment",
+		Target:     &c.flagExperiments,
+		Completion: complete.PredictSet(experiments.ValidExperiments()...),
+		Usage: "Name of an experiment to enable. Experiments should NOT be used in production, and " +
+			"the associated APIs may have backwards incompatible changes between releases. This " +
+			"flag can be specified multiple times to specify multiple experiments. This can also be " +
+			fmt.Sprintf("specified via the %s environment variable as a comma-separated list. ", EnvVaultExperiments) +
+			"Valid experiments are: " + strings.Join(experiments.ValidExperiments(), ", "),
+	})
+
 	f = set.NewFlagSet("Dev Options")
 
 	f.BoolVar(&BoolVar{
@@ -1105,6 +1118,11 @@ func (c *ServerCommand) Run(args []string) int {
 		}
 	}
 
+	if err := server.ExperimentsFromEnvAndCLI(config, EnvVaultExperiments, c.flagExperiments); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
 	// If mlockall(2) isn't supported, show a warning. We disable this in dev
 	// because it is quite scary to see when first using Vault. We also disable
 	// this if the user has explicitly disabled mlock in configuration.
@@ -1173,6 +1191,12 @@ func (c *ServerCommand) Run(args []string) int {
 	info[key] = strings.Join(envVarKeys, ", ")
 	infoKeys = append(infoKeys, key)
 
+	if len(config.Experiments) != 0 {
+		expKey := "experiments"
+		info[expKey] = strings.Join(config.Experiments, ", ")
+		infoKeys = append(infoKeys, expKey)
+	}
+
 	barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(c, config, infoKeys, info)
 	// Check error here
 	if err != nil {
@@ -2637,6 +2661,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.
 		License:                        config.License,
 		LicensePath:                    config.LicensePath,
 		DisableSSCTokens:               config.DisableSSCTokens,
+		Experiments:                    config.Experiments,
 	}
 
 	if c.flagDev {

command/server/config.go

@@ -17,9 +17,11 @@ import (
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/ast"
+	"github.com/hashicorp/vault/helper/experiments"
 	"github.com/hashicorp/vault/helper/osutil"
 	"github.com/hashicorp/vault/internalshared/configutil"
 	"github.com/hashicorp/vault/sdk/helper/consts"
+	"github.com/hashicorp/vault/sdk/helper/strutil"
 )
 
 const (
@@ -28,9 +30,14 @@ const (
 	VaultDevKeyFilename  = "vault-key.pem"
 )
 
-var entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError {
-	return nil
-}
+var (
+	entConfigValidate = func(_ *Config, _ string) []configutil.ConfigError {
+		return nil
+	}
+
+	// Modified internally for testing.
+	validExperiments = experiments.ValidExperiments()
+)
 
 // Config is the configuration for the vault server.
 type Config struct {
@@ -45,6 +52,8 @@ type Config struct {
 
 	ServiceRegistration *ServiceRegistration `hcl:"-"`
 
+	Experiments []string `hcl:"experiments"`
+
 	CacheSize                int         `hcl:"cache_size"`
 	DisableCache             bool        `hcl:"-"`
 	DisableCacheRaw          interface{} `hcl:"disable_cache"`
@@ -433,6 +442,8 @@ func (c *Config) Merge(c2 *Config) *Config {
 
 	result.entConfig = c.entConfig.Merge(c2.entConfig)
 
+	result.Experiments = mergeExperiments(c.Experiments, c2.Experiments)
+
 	return result
 }
 
@@ -699,6 +710,10 @@ func ParseConfig(d, source string) (*Config, error) {
 		}
 	}
 
+	if err := validateExperiments(result.Experiments); err != nil {
+		return nil, fmt.Errorf("error validating experiment(s) from config: %w", err)
+	}
+
 	if err := result.parseConfig(list); err != nil {
 		return nil, fmt.Errorf("error parsing enterprise config: %w", err)
 	}
@@ -715,6 +730,69 @@ func ParseConfig(d, source string) (*Config, error) {
 	return result, nil
 }
 
+func ExperimentsFromEnvAndCLI(config *Config, envKey string, flagExperiments []string) error {
+	if envExperimentsRaw := os.Getenv(envKey); envExperimentsRaw != "" {
+		envExperiments := strings.Split(envExperimentsRaw, ",")
+		err := validateExperiments(envExperiments)
+		if err != nil {
+			return fmt.Errorf("error validating experiment(s) from environment variable %q: %w", envKey, err)
+		}
+
+		config.Experiments = mergeExperiments(config.Experiments, envExperiments)
+	}
+
+	if len(flagExperiments) != 0 {
+		err := validateExperiments(flagExperiments)
+		if err != nil {
+			return fmt.Errorf("error validating experiment(s) from command line flag: %w", err)
+		}
+
+		config.Experiments = mergeExperiments(config.Experiments, flagExperiments)
+	}
+
+	return nil
+}
+
+// Validate checks each experiment is a known experiment.
+func validateExperiments(experiments []string) error {
+	var invalid []string
+
+	for _, experiment := range experiments {
+		if !strutil.StrListContains(validExperiments, experiment) {
+			invalid = append(invalid, experiment)
+		}
+	}
+
+	if len(invalid) != 0 {
+		return fmt.Errorf("valid experiment(s) are %s, but received the following invalid experiment(s): %s",
+			strings.Join(validExperiments, ", "),
+			strings.Join(invalid, ", "))
+	}
+
+	return nil
+}
+
+// mergeExperiments returns the logical OR of the two sets.
+func mergeExperiments(left, right []string) []string {
+	processed := map[string]struct{}{}
+	var result []string
+	for _, l := range left {
+		if _, seen := processed[l]; !seen {
+			result = append(result, l)
+		}
+		processed[l] = struct{}{}
+	}
+
+	for _, r := range right {
+		if _, seen := processed[r]; !seen {
+			result = append(result, r)
+			processed[r] = struct{}{}
+		}
+	}
+
+	return result
+}
+
 // LoadConfigDir loads all the configurations in the given directory
 // in alphabetical order.
 func LoadConfigDir(dir string) (*Config, error) {
@@ -1032,6 +1110,7 @@ func (c *Config) Sanitized() map[string]interface{} {
 		"enable_response_header_raft_node_id": c.EnableResponseHeaderRaftNodeID,
 
 		"log_requests_level": c.LogRequestsLevel,
+		"experiments":        c.Experiments,
 
 		"detect_deadlocks": c.DetectDeadlocks,
 	}

command/server/config_test.go

@@ -1,6 +1,9 @@
 package server
 
 import (
+	"fmt"
+	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -71,3 +74,112 @@ func TestUnknownFieldValidationHcl(t *testing.T) {
 func TestUnknownFieldValidationListenerAndStorage(t *testing.T) {
 	testUnknownFieldValidationStorageAndListener(t)
 }
+
+func TestExperimentsConfigParsing(t *testing.T) {
+	const envKey = "VAULT_EXPERIMENTS"
+	originalValue := validExperiments
+	validExperiments = []string{"foo", "bar", "baz"}
+	t.Cleanup(func() {
+		validExperiments = originalValue
+	})
+
+	for name, tc := range map[string]struct {
+		fromConfig    []string
+		fromEnv       []string
+		fromCLI       []string
+		expected      []string
+		expectedError string
+	}{
+		// Multiple sources.
+		"duplication":  {[]string{"foo"}, []string{"foo"}, []string{"foo"}, []string{"foo"}, ""},
+		"disjoint set": {[]string{"foo"}, []string{"bar"}, []string{"baz"}, []string{"foo", "bar", "baz"}, ""},
+
+		// Single source.
+		"config only": {[]string{"foo"}, nil, nil, []string{"foo"}, ""},
+		"env only":    {nil, []string{"foo"}, nil, []string{"foo"}, ""},
+		"CLI only":    {nil, nil, []string{"foo"}, []string{"foo"}, ""},
+
+		// Validation errors.
+		"config invalid": {[]string{"invalid"}, nil, nil, nil, "from config"},
+		"env invalid":    {nil, []string{"invalid"}, nil, nil, "from environment variable"},
+		"CLI invalid":    {nil, nil, []string{"invalid"}, nil, "from command line flag"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var configString string
+			t.Setenv(envKey, strings.Join(tc.fromEnv, ","))
+			if len(tc.fromConfig) != 0 {
+				configString = fmt.Sprintf("experiments = [\"%s\"]", strings.Join(tc.fromConfig, "\", \""))
+			}
+			config, err := ParseConfig(configString, "")
+			if err == nil {
+				err = ExperimentsFromEnvAndCLI(config, envKey, tc.fromCLI)
+			}
+
+			switch tc.expectedError {
+			case "":
+				if err != nil {
+					t.Fatal(err)
+				}
+
+			default:
+				if err == nil || !strings.Contains(err.Error(), tc.expectedError) {
+					t.Fatalf("Expected error to contain %q, but got: %s", tc.expectedError, err)
+				}
+			}
+		})
+	}
+}
+
+func TestValidate(t *testing.T) {
+	originalValue := validExperiments
+	for name, tc := range map[string]struct {
+		validSet    []string
+		input       []string
+		expectError bool
+	}{
+		// Valid cases
+		"minimal valid": {[]string{"foo"}, []string{"foo"}, false},
+		"valid subset":  {[]string{"foo", "bar"}, []string{"bar"}, false},
+		"repeated":      {[]string{"foo"}, []string{"foo", "foo"}, false},
+
+		// Error cases
+		"partially valid":      {[]string{"foo", "bar"}, []string{"foo", "baz"}, true},
+		"empty":                {[]string{"foo"}, []string{""}, true},
+		"no valid experiments": {[]string{}, []string{"foo"}, true},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Cleanup(func() {
+				validExperiments = originalValue
+			})
+
+			validExperiments = tc.validSet
+			err := validateExperiments(tc.input)
+			if tc.expectError && err == nil {
+				t.Fatal("Expected error but got none")
+			}
+			if !tc.expectError && err != nil {
+				t.Fatal("Did not expect error but got", err)
+			}
+		})
+	}
+}
+
+func TestMerge(t *testing.T) {
+	for name, tc := range map[string]struct {
+		left     []string
+		right    []string
+		expected []string
+	}{
+		"disjoint":    {[]string{"foo"}, []string{"bar"}, []string{"foo", "bar"}},
+		"empty left":  {[]string{}, []string{"foo"}, []string{"foo"}},
+		"empty right": {[]string{"foo"}, []string{}, []string{"foo"}},
+		"overlapping": {[]string{"foo", "bar"}, []string{"foo", "baz"}, []string{"foo", "bar", "baz"}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			result := mergeExperiments(tc.left, tc.right)
+			if !reflect.DeepEqual(tc.expected, result) {
+				t.Fatalf("Expected %v but got %v", tc.expected, result)
+			}
+		})
+	}
+}

command/server/config_test_helpers.go

@@ -738,6 +738,7 @@ func testConfig_Sanitized(t *testing.T) {
 		"disable_indexing":                    false,
 		"disable_mlock":                       true,
 		"disable_performance_standby":         false,
+		"experiments":                         []string(nil),
 		"plugin_file_uid":                     0,
 		"plugin_file_permissions":             0,
 		"disable_printable_check":             false,

helper/experiments/experiments.go

@@ -0,0 +1,16 @@
+package experiments
+
+const VaultExperimentEventsBeta1 = "events.beta1"
+
+var validExperiments = []string{
+	VaultExperimentEventsBeta1,
+}
+
+// ValidExperiments exposes the list without exposing a mutable global variable.
+// Experiments can only be enabled when starting a server, and will typically
+// enable pre-GA API functionality.
+func ValidExperiments() []string {
+	result := make([]string, len(validExperiments))
+	copy(result, validExperiments)
+	return result
+}

http/sys_config_state_test.go

@@ -38,6 +38,7 @@ func TestSysConfigState_Sanitized(t *testing.T) {
 		"disable_performance_standby":         false,
 		"disable_printable_check":             false,
 		"disable_sealwrap":                    false,
+		"experiments":                         nil,
 		"raw_storage_endpoint":                false,
 		"detect_deadlocks":                    "",
 		"introspection_endpoint":              false,