Config file environment expansion

CHANGELOG.md

@@ -18,6 +18,8 @@ this platform. FreeBSD builds will return in a future release.
 
 - [FEATURE] New integration:
   [consul_exporter](https://github.com/prometheus/consul_exporter) (@rfratto)
+
+- [FEATURE] Add optional environment variable substitution of configuration file. (@dcseifert)  
 
 - [ENHANCEMENT] `min_wal_time` and `max_wal_time` have been added to the
   instance config settings, guaranteeing that data in the WAL will exist for at

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	contrib.go.opencensus.io/exporter/prometheus v0.2.0
 	github.com/cortexproject/cortex v1.4.1-0.20201022071705-85942c5703cf
+	github.com/drone/envsubst v1.0.2
 	github.com/go-kit/kit v0.10.0
 	github.com/go-playground/validator/v10 v10.4.0 // indirect
 	github.com/gogo/protobuf v1.3.1

go.sum

@@ -376,6 +376,8 @@ github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo=
+github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=

pkg/config/config.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/drone/envsubst"
 	"github.com/grafana/agent/pkg/integrations"
 	"github.com/grafana/agent/pkg/loki"
 	"github.com/grafana/agent/pkg/prom"
@@ -65,19 +66,27 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) {
 }
 
 // LoadFile reads a file and passes the contents to Load
-func LoadFile(filename string, c *Config) error {
+func LoadFile(filename string, expandEnvVars bool, c *Config) error {
 	buf, err := ioutil.ReadFile(filename)
 	if err != nil {
 		return errors.Wrap(err, "error reading config file")
 	}
-
-	return LoadBytes(buf, c)
+	return LoadBytes(buf, expandEnvVars, c)
 }
 
 // LoadBytes unmarshals a config from a buffer. Defaults are not
 // applied to the file and must be done manually if LoadBytes
 // is called directly.
-func LoadBytes(buf []byte, c *Config) error {
+func LoadBytes(buf []byte, expandEnvVars bool, c *Config) error {
+	// (Optionally) expand with environment variables
+	if expandEnvVars {
+		s, err := envsubst.EvalEnv(string(buf))
+		if err != nil {
+			return fmt.Errorf("unable to substitute config with environment variables: %w", err)
+		}
+		buf = []byte(s)
+	}
+	// Unmarshal yaml config
 	return yaml.UnmarshalStrict(buf, c)
 }
 
@@ -90,15 +99,17 @@ func Load(fs *flag.FlagSet, args []string) (*Config, error) {
 
 // load allows for tests to inject a function for retreiving the config file that
 // doesn't require having a literal file on disk.
-func load(fs *flag.FlagSet, args []string, loader func(string, *Config) error) (*Config, error) {
+func load(fs *flag.FlagSet, args []string, loader func(string, bool, *Config) error) (*Config, error) {
 	var (
-		printVersion bool
-		cfg          Config
-		file         string
+		printVersion    bool
+		cfg             Config
+		file            string
+		configExpandEnv bool
 	)
 
 	fs.StringVar(&file, "config.file", "", "configuration file to load")
 	fs.BoolVar(&printVersion, "version", false, "Print this build's version information")
+	fs.BoolVar(&configExpandEnv, "config.expand-env", false, "Expands ${var} in config according to the values of the environment variables.")
 	cfg.RegisterFlags(fs)
 
 	if err := fs.Parse(args); err != nil {
@@ -112,7 +123,7 @@ func load(fs *flag.FlagSet, args []string, loader func(string, *Config) error) (
 
 	if file == "" {
 		return nil, fmt.Errorf("-config.file flag required")
-	} else if err := loader(file, &cfg); err != nil {
+	} else if err := loader(file, configExpandEnv, &cfg); err != nil {
 		return nil, fmt.Errorf("error loading config file %s: %w", file, err)
 	}
 

pkg/config/config_test.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"flag"
+	"os"
 	"testing"
 	"time"
 
@@ -20,8 +21,8 @@ prometheus:
     scrape_timeout: 33s`
 
 	fs := flag.NewFlagSet("test", flag.ExitOnError)
-	c, err := load(fs, []string{"-config.file", "test"}, func(_ string, c *Config) error {
-		return LoadBytes([]byte(cfg), c)
+	c, err := load(fs, []string{"-config.file", "test"}, func(_ string, _ bool, c *Config) error {
+		return LoadBytes([]byte(cfg), false, c)
 	})
 	require.NoError(t, err)
 	require.NotEmpty(t, c.Prometheus.ServiceConfig.Lifecycler.InfNames)
@@ -42,8 +43,29 @@ prometheus:
 	}
 
 	fs := flag.NewFlagSet("test", flag.ExitOnError)
-	c, err := load(fs, []string{"-config.file", "test"}, func(_ string, c *Config) error {
-		return LoadBytes([]byte(cfg), c)
+	c, err := load(fs, []string{"-config.file", "test"}, func(_ string, _ bool, c *Config) error {
+		return LoadBytes([]byte(cfg), false, c)
+	})
+	require.NoError(t, err)
+	require.Equal(t, expect, c.Prometheus.Global)
+}
+
+func TestConfig_OverrideByEnvironmentOnLoad(t *testing.T) {
+	cfg := `
+prometheus:
+  wal_directory: /tmp/wal
+  global:
+    scrape_timeout: ${SCRAPE_TIMEOUT}`
+	expect := promCfg.GlobalConfig{
+		ScrapeInterval:     model.Duration(1 * time.Minute),
+		ScrapeTimeout:      model.Duration(33 * time.Second),
+		EvaluationInterval: model.Duration(1 * time.Minute),
+	}
+	_ = os.Setenv("SCRAPE_TIMEOUT", "33s")
+
+	fs := flag.NewFlagSet("test", flag.ExitOnError)
+	c, err := load(fs, []string{"-config.file", "test"}, func(_ string, _ bool, c *Config) error {
+		return LoadBytes([]byte(cfg), true, c)
 	})
 	require.NoError(t, err)
 	require.Equal(t, expect, c.Prometheus.Global)
@@ -59,10 +81,11 @@ prometheus:
 	args := []string{
 		"-config.file", "test",
 		"-prometheus.wal-directory", "/tmp/wal",
+		"-config.expand-env",
 	}
 
-	c, err := load(fs, args, func(_ string, c *Config) error {
-		return LoadBytes([]byte(cfg), c)
+	c, err := load(fs, args, func(_ string, _ bool, c *Config) error {
+		return LoadBytes([]byte(cfg), false, c)
 	})
 	require.NoError(t, err)
 	require.Equal(t, "/tmp/wal", c.Prometheus.WALDir)
@@ -77,7 +100,7 @@ prometheus:
     scrape_timeout: 10s
     scrape_timeout: 15s`
 		var c Config
-		err := LoadBytes([]byte(cfg), &c)
+		err := LoadBytes([]byte(cfg), false, &c)
 		require.Error(t, err)
 	})
 
@@ -88,7 +111,7 @@ prometheus:
   global:
   scrape_timeout: 10s`
 		var c Config
-		err := LoadBytes([]byte(cfg), &c)
+		err := LoadBytes([]byte(cfg), false, &c)
 		require.Error(t, err)
 	})
 }