Expose hash of the config files used as metric (#2874)

This allows to monitor the roll-out of a new config file across the
cluster and can helps to detect a mismatch in active config files. It
supports both start-up and runtime config.

Signed-off-by: Christian Simon <[email protected]>

Author: simonswine

CHANGELOG.md

@@ -17,6 +17,7 @@
 * [CHANGE] Added the `engine` label to the metrics exposed by the Prometheus query engine, to distinguish between `ruler` and `querier` metrics. #2854
 * [CHANGE] Added ruler to the single binary when started with `-target=all` (default). #2854
 * [CHANGE] Experimental TSDB: compact head when opening TSDB. This should only affect ingester startup after it was unable to compact head in previous run. #2870
+* [CHANGE] Metric `cortex_overrides_last_reload_successful` has been renamed to `cortex_runtime_config_last_reload_successful`. #2874
 * [FEATURE] Introduced `ruler.for-outage-tolerance`, Max time to tolerate outage for restoring "for" state of alert. #2783
 * [FEATURE] Introduced `ruler.for-grace-period`, Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. #2783
 * [FEATURE] Introduced `ruler.resend-delay`, Minimum amount of time to wait before resending an alert to Alertmanager. #2783
@@ -62,6 +63,7 @@
 * [ENHANCEMENT] Ruler: `-ruler.alertmanager-url` now supports multiple URLs. Each URL is treated as a separate Alertmanager group. Support for multiple Alertmanagers in a group can be achieved by using DNS service discovery. #2851
 * [ENHANCEMENT] Experimental TSDB: Cortex Flusher now works with blocks engine. Flusher needs to be provided with blocks-engine configuration, existing Flusher flags are not used (they are only relevant for chunks engine). Note that flush errors are only reported via log. #2877
 * [ENHANCEMENT] Flusher: Added `-flusher.exit-after-flush` option (defaults to true) to control whether Cortex should stop completely after Flusher has finished its work. #2877
+* [ENHANCEMENT] Added metrics `cortex_config_hash` and `cortex_runtime_config_hash` to expose hash of the currently active config file. #2874
 * [BUGFIX] Fixed a bug in the index intersect code causing storage to return more chunks/series than required. #2796
 * [BUGFIX] Fixed the number of reported keys in the background cache queue. #2764
 * [BUGFIX] Fix race in processing of headers in sharded queries. #2762

cmd/cortex/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"crypto/sha256"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -29,11 +30,21 @@ var (
 	Revision string
 )
 
+// configHash exposes information about the loaded config
+var configHash *prometheus.GaugeVec = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Name: "cortex_config_hash",
+		Help: "Hash of the currently active config file.",
+	},
+	[]string{"sha256"},
+)
+
 func init() {
 	version.Version = Version
 	version.Branch = Branch
 	version.Revision = Revision
 	prometheus.MustRegister(version.NewCollector("cortex"))
+	prometheus.MustRegister(configHash)
 }
 
 const (
@@ -184,6 +195,12 @@ func LoadConfig(filename string, expandENV bool, cfg *cortex.Config) error {
 		return errors.Wrap(err, "Error reading config file")
 	}
 
+	// create a sha256 hash of the config before expansion and expose it via
+	// the config_info metric
+	hash := sha256.Sum256(buf)
+	configHash.Reset()
+	configHash.WithLabelValues(fmt.Sprintf("%x", hash)).Set(1)
+
 	if expandENV {
 		buf = expandEnv(buf)
 	}

pkg/cortex/runtime_config.go

@@ -1,7 +1,7 @@
 package cortex
 
 import (
-	"os"
+	"io"
 
 	"gopkg.in/yaml.v2"
 
@@ -19,15 +19,10 @@ type runtimeConfigValues struct {
 	Multi kv.MultiRuntimeConfig `yaml:"multi_kv_config"`
 }
 
-func loadRuntimeConfig(filename string) (interface{}, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-
+func loadRuntimeConfig(r io.Reader) (interface{}, error) {
 	var overrides = &runtimeConfigValues{}
 
-	decoder := yaml.NewDecoder(f)
+	decoder := yaml.NewDecoder(r)
 	decoder.SetStrict(true)
 	if err := decoder.Decode(&overrides); err != nil {
 		return nil, err

pkg/util/runtimeconfig/manager.go

@@ -1,27 +1,36 @@
 package runtimeconfig
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"errors"
 	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"sync"
 	"time"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 
 	"github.com/cortexproject/cortex/pkg/util"
 	"github.com/cortexproject/cortex/pkg/util/services"
 )
 
 // Loader loads the configuration from file.
-type Loader func(filename string) (interface{}, error)
+type Loader func(r io.Reader) (interface{}, error)
 
 // ManagerConfig holds the config for an Manager instance.
 // It holds config related to loading per-tenant config.
 type ManagerConfig struct {
 	ReloadPeriod time.Duration `yaml:"period"`
-	LoadPath     string        `yaml:"file"`
-	Loader       Loader        `yaml:"-"`
+	// LoadPath contains the path to the runtime config file, requires an
+	// non-empty value
+	LoadPath string `yaml:"file"`
+	Loader   Loader `yaml:"-"`
 }
 
 // RegisterFlags registers flags.
@@ -44,20 +53,25 @@ type Manager struct {
 	config    interface{}
 
 	configLoadSuccess prometheus.Gauge
+	configHash        *prometheus.GaugeVec
 }
 
 // NewRuntimeConfigManager creates an instance of Manager and starts reload config loop based on config
 func NewRuntimeConfigManager(cfg ManagerConfig, registerer prometheus.Registerer) (*Manager, error) {
+	if cfg.LoadPath == "" {
+		return nil, errors.New("LoadPath is empty")
+	}
+
 	mgr := Manager{
 		cfg: cfg,
-		configLoadSuccess: prometheus.NewGauge(prometheus.GaugeOpts{
-			Name: "cortex_overrides_last_reload_successful",
-			Help: "Whether the last config reload attempt was successful.",
+		configLoadSuccess: promauto.With(registerer).NewGauge(prometheus.GaugeOpts{
+			Name: "cortex_runtime_config_last_reload_successful",
+			Help: "Whether the last runtime-config reload attempt was successful.",
 		}),
-	}
-
-	if registerer != nil {
-		registerer.MustRegister(mgr.configLoadSuccess)
+		configHash: promauto.With(registerer).NewGaugeVec(prometheus.GaugeOpts{
+			Name: "cortex_runtime_config_hash",
+			Help: "Hash of the currently active runtime config file.",
+		}, []string{"sha256"}),
 	}
 
 	mgr.Service = services.NewBasicService(mgr.start, mgr.loop, mgr.stop)
@@ -131,7 +145,14 @@ func (om *Manager) loop(ctx context.Context) error {
 // loadConfig loads configuration using the loader function, and if successful,
 // stores it as current configuration and notifies listeners.
 func (om *Manager) loadConfig() error {
-	cfg, err := om.cfg.Loader(om.cfg.LoadPath)
+	buf, err := ioutil.ReadFile(om.cfg.LoadPath)
+	if err != nil {
+		om.configLoadSuccess.Set(0)
+		return err
+	}
+	hash := sha256.Sum256(buf)
+
+	cfg, err := om.cfg.Loader(bytes.NewReader(buf))
 	if err != nil {
 		om.configLoadSuccess.Set(0)
 		return err
@@ -141,6 +162,10 @@ func (om *Manager) loadConfig() error {
 	om.setConfig(cfg)
 	om.callListeners(cfg)
 
+	// expose hash of runtime config
+	om.configHash.Reset()
+	om.configHash.WithLabelValues(fmt.Sprintf("%x", hash[:])).Set(1)
+
 	return nil
 }
 

pkg/util/runtimeconfig/manager_test.go

@@ -2,11 +2,18 @@ package runtimeconfig
 
 import (
 	"context"
+	"crypto/sha256"
+	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
 	"gopkg.in/yaml.v2"
@@ -35,24 +42,40 @@ func (l *TestLimits) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return unmarshal((*plain)(l))
 }
 
-func testLoadOverrides(filename string) (interface{}, error) {
-	f, err := os.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
+func testLoadOverrides(r io.Reader) (interface{}, error) {
 	var overrides = &testOverrides{}
 
-	decoder := yaml.NewDecoder(f)
+	decoder := yaml.NewDecoder(r)
 	decoder.SetStrict(true)
 	if err := decoder.Decode(&overrides); err != nil {
 		return nil, err
 	}
-
 	return overrides, nil
 }
 
+func newTestOverridesManagerConfig(t *testing.T, i int32) (*atomic.Int32, ManagerConfig) {
+	var config = atomic.NewInt32(i)
+
+	// create empty file
+	tempFile, err := ioutil.TempFile("", "test-validation")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		tempFile.Close()
+		os.Remove(tempFile.Name())
+	})
+
+	// testing NewRuntimeConfigManager with overrides reload config set
+	return config, ManagerConfig{
+		ReloadPeriod: 5 * time.Second,
+		LoadPath:     tempFile.Name(),
+		Loader: func(_ io.Reader) (i interface{}, err error) {
+			val := int(config.Load())
+			return val, nil
+		},
+	}
+}
+
 func TestNewOverridesManager(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "test-validation")
 	require.NoError(t, err)
@@ -98,9 +121,10 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) {
 		require.NoError(t, os.Remove(tempFile.Name()))
 	}()
 
-	err = ioutil.WriteFile(tempFile.Name(), []byte(`overrides:
+	config := []byte(`overrides:
   user1:
-    limit2: 150`), 0600)
+    limit2: 150`)
+	err = ioutil.WriteFile(tempFile.Name(), config, 0600)
 	require.NoError(t, err)
 
 	defaultTestLimits = &TestLimits{Limit1: 100}
@@ -112,17 +136,30 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) {
 		Loader:       testLoadOverrides,
 	}
 
-	overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig, nil)
+	reg := prometheus.NewPedanticRegistry()
+
+	overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig, reg)
 	require.NoError(t, err)
 	require.NoError(t, services.StartAndAwaitRunning(context.Background(), overridesManager))
 
+	// check if the metrics is set to the config map value before
+	assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(`
+					# HELP cortex_runtime_config_hash Hash of the currently active runtime config file.
+					# TYPE cortex_runtime_config_hash gauge
+					cortex_runtime_config_hash{sha256="%s"} 1
+					# HELP cortex_runtime_config_last_reload_successful Whether the last runtime-config reload attempt was successful.
+					# TYPE cortex_runtime_config_last_reload_successful gauge
+					cortex_runtime_config_last_reload_successful 1
+				`, fmt.Sprintf("%x", sha256.Sum256(config))))))
+
 	// need to use buffer, otherwise loadConfig will throw away update
 	ch := overridesManager.CreateListenerChannel(1)
 
 	// rewrite file
-	err = ioutil.WriteFile(tempFile.Name(), []byte(`overrides:
+	config = []byte(`overrides:
   user2:
-    limit2: 200`), 0600)
+    limit2: 200`)
+	err = ioutil.WriteFile(tempFile.Name(), config, 0600)
 	require.NoError(t, err)
 
 	// reload
@@ -141,6 +178,16 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) {
 	require.Equal(t, 200, to.Overrides["user2"].Limit2) // new overrides
 	require.Equal(t, 100, to.Overrides["user2"].Limit1) // from defaults
 
+	// check if the metrics have been updated
+	assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(`
+					# HELP cortex_runtime_config_hash Hash of the currently active runtime config file.
+					# TYPE cortex_runtime_config_hash gauge
+					cortex_runtime_config_hash{sha256="%s"} 1
+					# HELP cortex_runtime_config_last_reload_successful Whether the last runtime-config reload attempt was successful.
+					# TYPE cortex_runtime_config_last_reload_successful gauge
+					cortex_runtime_config_last_reload_successful 1
+				`, fmt.Sprintf("%x", sha256.Sum256(config))))))
+
 	// Cleaning up
 	require.NoError(t, services.StopAndAwaitTerminated(context.Background(), overridesManager))
 
@@ -149,17 +196,7 @@ func TestOverridesManager_ListenerWithDefaultLimits(t *testing.T) {
 }
 
 func TestOverridesManager_ListenerChannel(t *testing.T) {
-	var config = atomic.NewInt32(555)
-
-	// testing NewRuntimeConfigManager with overrides reload config set
-	overridesManagerConfig := ManagerConfig{
-		ReloadPeriod: 5 * time.Second,
-		LoadPath:     "ignored",
-		Loader: func(filename string) (i interface{}, err error) {
-			val := int(config.Load())
-			return val, nil
-		},
-	}
+	config, overridesManagerConfig := newTestOverridesManagerConfig(t, 555)
 
 	overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig, nil)
 	require.NoError(t, err)
@@ -199,17 +236,7 @@ func TestOverridesManager_ListenerChannel(t *testing.T) {
 }
 
 func TestOverridesManager_StopClosesListenerChannels(t *testing.T) {
-	var config = atomic.NewInt32(555)
-
-	// testing NewRuntimeConfigManager with overrides reload config set
-	overridesManagerConfig := ManagerConfig{
-		ReloadPeriod: 5 * time.Second,
-		LoadPath:     "ignored",
-		Loader: func(filename string) (i interface{}, err error) {
-			val := int(config.Load())
-			return val, nil
-		},
-	}
+	_, overridesManagerConfig := newTestOverridesManagerConfig(t, 555)
 
 	overridesManager, err := NewRuntimeConfigManager(overridesManagerConfig, nil)
 	require.NoError(t, err)