Configurable delay on rule evaluation (#2423)

* ensure rules are evaluated on a delayed interval

Signed-off-by: Jacob Lisi <[email protected]>

* update changelog and docs

Signed-off-by: Jacob Lisi <[email protected]>

* refactor per PR comments

Signed-off-by: Jacob Lisi <[email protected]>

* update docs

Signed-off-by: Jacob Lisi <[email protected]>

Author: jtlisi

CHANGELOG.md

@@ -9,6 +9,7 @@
   * `cortex_querier_bucket_store_blocks_meta_sync_duration_seconds` > `cortex_querier_blocks_meta_sync_duration_seconds`
   * `cortex_querier_bucket_store_blocks_meta_sync_consistency_delay_seconds` > `cortex_querier_blocks_meta_sync_consistency_delay_seconds`
 * [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414
+* [FEATURE] Ruler: The `-ruler.evaluation-delay` flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423
 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370
 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 
 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392

docs/configuration/config-file-reference.md

@@ -700,6 +700,11 @@ The `ruler_config` configures the Cortex ruler.
 # CLI flag: -ruler.evaluation-interval
 [evaluation_interval: <duration> | default = 1m0s]
 
+# Duration to delay the evaluation of rules to ensure they underlying metrics
+# have been pushed to cortex.
+# CLI flag: -ruler.evaluation-delay-duration
+[evaluation_delay_duration: <duration> | default = 0s]
+
 # How frequently to poll for rule changes
 # CLI flag: -ruler.poll-interval
 [poll_interval: <duration> | default = 1m0s]

pkg/ruler/compat.go

@@ -2,8 +2,11 @@ package ruler
 
 import (
 	"context"
+	"time"
 
 	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/weaveworks/common/user"
 
@@ -78,3 +81,12 @@ func (t *tsdb) StartTime() (int64, error) {
 func (t *tsdb) Close() error {
 	return nil
 }
+
+// engineQueryFunc returns a new query function using the rules.EngineQueryFunc function
+// and passing an altered timestamp.
+func engineQueryFunc(engine *promql.Engine, q storage.Queryable, delay time.Duration) rules.QueryFunc {
+	orig := rules.EngineQueryFunc(engine, q)
+	return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) {
+		return orig(ctx, qs, t.Add(-delay))
+	}
+}

pkg/ruler/ruler.go

@@ -61,6 +61,9 @@ type Config struct {
 	ExternalURL flagext.URLValue `yaml:"external_url"`
 	// How frequently to evaluate rules by default.
 	EvaluationInterval time.Duration `yaml:"evaluation_interval"`
+	// Delay the evaluation of all rules by a set interval to give a buffer
+	// to metric that haven't been forwarded to cortex yet.
+	EvaluationDelay time.Duration `yaml:"evaluation_delay_duration"`
 	// How frequently to poll for updated rules.
 	PollInterval time.Duration `yaml:"poll_interval"`
 	// Rule Storage and Polling configuration.
@@ -103,6 +106,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.ExternalURL.URL, _ = url.Parse("") // Must be non-nil
 	f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.")
 	f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 1*time.Minute, "How frequently to evaluate rules")
+	f.DurationVar(&cfg.EvaluationDelay, "ruler.evaluation-delay-duration", 0, "Duration to delay the evaluation of rules to ensure they underlying metrics have been pushed to cortex.")
 	f.DurationVar(&cfg.PollInterval, "ruler.poll-interval", 1*time.Minute, "How frequently to poll for rule changes")
 	f.Var(&cfg.AlertmanagerURL, "ruler.alertmanager-url", "URL of the Alertmanager to send notifications to.")
 	f.BoolVar(&cfg.AlertmanagerDiscovery, "ruler.alertmanager-discovery", false, "Use DNS SRV records to discover alertmanager hosts.")
@@ -480,7 +484,7 @@ func (r *Ruler) newManager(ctx context.Context, userID string) (*promRules.Manag
 	opts := &promRules.ManagerOptions{
 		Appendable:  tsdb,
 		TSDB:        tsdb,
-		QueryFunc:   promRules.EngineQueryFunc(r.engine, r.queryable),
+		QueryFunc:   engineQueryFunc(r.engine, r.queryable, r.cfg.EvaluationDelay),
 		Context:     user.InjectOrgID(ctx, userID),
 		ExternalURL: r.alertURL,
 		NotifyFunc:  sendAlerts(notifier, r.alertURL.String()),