From 0e566582ff293d1b76204574a3eb11a335ce8b4b Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Thu, 11 Jul 2024 16:55:41 -0700 Subject: [PATCH 1/3] Add support for prometheus rule query offset Signed-off-by: Mustafain Ali Khan --- pkg/ruler/compat.go | 3 + pkg/ruler/ruler.go | 14 ++- pkg/ruler/ruler_test.go | 31 ++++++ pkg/ruler/rulespb/compat.go | 23 +++-- pkg/ruler/rulespb/compat_test.go | 10 +- pkg/ruler/rulespb/rules.pb.go | 162 +++++++++++++++++++++---------- pkg/ruler/rulespb/rules.proto | 4 +- pkg/ruler/store_mock_test.go | 36 +++++++ 8 files changed, 212 insertions(+), 71 deletions(-) diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index b9b4ba2b356..71fc80de1c1 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -358,6 +358,9 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi ResendDelay: cfg.ResendDelay, ConcurrentEvalsEnabled: cfg.ConcurrentEvalsEnabled, MaxConcurrentEvals: cfg.MaxConcurrentEvals, + DefaultRuleQueryOffset: func() time.Duration { + return cfg.RuleQueryOffset + }, }) } } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index f37bc3ce93d..e8e14abe490 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -121,6 +121,8 @@ type Config struct { // Client configs for interacting with the Alertmanager Notifier NotifierConfig `yaml:"alertmanager_client"` + //Default offset for all rule evaluation queries. + RuleQueryOffset time.Duration `yaml:"rule_query_offset"` // Max time to tolerate outage for restoring "for" state of alert. OutageTolerance time.Duration `yaml:"for_outage_tolerance"` // Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. @@ -194,6 +196,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.") f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 1*time.Minute, "How frequently to evaluate rules") f.DurationVar(&cfg.PollInterval, "ruler.poll-interval", 1*time.Minute, "How frequently to poll for rule changes") + f.DurationVar(&cfg.RuleQueryOffset, "ruler.rule-query-offset", 0*time.Minute, "Default offset for all rule evaluation queries") f.StringVar(&cfg.AlertmanagerURL, "ruler.alertmanager-url", "", "Comma-separated list of URL(s) of the Alertmanager(s) to send notifications to. Each Alertmanager URL is treated as a separate group in the configuration. Multiple Alertmanagers in HA per group can be supported by using DNS resolution via -ruler.alertmanager-discovery.") f.BoolVar(&cfg.AlertmanagerDiscovery, "ruler.alertmanager-discovery", false, "Use DNS SRV records to discover Alertmanager hosts.") @@ -913,11 +916,12 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB groupDesc := &GroupStateDesc{ Group: &rulespb.RuleGroupDesc{ - Name: group.Name(), - Namespace: string(decodedNamespace), - Interval: interval, - User: userID, - Limit: int64(group.Limit()), + Name: group.Name(), + Namespace: string(decodedNamespace), + Interval: interval, + User: userID, + Limit: int64(group.Limit()), + QueryOffset: group.QueryOffset(), }, EvaluationTimestamp: group.GetLastEvaluation(), diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index f66efede186..dc578725b0b 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -2533,3 +2533,34 @@ func TestRulerDisablesRuleGroups(t *testing.T) { }) } } + +func TestRuler_QueryOffset(t *testing.T) { + store := newMockRuleStore(mockRulesQueryOffset, nil) + cfg := defaultRulerConfig(t) + + r := newTestRuler(t, cfg, store, nil) + defer services.StopAndAwaitTerminated(context.Background(), r) //nolint:errcheck + + ctx := user.InjectOrgID(context.Background(), "user1") + rls, err := r.Rules(ctx, &RulesRequest{}) + require.NoError(t, err) + require.Len(t, rls.Groups, 1) + rg := rls.Groups[0] + expectedRg := mockRulesQueryOffset["user1"][0] + compareRuleGroupDescToStateDesc(t, expectedRg, rg) + + // test default query offset=0 + require.Equal(t, time.Duration(0), rg.GetGroup().QueryOffset) + + ctx = user.InjectOrgID(context.Background(), "user2") + rls, err = r.Rules(ctx, &RulesRequest{}) + require.NoError(t, err) + require.Len(t, rls.Groups, 1) + rg = rls.Groups[0] + expectedRg = mockRules["user2"][0] + compareRuleGroupDescToStateDesc(t, expectedRg, rg) + + // test group query offset is set + require.Equal(t, time.Minute*2, rg.GetGroup().QueryOffset) + rulespb.FromProto(rg.GetGroup()) +} diff --git a/pkg/ruler/rulespb/compat.go b/pkg/ruler/rulespb/compat.go index f735eb17346..4b8d4394c5f 100644 --- a/pkg/ruler/rulespb/compat.go +++ b/pkg/ruler/rulespb/compat.go @@ -14,12 +14,13 @@ import ( // ToProto transforms a formatted prometheus rulegroup to a rule group protobuf func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc { rg := RuleGroupDesc{ - Name: rl.Name, - Namespace: namespace, - Interval: time.Duration(rl.Interval), - Rules: formattedRuleToProto(rl.Rules), - User: user, - Limit: int64(rl.Limit), + Name: rl.Name, + Namespace: namespace, + Interval: time.Duration(rl.Interval), + Rules: formattedRuleToProto(rl.Rules), + User: user, + Limit: int64(rl.Limit), + QueryOffset: time.Duration(*rl.QueryOffset), } return &rg } @@ -43,11 +44,13 @@ func formattedRuleToProto(rls []rulefmt.RuleNode) []*RuleDesc { // FromProto generates a rulefmt RuleGroup func FromProto(rg *RuleGroupDesc) rulefmt.RuleGroup { + queryOffset := model.Duration(rg.QueryOffset) formattedRuleGroup := rulefmt.RuleGroup{ - Name: rg.GetName(), - Interval: model.Duration(rg.Interval), - Rules: make([]rulefmt.RuleNode, len(rg.GetRules())), - Limit: int(rg.Limit), + Name: rg.GetName(), + Interval: model.Duration(rg.Interval), + Rules: make([]rulefmt.RuleNode, len(rg.GetRules())), + Limit: int(rg.Limit), + QueryOffset: &queryOffset, } for i, rl := range rg.GetRules() { diff --git a/pkg/ruler/rulespb/compat_test.go b/pkg/ruler/rulespb/compat_test.go index c438a3fd078..b66fb3c3877 100644 --- a/pkg/ruler/rulespb/compat_test.go +++ b/pkg/ruler/rulespb/compat_test.go @@ -29,14 +29,18 @@ func TestProto(t *testing.T) { rules = append(rules, testRule) + queryOffset := model.Duration(30 * time.Second) rg := rulefmt.RuleGroup{ - Name: "group1", - Rules: rules, - Interval: model.Duration(time.Minute), + Name: "group1", + Rules: rules, + Interval: model.Duration(time.Minute), + QueryOffset: &queryOffset, } + desc := ToProto("test", "namespace", rg) assert.Equal(t, len(rules), len(desc.Rules)) + assert.Equal(t, 30*time.Second, desc.QueryOffset) ruleDesc := desc.Rules[0] diff --git a/pkg/ruler/rulespb/rules.pb.go b/pkg/ruler/rulespb/rules.pb.go index aa8675c9e09..5ac1eb30806 100644 --- a/pkg/ruler/rulespb/rules.pb.go +++ b/pkg/ruler/rulespb/rules.pb.go @@ -43,8 +43,9 @@ type RuleGroupDesc struct { // having to repeatedly redefine the proto description. It can also be leveraged // to create custom `ManagerOpts` based on rule configs which can then be passed // to the Prometheus Manager. - Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` - Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` + Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` + Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` + QueryOffset time.Duration `protobuf:"bytes,11,opt,name=queryOffset,proto3,stdduration" json:"queryOffset"` } func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } @@ -128,6 +129,13 @@ func (m *RuleGroupDesc) GetLimit() int64 { return 0 } +func (m *RuleGroupDesc) GetQueryOffset() time.Duration { + if m != nil { + return m.QueryOffset + } + return 0 +} + // RuleDesc is a proto representation of a Prometheus Rule type RuleDesc struct { Expr string `protobuf:"bytes,1,opt,name=expr,proto3" json:"expr,omitempty"` @@ -214,40 +222,42 @@ func init() { func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } var fileDescriptor_8e722d3e922f0937 = []byte{ - // 524 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x52, 0x4f, 0x6f, 0xd3, 0x30, - 0x1c, 0x8d, 0xd7, 0x34, 0x4b, 0x5d, 0x55, 0xab, 0x4c, 0x85, 0xbc, 0x81, 0xdc, 0x6a, 0x12, 0x52, - 0x4f, 0x89, 0x34, 0xc4, 0x81, 0x03, 0x42, 0xad, 0xa6, 0x21, 0x55, 0x1c, 0x50, 0x8e, 0x08, 0x69, - 0x72, 0x52, 0x37, 0x84, 0xa5, 0x71, 0xe4, 0x38, 0x68, 0xbb, 0xf1, 0x11, 0xb8, 0x20, 0xf1, 0x11, - 0xf8, 0x28, 0x3b, 0x96, 0xdb, 0xc4, 0xa1, 0xd0, 0xf4, 0x82, 0x38, 0xed, 0x23, 0x20, 0xdb, 0x09, - 0x7f, 0x0f, 0xc0, 0x81, 0x53, 0x7e, 0xcf, 0x2f, 0xcf, 0xef, 0xf9, 0xd9, 0xb0, 0x2b, 0xca, 0x94, - 0x15, 0x5e, 0x2e, 0xb8, 0xe4, 0xa8, 0xad, 0xc1, 0xc1, 0x20, 0xe6, 0x31, 0xd7, 0x2b, 0xbe, 0x9a, - 0x0c, 0x79, 0x40, 0x62, 0xce, 0xe3, 0x94, 0xf9, 0x1a, 0x85, 0xe5, 0xc2, 0x9f, 0x97, 0x82, 0xca, - 0x84, 0x67, 0x35, 0xbf, 0xff, 0x2b, 0x4f, 0xb3, 0x8b, 0x9a, 0xba, 0x1f, 0x27, 0xf2, 0x79, 0x19, - 0x7a, 0x11, 0x5f, 0xfa, 0x11, 0x17, 0x92, 0x9d, 0xe7, 0x82, 0xbf, 0x60, 0x91, 0xac, 0x91, 0x9f, - 0x9f, 0xc5, 0x0d, 0x11, 0xd6, 0x83, 0x91, 0x1e, 0xbe, 0xd9, 0x81, 0xbd, 0xa0, 0x4c, 0xd9, 0x23, - 0xc1, 0xcb, 0xfc, 0x98, 0x15, 0x11, 0x42, 0xd0, 0xce, 0xe8, 0x92, 0x61, 0x30, 0x02, 0xe3, 0x4e, - 0xa0, 0x67, 0x74, 0x1b, 0x76, 0xd4, 0xb7, 0xc8, 0x69, 0xc4, 0xf0, 0x8e, 0x26, 0xbe, 0x2f, 0xa0, - 0x87, 0xd0, 0x4d, 0x32, 0xc9, 0xc4, 0x4b, 0x9a, 0xe2, 0xd6, 0x08, 0x8c, 0xbb, 0x47, 0xfb, 0x9e, - 0x09, 0xeb, 0x35, 0x61, 0xbd, 0xe3, 0xfa, 0x30, 0x53, 0xf7, 0x72, 0x3d, 0xb4, 0xde, 0x7e, 0x1c, - 0x82, 0xe0, 0x9b, 0x08, 0xdd, 0x81, 0xa6, 0x19, 0x6c, 0x8f, 0x5a, 0xe3, 0xee, 0xd1, 0x9e, 0x67, - 0x4a, 0x53, 0xb9, 0x54, 0xa4, 0xc0, 0xb0, 0x2a, 0x59, 0x59, 0x30, 0x81, 0x1d, 0x93, 0x4c, 0xcd, - 0xc8, 0x83, 0xbb, 0x3c, 0x57, 0x1b, 0x17, 0xb8, 0xa3, 0xc5, 0x83, 0xdf, 0xac, 0x27, 0xd9, 0x45, - 0xd0, 0xfc, 0x84, 0x06, 0xb0, 0x9d, 0x26, 0xcb, 0x44, 0x62, 0x38, 0x02, 0xe3, 0x56, 0x60, 0xc0, - 0xcc, 0x76, 0xdb, 0x7d, 0x67, 0x66, 0xbb, 0xbb, 0x7d, 0x77, 0x66, 0xbb, 0x6e, 0xbf, 0x73, 0xf8, - 0xbe, 0x05, 0xdd, 0xc6, 0x5f, 0x19, 0xab, 0x4a, 0x9b, 0x4a, 0xd4, 0x8c, 0x6e, 0x42, 0x47, 0xb0, - 0x88, 0x8b, 0x79, 0xdd, 0x47, 0x8d, 0x94, 0x01, 0x4d, 0x99, 0x90, 0xba, 0x89, 0x4e, 0x60, 0x00, - 0xba, 0x07, 0x5b, 0x0b, 0x2e, 0xb0, 0xfd, 0xf7, 0xed, 0xa8, 0xff, 0x51, 0x06, 0x9d, 0x94, 0x86, - 0x2c, 0x2d, 0x70, 0x5b, 0x1f, 0xee, 0x86, 0xd7, 0xdc, 0xa2, 0xf7, 0x58, 0xad, 0x3f, 0xa1, 0x89, - 0x98, 0x4e, 0x94, 0xe6, 0xc3, 0x7a, 0xf8, 0x4f, 0xaf, 0xc0, 0xe8, 0x27, 0x73, 0x9a, 0x4b, 0x26, - 0x82, 0xda, 0x05, 0x9d, 0xc3, 0x2e, 0xcd, 0x32, 0x2e, 0xa9, 0x69, 0xd4, 0xf9, 0xaf, 0xa6, 0x3f, - 0x5a, 0xa1, 0x67, 0xb0, 0x77, 0xc6, 0x58, 0x7e, 0x92, 0x88, 0x24, 0x8b, 0x4f, 0xb8, 0xc0, 0xbd, - 0x3f, 0x55, 0x75, 0x4b, 0x25, 0xf8, 0xb2, 0x1e, 0xee, 0x29, 0xdd, 0xe9, 0x42, 0x0b, 0x4f, 0x17, - 0x5c, 0xe8, 0xf6, 0x7e, 0xde, 0x4c, 0xdf, 0x6c, 0x6f, 0xfa, 0x60, 0xb5, 0x21, 0xd6, 0xd5, 0x86, - 0x58, 0xd7, 0x1b, 0x02, 0x5e, 0x55, 0x04, 0xbc, 0xab, 0x08, 0xb8, 0xac, 0x08, 0x58, 0x55, 0x04, - 0x7c, 0xaa, 0x08, 0xf8, 0x5c, 0x11, 0xeb, 0xba, 0x22, 0xe0, 0xf5, 0x96, 0x58, 0xab, 0x2d, 0xb1, - 0xae, 0xb6, 0xc4, 0x7a, 0xba, 0xab, 0x1f, 0x5f, 0x1e, 0x86, 0x8e, 0xce, 0x70, 0xf7, 0x6b, 0x00, - 0x00, 0x00, 0xff, 0xff, 0x77, 0x8c, 0xa0, 0x4e, 0xd3, 0x03, 0x00, 0x00, + // 545 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x52, 0x41, 0x8b, 0xd3, 0x4c, + 0x18, 0xce, 0x6c, 0xd3, 0x6c, 0x3a, 0xa1, 0xec, 0x32, 0x5f, 0xf9, 0xc8, 0xae, 0x32, 0x2d, 0x0b, + 0x42, 0x4f, 0x29, 0xac, 0x78, 0xf0, 0x20, 0xd2, 0xb2, 0xae, 0x50, 0x04, 0x25, 0x47, 0x11, 0x96, + 0x49, 0x3a, 0x89, 0x71, 0xd3, 0x4c, 0x9c, 0x4c, 0x64, 0x7b, 0xf3, 0x27, 0x78, 0xf4, 0x27, 0xf8, + 0x53, 0xf6, 0x58, 0x6f, 0x8b, 0x48, 0xb5, 0xe9, 0x45, 0x3c, 0xed, 0x4f, 0x90, 0x99, 0x49, 0x74, + 0xd5, 0x83, 0xeb, 0xc1, 0x53, 0xde, 0x67, 0x9e, 0x3c, 0xf3, 0x3e, 0xf3, 0xbc, 0x2f, 0x74, 0x78, + 0x99, 0xd2, 0xc2, 0xcb, 0x39, 0x13, 0x0c, 0xb5, 0x15, 0xd8, 0xef, 0xc5, 0x2c, 0x66, 0xea, 0x64, + 0x24, 0x2b, 0x4d, 0xee, 0xe3, 0x98, 0xb1, 0x38, 0xa5, 0x23, 0x85, 0x82, 0x32, 0x1a, 0xcd, 0x4a, + 0x4e, 0x44, 0xc2, 0xb2, 0x9a, 0xdf, 0xfb, 0x95, 0x27, 0xd9, 0xa2, 0xa6, 0xee, 0xc6, 0x89, 0x78, + 0x5e, 0x06, 0x5e, 0xc8, 0xe6, 0xa3, 0x90, 0x71, 0x41, 0xcf, 0x72, 0xce, 0x5e, 0xd0, 0x50, 0xd4, + 0x68, 0x94, 0x9f, 0xc6, 0x0d, 0x11, 0xd4, 0x85, 0x96, 0x1e, 0x7c, 0xdc, 0x82, 0x5d, 0xbf, 0x4c, + 0xe9, 0x43, 0xce, 0xca, 0xfc, 0x88, 0x16, 0x21, 0x42, 0xd0, 0xcc, 0xc8, 0x9c, 0xba, 0x60, 0x00, + 0x86, 0x1d, 0x5f, 0xd5, 0xe8, 0x26, 0xec, 0xc8, 0x6f, 0x91, 0x93, 0x90, 0xba, 0x5b, 0x8a, 0xf8, + 0x71, 0x80, 0xee, 0x43, 0x3b, 0xc9, 0x04, 0xe5, 0xaf, 0x48, 0xea, 0xb6, 0x06, 0x60, 0xe8, 0x1c, + 0xee, 0x79, 0xda, 0xac, 0xd7, 0x98, 0xf5, 0x8e, 0xea, 0xc7, 0x4c, 0xec, 0xf3, 0x55, 0xdf, 0x78, + 0xfb, 0xa9, 0x0f, 0xfc, 0xef, 0x22, 0x74, 0x0b, 0xea, 0x64, 0x5c, 0x73, 0xd0, 0x1a, 0x3a, 0x87, + 0x3b, 0x9e, 0x0e, 0x4d, 0xfa, 0x92, 0x96, 0x7c, 0xcd, 0x4a, 0x67, 0x65, 0x41, 0xb9, 0x6b, 0x69, + 0x67, 0xb2, 0x46, 0x1e, 0xdc, 0x66, 0xb9, 0xbc, 0xb8, 0x70, 0x3b, 0x4a, 0xdc, 0xfb, 0xad, 0xf5, + 0x38, 0x5b, 0xf8, 0xcd, 0x4f, 0xa8, 0x07, 0xdb, 0x69, 0x32, 0x4f, 0x84, 0x0b, 0x07, 0x60, 0xd8, + 0xf2, 0x35, 0x40, 0x0f, 0xa0, 0xf3, 0xb2, 0xa4, 0x7c, 0xf1, 0x38, 0x8a, 0x0a, 0x2a, 0x5c, 0xe7, + 0xfa, 0x8f, 0xb8, 0xaa, 0x9b, 0x9a, 0x76, 0x7b, 0xd7, 0x9a, 0x9a, 0xf6, 0xf6, 0xae, 0x3d, 0x35, + 0x6d, 0x7b, 0xb7, 0x73, 0xf0, 0xbe, 0x05, 0xed, 0xe6, 0x19, 0xd2, 0xbf, 0x9c, 0x4c, 0x93, 0xac, + 0xac, 0xd1, 0xff, 0xd0, 0xe2, 0x34, 0x64, 0x7c, 0x56, 0xc7, 0x5a, 0x23, 0xe9, 0x93, 0xa4, 0x94, + 0x0b, 0x15, 0x68, 0xc7, 0xd7, 0x00, 0xdd, 0x81, 0xad, 0x88, 0x71, 0xd7, 0xbc, 0xbe, 0x3f, 0xf9, + 0x3f, 0xca, 0xa0, 0x95, 0x92, 0x80, 0xa6, 0x85, 0xdb, 0x56, 0x19, 0xfd, 0xe7, 0x35, 0xcb, 0xe0, + 0x3d, 0x92, 0xe7, 0x4f, 0x48, 0xc2, 0x27, 0x63, 0xa9, 0xf9, 0xb0, 0xea, 0xff, 0xd5, 0x32, 0x69, + 0xfd, 0x78, 0x46, 0x72, 0x41, 0xb9, 0x5f, 0x77, 0x41, 0x67, 0xd0, 0x21, 0x59, 0xc6, 0x04, 0xd1, + 0x83, 0xb1, 0xfe, 0x69, 0xd3, 0xab, 0xad, 0xd0, 0x33, 0xd8, 0x3d, 0xa5, 0x34, 0x3f, 0x4e, 0x78, + 0x92, 0xc5, 0xc7, 0x8c, 0xbb, 0xdd, 0x3f, 0x45, 0x75, 0x43, 0x3a, 0xf8, 0xba, 0xea, 0xef, 0x48, + 0xdd, 0x49, 0xa4, 0x84, 0x27, 0x11, 0xe3, 0x2a, 0xbd, 0x9f, 0x2f, 0x53, 0x93, 0xed, 0x4e, 0xee, + 0x2d, 0xd7, 0xd8, 0xb8, 0x58, 0x63, 0xe3, 0x72, 0x8d, 0xc1, 0xeb, 0x0a, 0x83, 0x77, 0x15, 0x06, + 0xe7, 0x15, 0x06, 0xcb, 0x0a, 0x83, 0xcf, 0x15, 0x06, 0x5f, 0x2a, 0x6c, 0x5c, 0x56, 0x18, 0xbc, + 0xd9, 0x60, 0x63, 0xb9, 0xc1, 0xc6, 0xc5, 0x06, 0x1b, 0x4f, 0xb7, 0xd5, 0x0e, 0xe7, 0x41, 0x60, + 0x29, 0x0f, 0xb7, 0xbf, 0x05, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xaa, 0xf7, 0xf4, 0x1a, 0x04, 0x00, + 0x00, } func (this *RuleGroupDesc) Equal(that interface{}) bool { @@ -300,6 +310,9 @@ func (this *RuleGroupDesc) Equal(that interface{}) bool { if this.Limit != that1.Limit { return false } + if this.QueryOffset != that1.QueryOffset { + return false + } return true } func (this *RuleDesc) Equal(that interface{}) bool { @@ -358,7 +371,7 @@ func (this *RuleGroupDesc) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&rulespb.RuleGroupDesc{") s = append(s, "Name: "+fmt.Sprintf("%#v", this.Name)+",\n") s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") @@ -371,6 +384,7 @@ func (this *RuleGroupDesc) GoString() string { s = append(s, "Options: "+fmt.Sprintf("%#v", this.Options)+",\n") } s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") + s = append(s, "QueryOffset: "+fmt.Sprintf("%#v", this.QueryOffset)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -418,6 +432,14 @@ func (m *RuleGroupDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.QueryOffset, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset):]) + if err1 != nil { + return 0, err1 + } + i -= n1 + i = encodeVarintRules(dAtA, i, uint64(n1)) + i-- + dAtA[i] = 0x5a if m.Limit != 0 { i = encodeVarintRules(dAtA, i, uint64(m.Limit)) i-- @@ -458,12 +480,12 @@ func (m *RuleGroupDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x22 } } - n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.Interval, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.Interval):]) - if err1 != nil { - return 0, err1 + n2, err2 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.Interval, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.Interval):]) + if err2 != nil { + return 0, err2 } - i -= n1 - i = encodeVarintRules(dAtA, i, uint64(n1)) + i -= n2 + i = encodeVarintRules(dAtA, i, uint64(n2)) i-- dAtA[i] = 0x1a if len(m.Namespace) > 0 { @@ -503,12 +525,12 @@ func (m *RuleDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - n2, err2 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.KeepFiringFor, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.KeepFiringFor):]) - if err2 != nil { - return 0, err2 + n3, err3 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.KeepFiringFor, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.KeepFiringFor):]) + if err3 != nil { + return 0, err3 } - i -= n2 - i = encodeVarintRules(dAtA, i, uint64(n2)) + i -= n3 + i = encodeVarintRules(dAtA, i, uint64(n3)) i-- dAtA[i] = 0x6a if len(m.Annotations) > 0 { @@ -539,12 +561,12 @@ func (m *RuleDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x2a } } - n3, err3 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.For, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.For):]) - if err3 != nil { - return 0, err3 + n4, err4 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.For, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.For):]) + if err4 != nil { + return 0, err4 } - i -= n3 - i = encodeVarintRules(dAtA, i, uint64(n3)) + i -= n4 + i = encodeVarintRules(dAtA, i, uint64(n4)) i-- dAtA[i] = 0x22 if len(m.Alert) > 0 { @@ -617,6 +639,8 @@ func (m *RuleGroupDesc) Size() (n int) { if m.Limit != 0 { n += 1 + sovRules(uint64(m.Limit)) } + l = github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset) + n += 1 + l + sovRules(uint64(l)) return n } @@ -685,6 +709,7 @@ func (this *RuleGroupDesc) String() string { `User:` + fmt.Sprintf("%v", this.User) + `,`, `Options:` + repeatedStringForOptions + `,`, `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, + `QueryOffset:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.QueryOffset), "Duration", "duration.Duration", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -958,6 +983,39 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field QueryOffset", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRules + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRules + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthRules + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(&m.QueryOffset, dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRules(dAtA[iNdEx:]) diff --git a/pkg/ruler/rulespb/rules.proto b/pkg/ruler/rulespb/rules.proto index 1a6f4182ec5..189c050eb47 100644 --- a/pkg/ruler/rulespb/rules.proto +++ b/pkg/ruler/rulespb/rules.proto @@ -28,6 +28,8 @@ message RuleGroupDesc { // to the Prometheus Manager. repeated google.protobuf.Any options = 9; int64 limit =10; + google.protobuf.Duration queryOffset = 11 + [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; } // RuleDesc is a proto representation of a Prometheus Rule @@ -46,4 +48,4 @@ message RuleDesc { (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" ]; google.protobuf.Duration keepFiringFor = 13 [(gogoproto.nullable) = false,(gogoproto.stdduration) = true, (gogoproto.jsontag) = "keep_firing_for"]; -} \ No newline at end of file +} diff --git a/pkg/ruler/store_mock_test.go b/pkg/ruler/store_mock_test.go index da3707e840c..5bd6025f01a 100644 --- a/pkg/ruler/store_mock_test.go +++ b/pkg/ruler/store_mock_test.go @@ -132,6 +132,42 @@ var ( }, }, } + mockRulesQueryOffset = map[string]rulespb.RuleGroupList{ + "user1": { + &rulespb.RuleGroupDesc{ + Name: "group1", + Namespace: "namespace1", + User: "user1", + Limit: 5, + Rules: []*rulespb.RuleDesc{ + { + Record: "UP_RULE", + Expr: "up", + }, + { + Alert: "UP_ALERT", + Expr: "up < 1", + }, + }, + Interval: interval, + }, + }, + "user2": { + &rulespb.RuleGroupDesc{ + Name: "group1", + Namespace: "namespace1", + User: "user2", + Rules: []*rulespb.RuleDesc{ + { + Record: "UP_RULE", + Expr: "up", + }, + }, + Interval: interval, + QueryOffset: 2 * time.Minute, + }, + }, + } ) func newMockRuleStore(rules map[string]rulespb.RuleGroupList, errorMap map[string]error) *mockRuleStore { From d96cf38c3076bab2d3b9d73fb1463032a5717bdb Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Thu, 11 Jul 2024 17:11:20 -0700 Subject: [PATCH 2/3] Fix tests Signed-off-by: Mustafain Ali Khan --- docs/configuration/config-file-reference.md | 4 ++++ pkg/ruler/api_test.go | 19 ++++++++++++++++--- pkg/ruler/ruler.go | 4 ++-- pkg/ruler/ruler_test.go | 3 +-- pkg/ruler/rulespb/compat.go | 6 +++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index df036ed2f6b..f4f7865552a 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -4147,6 +4147,10 @@ ruler_client: # CLI flag: -ruler.rule-path [rule_path: | default = "/rules"] +# Default offset for all rule evaluation queries +# CLI flag: -ruler.rule-query-offset +[rule_query_offset: | default = 0s] + # Comma-separated list of URL(s) of the Alertmanager(s) to send notifications # to. Each Alertmanager URL is treated as a separate group in the configuration. # Multiple Alertmanagers in HA per group can be supported by using DNS diff --git a/pkg/ruler/api_test.go b/pkg/ruler/api_test.go index 132ecfcf64d..cdddcf24321 100644 --- a/pkg/ruler/api_test.go +++ b/pkg/ruler/api_test.go @@ -263,7 +263,7 @@ interval: 15s err: errors.New("invalid rules config: rule group 'rg_name' has no rules"), }, { - name: "with a a valid rules file", + name: "with a valid rules file", status: 202, input: ` name: test @@ -279,7 +279,20 @@ rules: labels: test: test `, - output: "name: test\ninterval: 15s\nrules:\n - record: up_rule\n expr: up{}\n - alert: up_alert\n expr: sum(up{}) > 1\n for: 30s\n labels:\n test: test\n annotations:\n test: test\n", + output: "name: test\ninterval: 15s\nquery_offset: 0s\nrules:\n - record: up_rule\n expr: up{}\n - alert: up_alert\n expr: sum(up{}) > 1\n for: 30s\n labels:\n test: test\n annotations:\n test: test\n", + }, + { + name: "with a valid rule query offset", + status: 202, + input: ` +name: test +interval: 15s +query_offset: 2m +rules: +- record: up_rule + expr: up{} +`, + output: "name: test\ninterval: 15s\nquery_offset: 2m\nrules:\n - record: up_rule\n expr: up{}\n", }, } @@ -329,7 +342,7 @@ func TestRuler_DeleteNamespace(t *testing.T) { router.ServeHTTP(w, req) require.Equal(t, http.StatusOK, w.Code) - require.Equal(t, "name: group1\ninterval: 1m\nrules:\n - record: UP_RULE\n expr: up\n - alert: UP_ALERT\n expr: up < 1\n", w.Body.String()) + require.Equal(t, "name: group1\ninterval: 1m\nquery_offset: 0s\nrules:\n - record: UP_RULE\n expr: up\n - alert: UP_ALERT\n expr: up < 1\n", w.Body.String()) // Delete namespace1 req = requestFor(t, http.MethodDelete, "https://localhost:8080/api/v1/rules/namespace1", nil, "user1") diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index e8e14abe490..0d6f8a57cb1 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -105,6 +105,8 @@ type Config struct { PollInterval time.Duration `yaml:"poll_interval"` // Path to store rule files for prom manager. RulePath string `yaml:"rule_path"` + // Default offset for all rule evaluation queries. + RuleQueryOffset time.Duration `yaml:"rule_query_offset"` // URL of the Alertmanager to send notifications to. // If you are configuring the ruler to send to a Cortex Alertmanager, @@ -121,8 +123,6 @@ type Config struct { // Client configs for interacting with the Alertmanager Notifier NotifierConfig `yaml:"alertmanager_client"` - //Default offset for all rule evaluation queries. - RuleQueryOffset time.Duration `yaml:"rule_query_offset"` // Max time to tolerate outage for restoring "for" state of alert. OutageTolerance time.Duration `yaml:"for_outage_tolerance"` // Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period. diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index dc578725b0b..9fc391fb729 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -2549,7 +2549,7 @@ func TestRuler_QueryOffset(t *testing.T) { expectedRg := mockRulesQueryOffset["user1"][0] compareRuleGroupDescToStateDesc(t, expectedRg, rg) - // test default query offset=0 + // test default query offset=0 when not defined at group level require.Equal(t, time.Duration(0), rg.GetGroup().QueryOffset) ctx = user.InjectOrgID(context.Background(), "user2") @@ -2562,5 +2562,4 @@ func TestRuler_QueryOffset(t *testing.T) { // test group query offset is set require.Equal(t, time.Minute*2, rg.GetGroup().QueryOffset) - rulespb.FromProto(rg.GetGroup()) } diff --git a/pkg/ruler/rulespb/compat.go b/pkg/ruler/rulespb/compat.go index 4b8d4394c5f..4f916d09c4b 100644 --- a/pkg/ruler/rulespb/compat.go +++ b/pkg/ruler/rulespb/compat.go @@ -13,6 +13,10 @@ import ( // ToProto transforms a formatted prometheus rulegroup to a rule group protobuf func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc { + queryOffset := time.Duration(0) + if rl.QueryOffset != nil { + queryOffset = time.Duration(*rl.QueryOffset) + } rg := RuleGroupDesc{ Name: rl.Name, Namespace: namespace, @@ -20,7 +24,7 @@ func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc Rules: formattedRuleToProto(rl.Rules), User: user, Limit: int64(rl.Limit), - QueryOffset: time.Duration(*rl.QueryOffset), + QueryOffset: queryOffset, } return &rg } From feba91aa1fe7372a263d32a0aa3742a968ca97d0 Mon Sep 17 00:00:00 2001 From: Mustafain Ali Khan Date: Fri, 12 Jul 2024 12:55:09 -0700 Subject: [PATCH 3/3] Use per-tenant limit for global query offset Signed-off-by: Mustafain Ali Khan --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 8 +- pkg/ruler/compat.go | 3 +- pkg/ruler/ruler.go | 6 +- pkg/ruler/ruler_test.go | 11 ++- pkg/ruler/rulespb/compat.go | 10 ++- pkg/ruler/rulespb/compat_test.go | 2 +- pkg/ruler/rulespb/rules.pb.go | 93 ++++++++++++--------- pkg/ruler/rulespb/rules.proto | 2 +- pkg/ruler/store_mock_test.go | 3 +- pkg/util/validation/limits.go | 7 ++ 11 files changed, 89 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb01005e0d3..9042c8be3ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * [FEATURE] Ingester: Add `ingester.instance-limits.max-inflight-query-requests` to allow limiting ingester concurrent queries. #6081 * [FEATURE] Distributor: Add `validation.max-native-histogram-buckets` to limit max number of bucket count. Distributor will try to automatically reduce histogram resolution until it is within the bucket limit or resolution cannot be reduced anymore. #6104 * [FEATURE] Store Gateway: Token bucket limiter. #6016 +* [FEATURE] Ruler: Add support for `query_offset` field on RuleGroup and new `ruler_query_offset` per-tenant limit. #6085 * [ENHANCEMENT] rulers: Add support to persist tokens in rulers. #5987 * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index f4f7865552a..0e646f77881 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3349,6 +3349,10 @@ query_rejection: # CLI flag: -ruler.max-rule-groups-per-tenant [ruler_max_rule_groups_per_tenant: | default = 0] +# Duration to offset all rule evaluation queries per-tenant. +# CLI flag: -ruler.query-offset +[ruler_query_offset: | default = 0s] + # The default tenant's shard size when the shuffle-sharding strategy is used. # Must be set when the store-gateway sharding is enabled with the # shuffle-sharding strategy. When this setting is specified in the per-tenant @@ -4147,10 +4151,6 @@ ruler_client: # CLI flag: -ruler.rule-path [rule_path: | default = "/rules"] -# Default offset for all rule evaluation queries -# CLI flag: -ruler.rule-query-offset -[rule_query_offset: | default = 0s] - # Comma-separated list of URL(s) of the Alertmanager(s) to send notifications # to. Each Alertmanager URL is treated as a separate group in the configuration. # Multiple Alertmanagers in HA per group can be supported by using DNS diff --git a/pkg/ruler/compat.go b/pkg/ruler/compat.go index 71fc80de1c1..9c3fd2f0f91 100644 --- a/pkg/ruler/compat.go +++ b/pkg/ruler/compat.go @@ -178,6 +178,7 @@ type RulesLimits interface { RulerTenantShardSize(userID string) int RulerMaxRuleGroupsPerTenant(userID string) int RulerMaxRulesPerRuleGroup(userID string) int + RulerQueryOffset(userID string) time.Duration DisabledRuleGroups(userID string) validation.DisabledRuleGroups } @@ -359,7 +360,7 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi ConcurrentEvalsEnabled: cfg.ConcurrentEvalsEnabled, MaxConcurrentEvals: cfg.MaxConcurrentEvals, DefaultRuleQueryOffset: func() time.Duration { - return cfg.RuleQueryOffset + return overrides.RulerQueryOffset(userID) }, }) } diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 0d6f8a57cb1..e87b9b07009 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -105,8 +105,6 @@ type Config struct { PollInterval time.Duration `yaml:"poll_interval"` // Path to store rule files for prom manager. RulePath string `yaml:"rule_path"` - // Default offset for all rule evaluation queries. - RuleQueryOffset time.Duration `yaml:"rule_query_offset"` // URL of the Alertmanager to send notifications to. // If you are configuring the ruler to send to a Cortex Alertmanager, @@ -196,7 +194,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.Var(&cfg.ExternalURL, "ruler.external.url", "URL of alerts return path.") f.DurationVar(&cfg.EvaluationInterval, "ruler.evaluation-interval", 1*time.Minute, "How frequently to evaluate rules") f.DurationVar(&cfg.PollInterval, "ruler.poll-interval", 1*time.Minute, "How frequently to poll for rule changes") - f.DurationVar(&cfg.RuleQueryOffset, "ruler.rule-query-offset", 0*time.Minute, "Default offset for all rule evaluation queries") f.StringVar(&cfg.AlertmanagerURL, "ruler.alertmanager-url", "", "Comma-separated list of URL(s) of the Alertmanager(s) to send notifications to. Each Alertmanager URL is treated as a separate group in the configuration. Multiple Alertmanagers in HA per group can be supported by using DNS resolution via -ruler.alertmanager-discovery.") f.BoolVar(&cfg.AlertmanagerDiscovery, "ruler.alertmanager-discovery", false, "Use DNS SRV records to discover Alertmanager hosts.") @@ -914,6 +911,7 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB } interval := group.Interval() + queryOffset := group.QueryOffset() groupDesc := &GroupStateDesc{ Group: &rulespb.RuleGroupDesc{ Name: group.Name(), @@ -921,7 +919,7 @@ func (r *Ruler) getLocalRules(userID string, rulesRequest RulesRequest, includeB Interval: interval, User: userID, Limit: int64(group.Limit()), - QueryOffset: group.QueryOffset(), + QueryOffset: &queryOffset, }, EvaluationTimestamp: group.GetLastEvaluation(), diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 9fc391fb729..befe7d01f44 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -88,6 +88,7 @@ type ruleLimits struct { maxRuleGroups int disabledRuleGroups validation.DisabledRuleGroups maxQueryLength time.Duration + queryOffset time.Duration } func (r ruleLimits) EvaluationDelay(_ string) time.Duration { @@ -112,6 +113,10 @@ func (r ruleLimits) DisabledRuleGroups(userID string) validation.DisabledRuleGro func (r ruleLimits) MaxQueryLength(_ string) time.Duration { return r.maxQueryLength } +func (r ruleLimits) RulerQueryOffset(_ string) time.Duration { + return r.queryOffset +} + func newEmptyQueryable() storage.Queryable { return storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { return emptyQuerier{}, nil @@ -2550,7 +2555,8 @@ func TestRuler_QueryOffset(t *testing.T) { compareRuleGroupDescToStateDesc(t, expectedRg, rg) // test default query offset=0 when not defined at group level - require.Equal(t, time.Duration(0), rg.GetGroup().QueryOffset) + gotOffset := rg.GetGroup().QueryOffset + require.Equal(t, time.Duration(0), *gotOffset) ctx = user.InjectOrgID(context.Background(), "user2") rls, err = r.Rules(ctx, &RulesRequest{}) @@ -2561,5 +2567,6 @@ func TestRuler_QueryOffset(t *testing.T) { compareRuleGroupDescToStateDesc(t, expectedRg, rg) // test group query offset is set - require.Equal(t, time.Minute*2, rg.GetGroup().QueryOffset) + gotOffset = rg.GetGroup().QueryOffset + require.Equal(t, time.Minute*2, *gotOffset) } diff --git a/pkg/ruler/rulespb/compat.go b/pkg/ruler/rulespb/compat.go index 4f916d09c4b..16bf075e27f 100644 --- a/pkg/ruler/rulespb/compat.go +++ b/pkg/ruler/rulespb/compat.go @@ -13,9 +13,10 @@ import ( // ToProto transforms a formatted prometheus rulegroup to a rule group protobuf func ToProto(user string, namespace string, rl rulefmt.RuleGroup) *RuleGroupDesc { - queryOffset := time.Duration(0) + var queryOffset *time.Duration if rl.QueryOffset != nil { - queryOffset = time.Duration(*rl.QueryOffset) + offset := time.Duration(*rl.QueryOffset) + queryOffset = &offset } rg := RuleGroupDesc{ Name: rl.Name, @@ -48,7 +49,10 @@ func formattedRuleToProto(rls []rulefmt.RuleNode) []*RuleDesc { // FromProto generates a rulefmt RuleGroup func FromProto(rg *RuleGroupDesc) rulefmt.RuleGroup { - queryOffset := model.Duration(rg.QueryOffset) + var queryOffset model.Duration + if rg.QueryOffset != nil { + queryOffset = model.Duration(*rg.QueryOffset) + } formattedRuleGroup := rulefmt.RuleGroup{ Name: rg.GetName(), Interval: model.Duration(rg.Interval), diff --git a/pkg/ruler/rulespb/compat_test.go b/pkg/ruler/rulespb/compat_test.go index b66fb3c3877..736366714dc 100644 --- a/pkg/ruler/rulespb/compat_test.go +++ b/pkg/ruler/rulespb/compat_test.go @@ -40,7 +40,7 @@ func TestProto(t *testing.T) { desc := ToProto("test", "namespace", rg) assert.Equal(t, len(rules), len(desc.Rules)) - assert.Equal(t, 30*time.Second, desc.QueryOffset) + assert.Equal(t, 30*time.Second, *desc.QueryOffset) ruleDesc := desc.Rules[0] diff --git a/pkg/ruler/rulespb/rules.pb.go b/pkg/ruler/rulespb/rules.pb.go index 5ac1eb30806..8f09b2cb40b 100644 --- a/pkg/ruler/rulespb/rules.pb.go +++ b/pkg/ruler/rulespb/rules.pb.go @@ -43,9 +43,9 @@ type RuleGroupDesc struct { // having to repeatedly redefine the proto description. It can also be leveraged // to create custom `ManagerOpts` based on rule configs which can then be passed // to the Prometheus Manager. - Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` - Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` - QueryOffset time.Duration `protobuf:"bytes,11,opt,name=queryOffset,proto3,stdduration" json:"queryOffset"` + Options []*types.Any `protobuf:"bytes,9,rep,name=options,proto3" json:"options,omitempty"` + Limit int64 `protobuf:"varint,10,opt,name=limit,proto3" json:"limit,omitempty"` + QueryOffset *time.Duration `protobuf:"bytes,11,opt,name=queryOffset,proto3,stdduration" json:"queryOffset,omitempty"` } func (m *RuleGroupDesc) Reset() { *m = RuleGroupDesc{} } @@ -129,11 +129,11 @@ func (m *RuleGroupDesc) GetLimit() int64 { return 0 } -func (m *RuleGroupDesc) GetQueryOffset() time.Duration { +func (m *RuleGroupDesc) GetQueryOffset() *time.Duration { if m != nil { return m.QueryOffset } - return 0 + return nil } // RuleDesc is a proto representation of a Prometheus Rule @@ -222,13 +222,13 @@ func init() { func init() { proto.RegisterFile("rules.proto", fileDescriptor_8e722d3e922f0937) } var fileDescriptor_8e722d3e922f0937 = []byte{ - // 545 bytes of a gzipped FileDescriptorProto + // 548 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x52, 0x41, 0x8b, 0xd3, 0x4c, 0x18, 0xce, 0x6c, 0xd3, 0x6c, 0x3a, 0xa1, 0xec, 0x32, 0x5f, 0xf9, 0xc8, 0xae, 0x32, 0x2d, 0x0b, 0x42, 0x4f, 0x29, 0xac, 0x78, 0xf0, 0x20, 0xd2, 0xb2, 0xae, 0x50, 0x04, 0x25, 0x47, 0x11, 0x96, 0x49, 0x3a, 0x89, 0x71, 0xd3, 0x4c, 0x9c, 0x4c, 0x64, 0x7b, 0xf3, 0x27, 0x78, 0xf4, 0x27, 0xf8, 0x53, 0xf6, 0x58, 0x6f, 0x8b, 0x48, 0xb5, 0xe9, 0x45, 0x3c, 0xed, 0x4f, 0x90, 0x99, 0x49, 0x74, - 0xd5, 0x83, 0xeb, 0xc1, 0x53, 0xde, 0x67, 0x9e, 0x3c, 0xf3, 0x3e, 0xf3, 0xbc, 0x2f, 0x74, 0x78, + 0xd5, 0x83, 0xeb, 0xc1, 0x53, 0xde, 0x67, 0x9e, 0x3c, 0xf3, 0x3e, 0xef, 0x33, 0x2f, 0x74, 0x78, 0x99, 0xd2, 0xc2, 0xcb, 0x39, 0x13, 0x0c, 0xb5, 0x15, 0xd8, 0xef, 0xc5, 0x2c, 0x66, 0xea, 0x64, 0x24, 0x2b, 0x4d, 0xee, 0xe3, 0x98, 0xb1, 0x38, 0xa5, 0x23, 0x85, 0x82, 0x32, 0x1a, 0xcd, 0x4a, 0x4e, 0x44, 0xc2, 0xb2, 0x9a, 0xdf, 0xfb, 0x95, 0x27, 0xd9, 0xa2, 0xa6, 0xee, 0xc6, 0x89, 0x78, @@ -237,27 +237,27 @@ var fileDescriptor_8e722d3e922f0937 = []byte{ 0xe9, 0x43, 0xce, 0xca, 0xfc, 0x88, 0x16, 0x21, 0x42, 0xd0, 0xcc, 0xc8, 0x9c, 0xba, 0x60, 0x00, 0x86, 0x1d, 0x5f, 0xd5, 0xe8, 0x26, 0xec, 0xc8, 0x6f, 0x91, 0x93, 0x90, 0xba, 0x5b, 0x8a, 0xf8, 0x71, 0x80, 0xee, 0x43, 0x3b, 0xc9, 0x04, 0xe5, 0xaf, 0x48, 0xea, 0xb6, 0x06, 0x60, 0xe8, 0x1c, - 0xee, 0x79, 0xda, 0xac, 0xd7, 0x98, 0xf5, 0x8e, 0xea, 0xc7, 0x4c, 0xec, 0xf3, 0x55, 0xdf, 0x78, - 0xfb, 0xa9, 0x0f, 0xfc, 0xef, 0x22, 0x74, 0x0b, 0xea, 0x64, 0x5c, 0x73, 0xd0, 0x1a, 0x3a, 0x87, - 0x3b, 0x9e, 0x0e, 0x4d, 0xfa, 0x92, 0x96, 0x7c, 0xcd, 0x4a, 0x67, 0x65, 0x41, 0xb9, 0x6b, 0x69, - 0x67, 0xb2, 0x46, 0x1e, 0xdc, 0x66, 0xb9, 0xbc, 0xb8, 0x70, 0x3b, 0x4a, 0xdc, 0xfb, 0xad, 0xf5, - 0x38, 0x5b, 0xf8, 0xcd, 0x4f, 0xa8, 0x07, 0xdb, 0x69, 0x32, 0x4f, 0x84, 0x0b, 0x07, 0x60, 0xd8, - 0xf2, 0x35, 0x40, 0x0f, 0xa0, 0xf3, 0xb2, 0xa4, 0x7c, 0xf1, 0x38, 0x8a, 0x0a, 0x2a, 0x5c, 0xe7, - 0xfa, 0x8f, 0xb8, 0xaa, 0x9b, 0x9a, 0x76, 0x7b, 0xd7, 0x9a, 0x9a, 0xf6, 0xf6, 0xae, 0x3d, 0x35, - 0x6d, 0x7b, 0xb7, 0x73, 0xf0, 0xbe, 0x05, 0xed, 0xe6, 0x19, 0xd2, 0xbf, 0x9c, 0x4c, 0x93, 0xac, - 0xac, 0xd1, 0xff, 0xd0, 0xe2, 0x34, 0x64, 0x7c, 0x56, 0xc7, 0x5a, 0x23, 0xe9, 0x93, 0xa4, 0x94, - 0x0b, 0x15, 0x68, 0xc7, 0xd7, 0x00, 0xdd, 0x81, 0xad, 0x88, 0x71, 0xd7, 0xbc, 0xbe, 0x3f, 0xf9, - 0x3f, 0xca, 0xa0, 0x95, 0x92, 0x80, 0xa6, 0x85, 0xdb, 0x56, 0x19, 0xfd, 0xe7, 0x35, 0xcb, 0xe0, - 0x3d, 0x92, 0xe7, 0x4f, 0x48, 0xc2, 0x27, 0x63, 0xa9, 0xf9, 0xb0, 0xea, 0xff, 0xd5, 0x32, 0x69, - 0xfd, 0x78, 0x46, 0x72, 0x41, 0xb9, 0x5f, 0x77, 0x41, 0x67, 0xd0, 0x21, 0x59, 0xc6, 0x04, 0xd1, - 0x83, 0xb1, 0xfe, 0x69, 0xd3, 0xab, 0xad, 0xd0, 0x33, 0xd8, 0x3d, 0xa5, 0x34, 0x3f, 0x4e, 0x78, - 0x92, 0xc5, 0xc7, 0x8c, 0xbb, 0xdd, 0x3f, 0x45, 0x75, 0x43, 0x3a, 0xf8, 0xba, 0xea, 0xef, 0x48, - 0xdd, 0x49, 0xa4, 0x84, 0x27, 0x11, 0xe3, 0x2a, 0xbd, 0x9f, 0x2f, 0x53, 0x93, 0xed, 0x4e, 0xee, - 0x2d, 0xd7, 0xd8, 0xb8, 0x58, 0x63, 0xe3, 0x72, 0x8d, 0xc1, 0xeb, 0x0a, 0x83, 0x77, 0x15, 0x06, - 0xe7, 0x15, 0x06, 0xcb, 0x0a, 0x83, 0xcf, 0x15, 0x06, 0x5f, 0x2a, 0x6c, 0x5c, 0x56, 0x18, 0xbc, - 0xd9, 0x60, 0x63, 0xb9, 0xc1, 0xc6, 0xc5, 0x06, 0x1b, 0x4f, 0xb7, 0xd5, 0x0e, 0xe7, 0x41, 0x60, - 0x29, 0x0f, 0xb7, 0xbf, 0x05, 0x00, 0x00, 0xff, 0xff, 0xa5, 0xaa, 0xf7, 0xf4, 0x1a, 0x04, 0x00, - 0x00, + 0xee, 0x79, 0xda, 0xac, 0xd7, 0x98, 0xf5, 0x8e, 0xea, 0x61, 0x26, 0xf6, 0xf9, 0xaa, 0x6f, 0xbc, + 0xfd, 0xd4, 0x07, 0xfe, 0x77, 0x11, 0xba, 0x05, 0x75, 0x32, 0xae, 0x39, 0x68, 0x0d, 0x9d, 0xc3, + 0x1d, 0x4f, 0x87, 0x26, 0x7d, 0x49, 0x4b, 0xbe, 0x66, 0xa5, 0xb3, 0xb2, 0xa0, 0xdc, 0xb5, 0xb4, + 0x33, 0x59, 0x23, 0x0f, 0x6e, 0xb3, 0x5c, 0x5e, 0x5c, 0xb8, 0x1d, 0x25, 0xee, 0xfd, 0xd6, 0x7a, + 0x9c, 0x2d, 0xfc, 0xe6, 0x27, 0xd4, 0x83, 0xed, 0x34, 0x99, 0x27, 0xc2, 0x85, 0x03, 0x30, 0x6c, + 0xf9, 0x1a, 0xa0, 0x07, 0xd0, 0x79, 0x59, 0x52, 0xbe, 0x78, 0x1c, 0x45, 0x05, 0x15, 0xae, 0x73, + 0x9d, 0x21, 0x80, 0x1a, 0xe2, 0xaa, 0x6e, 0x6a, 0xda, 0xed, 0x5d, 0x6b, 0x6a, 0xda, 0xdb, 0xbb, + 0xf6, 0xd4, 0xb4, 0xed, 0xdd, 0xce, 0xc1, 0xfb, 0x16, 0xb4, 0x9b, 0x31, 0xa4, 0x7f, 0xf9, 0x32, + 0x4d, 0xb2, 0xb2, 0x46, 0xff, 0x43, 0x8b, 0xd3, 0x90, 0xf1, 0x59, 0x1d, 0x6b, 0x8d, 0xa4, 0x4f, + 0x92, 0x52, 0x2e, 0x54, 0xa0, 0x1d, 0x5f, 0x03, 0x74, 0x07, 0xb6, 0x22, 0xc6, 0x5d, 0xf3, 0xfa, + 0x21, 0xcb, 0xff, 0x51, 0x06, 0xad, 0x94, 0x04, 0x34, 0x2d, 0xdc, 0xb6, 0xca, 0xe8, 0x3f, 0xaf, + 0x59, 0x06, 0xef, 0x91, 0x3c, 0x7f, 0x42, 0x12, 0x3e, 0x19, 0x4b, 0xcd, 0x87, 0x55, 0xff, 0xaf, + 0x96, 0x49, 0xeb, 0xc7, 0x33, 0x92, 0x0b, 0xca, 0xfd, 0xba, 0x0b, 0x3a, 0x83, 0x0e, 0xc9, 0x32, + 0x26, 0x88, 0x7e, 0x18, 0xeb, 0x9f, 0x36, 0xbd, 0xda, 0x0a, 0x3d, 0x83, 0xdd, 0x53, 0x4a, 0xf3, + 0xe3, 0x84, 0x27, 0x59, 0x7c, 0xcc, 0xb8, 0xdb, 0xfd, 0x53, 0x54, 0x37, 0xa4, 0x83, 0xaf, 0xab, + 0xfe, 0x8e, 0xd4, 0x9d, 0x44, 0x4a, 0x78, 0x12, 0x31, 0xae, 0xd2, 0xfb, 0xf9, 0x32, 0xf5, 0xb2, + 0xdd, 0xc9, 0xbd, 0xe5, 0x1a, 0x1b, 0x17, 0x6b, 0x6c, 0x5c, 0xae, 0x31, 0x78, 0x5d, 0x61, 0xf0, + 0xae, 0xc2, 0xe0, 0xbc, 0xc2, 0x60, 0x59, 0x61, 0xf0, 0xb9, 0xc2, 0xe0, 0x4b, 0x85, 0x8d, 0xcb, + 0x0a, 0x83, 0x37, 0x1b, 0x6c, 0x2c, 0x37, 0xd8, 0xb8, 0xd8, 0x60, 0xe3, 0xe9, 0xb6, 0xda, 0xe1, + 0x3c, 0x08, 0x2c, 0xe5, 0xe1, 0xf6, 0xb7, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x1c, 0xe8, 0x17, + 0x1a, 0x04, 0x00, 0x00, } func (this *RuleGroupDesc) Equal(that interface{}) bool { @@ -310,7 +310,13 @@ func (this *RuleGroupDesc) Equal(that interface{}) bool { if this.Limit != that1.Limit { return false } - if this.QueryOffset != that1.QueryOffset { + if this.QueryOffset != nil && that1.QueryOffset != nil { + if *this.QueryOffset != *that1.QueryOffset { + return false + } + } else if this.QueryOffset != nil { + return false + } else if that1.QueryOffset != nil { return false } return true @@ -432,14 +438,16 @@ func (m *RuleGroupDesc) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.QueryOffset, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset):]) - if err1 != nil { - return 0, err1 + if m.QueryOffset != nil { + n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(*m.QueryOffset, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(*m.QueryOffset):]) + if err1 != nil { + return 0, err1 + } + i -= n1 + i = encodeVarintRules(dAtA, i, uint64(n1)) + i-- + dAtA[i] = 0x5a } - i -= n1 - i = encodeVarintRules(dAtA, i, uint64(n1)) - i-- - dAtA[i] = 0x5a if m.Limit != 0 { i = encodeVarintRules(dAtA, i, uint64(m.Limit)) i-- @@ -639,8 +647,10 @@ func (m *RuleGroupDesc) Size() (n int) { if m.Limit != 0 { n += 1 + sovRules(uint64(m.Limit)) } - l = github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryOffset) - n += 1 + l + sovRules(uint64(l)) + if m.QueryOffset != nil { + l = github_com_gogo_protobuf_types.SizeOfStdDuration(*m.QueryOffset) + n += 1 + l + sovRules(uint64(l)) + } return n } @@ -709,7 +719,7 @@ func (this *RuleGroupDesc) String() string { `User:` + fmt.Sprintf("%v", this.User) + `,`, `Options:` + repeatedStringForOptions + `,`, `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, - `QueryOffset:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.QueryOffset), "Duration", "duration.Duration", 1), `&`, ``, 1) + `,`, + `QueryOffset:` + strings.Replace(fmt.Sprintf("%v", this.QueryOffset), "Duration", "duration.Duration", 1) + `,`, `}`, }, "") return s @@ -1012,7 +1022,10 @@ func (m *RuleGroupDesc) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(&m.QueryOffset, dAtA[iNdEx:postIndex]); err != nil { + if m.QueryOffset == nil { + m.QueryOffset = new(time.Duration) + } + if err := github_com_gogo_protobuf_types.StdDurationUnmarshal(m.QueryOffset, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/pkg/ruler/rulespb/rules.proto b/pkg/ruler/rulespb/rules.proto index 189c050eb47..ed4e98a76b2 100644 --- a/pkg/ruler/rulespb/rules.proto +++ b/pkg/ruler/rulespb/rules.proto @@ -29,7 +29,7 @@ message RuleGroupDesc { repeated google.protobuf.Any options = 9; int64 limit =10; google.protobuf.Duration queryOffset = 11 - [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; + [(gogoproto.nullable) = true, (gogoproto.stdduration) = true]; } // RuleDesc is a proto representation of a Prometheus Rule diff --git a/pkg/ruler/store_mock_test.go b/pkg/ruler/store_mock_test.go index 5bd6025f01a..30a53fdab16 100644 --- a/pkg/ruler/store_mock_test.go +++ b/pkg/ruler/store_mock_test.go @@ -132,6 +132,7 @@ var ( }, }, } + queryOffset = 2 * time.Minute mockRulesQueryOffset = map[string]rulespb.RuleGroupList{ "user1": { &rulespb.RuleGroupDesc{ @@ -164,7 +165,7 @@ var ( }, }, Interval: interval, - QueryOffset: 2 * time.Minute, + QueryOffset: &queryOffset, }, }, } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index fc9faab0780..2e885a6c407 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -178,6 +178,7 @@ type Limits struct { RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"` RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"` RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"` + RulerQueryOffset model.Duration `yaml:"ruler_query_offset" json:"ruler_query_offset"` // Store-gateway. StoreGatewayTenantShardSize float64 `yaml:"store_gateway_tenant_shard_size" json:"store_gateway_tenant_shard_size"` @@ -268,6 +269,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.RulerTenantShardSize, "ruler.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by ruler. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") f.IntVar(&l.RulerMaxRulesPerRuleGroup, "ruler.max-rules-per-rule-group", 0, "Maximum number of rules per rule group per-tenant. 0 to disable.") f.IntVar(&l.RulerMaxRuleGroupsPerTenant, "ruler.max-rule-groups-per-tenant", 0, "Maximum number of rule groups per-tenant. 0 to disable.") + f.Var(&l.RulerQueryOffset, "ruler.query-offset", "Duration to offset all rule evaluation queries per-tenant.") f.Var(&l.CompactorBlocksRetentionPeriod, "compactor.blocks-retention-period", "Delete blocks containing samples older than the specified retention period. 0 to disable.") f.IntVar(&l.CompactorTenantShardSize, "compactor.tenant-shard-size", 0, "The default tenant's shard size when the shuffle-sharding strategy is used by the compactor. When this setting is specified in the per-tenant overrides, a value of 0 disables shuffle sharding for the tenant.") @@ -791,6 +793,11 @@ func (o *Overrides) RulerMaxRuleGroupsPerTenant(userID string) int { return o.GetOverridesForUser(userID).RulerMaxRuleGroupsPerTenant } +// RulerQueryOffset returns the rule query offset for a given user. +func (o *Overrides) RulerQueryOffset(userID string) time.Duration { + return time.Duration(o.GetOverridesForUser(userID).RulerQueryOffset) +} + // StoreGatewayTenantShardSize returns the store-gateway shard size for a given user. func (o *Overrides) StoreGatewayTenantShardSize(userID string) float64 { return o.GetOverridesForUser(userID).StoreGatewayTenantShardSize