From a1ae6004e063de85231cc28d61a216901f5e325c Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 09:59:18 -0800 Subject: [PATCH 01/17] feat(metrics): Add Raft leadership metrics for Alpha. --- worker/draft.go | 10 ++++++++++ x/metrics.go | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/worker/draft.go b/worker/draft.go index c508adf7e92..30a9491af62 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -1113,6 +1113,16 @@ func (n *node) Run() { if rd.SoftState != nil { groups().triggerMembershipSync() leader = rd.RaftState == raft.StateLeader + if rd.SoftState.Lead != raft.None { + ostats.Record(context.Background(), x.RaftHasLeader.M(1)) + } else { + ostats.Record(context.Background(), x.RaftHasLeader.M(0)) + } + if leader { + ostats.Record(context.Background(), x.RaftIsLeader.M(1)) + } else { + ostats.Record(context.Background(), x.RaftIsLeader.M(0)) + } } if leader { // Leader can send messages in parallel with writing to disk. diff --git a/x/metrics.go b/x/metrics.go index 4f53f56ac42..d186e5e4c49 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -111,6 +111,11 @@ var ( // PLCacheHitRatio records the hit ratio of posting list cache. PLCacheHitRatio = stats.Float64("hit_ratio_posting_cache", "Hit ratio of posting list cache", stats.UnitDimensionless) + // MaxAssignedTs records the latest max assigned timestamp. + RaftHasLeader = stats.Int64("has_leader", + "Whether or not a leader exists", stats.UnitDimensionless) + RaftIsLeader = stats.Int64("is_leader", + "Whether or not a leader exists", stats.UnitDimensionless) // Conf holds the metrics config. // TODO: Request statistics, latencies, 500, timeouts From 7040e84a1fc5cf2a2e5c1a0d3419bd95b3e295cb Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 10:08:38 -0800 Subject: [PATCH 02/17] feat(metrics): Add Raft leadership metrics for Zero. --- dgraph/cmd/zero/raft.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index d91961190e0..793a18495c1 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -859,6 +859,16 @@ func (n *node) Run() { n.server.updateLeases() } leader = rd.RaftState == raft.StateLeader + if rd.SoftState.Lead != raft.None { + ostats.Record(context.Background(), x.RaftHasLeader.M(1)) + } else { + ostats.Record(context.Background(), x.RaftHasLeader.M(0)) + } + if leader { + ostats.Record(context.Background(), x.RaftIsLeader.M(1)) + } else { + ostats.Record(context.Background(), x.RaftIsLeader.M(0)) + } // Oracle stream would close the stream once it steps down as leader // predicate move would cancel any in progress move on stepping down. n.triggerLeaderChange() From 1c3e072115dd8a215aa2744235d9e4562d0718af Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 10:41:19 -0800 Subject: [PATCH 03/17] chore: Fix ostats import in Zero. --- dgraph/cmd/zero/raft.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index 793a18495c1..aa9925f8708 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -27,8 +27,6 @@ import ( "sync" "time" - otrace "go.opencensus.io/trace" - "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" @@ -39,6 +37,8 @@ import ( "github.com/pkg/errors" "go.etcd.io/etcd/raft" "go.etcd.io/etcd/raft/raftpb" + ostats "go.opencensus.io/stats" + otrace "go.opencensus.io/trace" ) type node struct { From 5e65b3b9c4268d3617da5540fc0d8e43102ce2b6 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 10:52:56 -0800 Subject: [PATCH 04/17] Add view for leadership metrics. --- x/metrics.go | 70 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/x/metrics.go b/x/metrics.go index d186e5e4c49..90d8bf8861f 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -168,34 +168,6 @@ var ( Aggregation: view.Count(), TagKeys: allTagKeys, }, - { - Name: RaftAppliedIndex.Name(), - Measure: RaftAppliedIndex, - Description: RaftAppliedIndex.Description(), - Aggregation: view.LastValue(), - TagKeys: allTagKeys, - }, - { - Name: RaftApplyCh.Name(), - Measure: RaftApplyCh, - Description: RaftApplyCh.Description(), - Aggregation: view.LastValue(), - TagKeys: allTagKeys, - }, - { - Name: RaftPendingSize.Name(), - Measure: RaftPendingSize, - Description: RaftPendingSize.Description(), - Aggregation: view.LastValue(), - TagKeys: allTagKeys, - }, - { - Name: MaxAssignedTs.Name(), - Measure: MaxAssignedTs, - Description: MaxAssignedTs.Description(), - Aggregation: view.LastValue(), - TagKeys: allTagKeys, - }, { Name: TxnAborts.Name(), Measure: TxnAborts, @@ -282,6 +254,48 @@ var ( Aggregation: view.LastValue(), TagKeys: allTagKeys, }, + { + Name: RaftAppliedIndex.Name(), + Measure: RaftAppliedIndex, + Description: RaftAppliedIndex.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, + { + Name: RaftApplyCh.Name(), + Measure: RaftApplyCh, + Description: RaftApplyCh.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, + { + Name: RaftPendingSize.Name(), + Measure: RaftPendingSize, + Description: RaftPendingSize.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, + { + Name: RaftHasLeader.Name(), + Measure: RaftHasLeader, + Description: RaftHasLeader.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, + { + Name: RaftIsLeader.Name(), + Measure: RaftIsLeader, + Description: RaftIsLeader.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, + { + Name: MaxAssignedTs.Name(), + Measure: MaxAssignedTs, + Description: MaxAssignedTs.Description(), + Aggregation: view.LastValue(), + TagKeys: allTagKeys, + }, } ) From a8727f7aab6f2e4cf96c5ef440060c3c7802a648 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 11:01:39 -0800 Subject: [PATCH 05/17] fix: Update metric descriptions. --- x/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/metrics.go b/x/metrics.go index 90d8bf8861f..79102ea3f1e 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -113,9 +113,9 @@ var ( "Hit ratio of posting list cache", stats.UnitDimensionless) // MaxAssignedTs records the latest max assigned timestamp. RaftHasLeader = stats.Int64("has_leader", - "Whether or not a leader exists", stats.UnitDimensionless) + "Whether or not a leader exists for the group", stats.UnitDimensionless) RaftIsLeader = stats.Int64("is_leader", - "Whether or not a leader exists", stats.UnitDimensionless) + "Whether or not this instance is the leader of the group", stats.UnitDimensionless) // Conf holds the metrics config. // TODO: Request statistics, latencies, 500, timeouts From 608c54e6d53a9830039d414ce7de740a856ea6b5 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 11:04:23 -0800 Subject: [PATCH 06/17] fix: Prefix metrics with dgraph_raft. --- x/metrics.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/metrics.go b/x/metrics.go index 79102ea3f1e..b320a57bc78 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -112,9 +112,9 @@ var ( PLCacheHitRatio = stats.Float64("hit_ratio_posting_cache", "Hit ratio of posting list cache", stats.UnitDimensionless) // MaxAssignedTs records the latest max assigned timestamp. - RaftHasLeader = stats.Int64("has_leader", + RaftHasLeader = stats.Int64("raft_has_leader", "Whether or not a leader exists for the group", stats.UnitDimensionless) - RaftIsLeader = stats.Int64("is_leader", + RaftIsLeader = stats.Int64("raft_is_leader", "Whether or not this instance is the leader of the group", stats.UnitDimensionless) // Conf holds the metrics config. From 33f7e02d274dcdf69da9272a985306044f9b2d81 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 11:33:47 -0800 Subject: [PATCH 07/17] Add group label for Raft metrics. Still needs to be set correctly. --- worker/draft.go | 10 ++++++---- x/metrics.go | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/worker/draft.go b/worker/draft.go index 30a9491af62..172ae193ffa 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -1113,15 +1113,17 @@ func (n *node) Run() { if rd.SoftState != nil { groups().triggerMembershipSync() leader = rd.RaftState == raft.StateLeader + // ctx, _ := tag.New(context.Background(), tag.Upsert(x.KeyGroup, "10")) + ctx := context.Background() if rd.SoftState.Lead != raft.None { - ostats.Record(context.Background(), x.RaftHasLeader.M(1)) + ostats.Record(ctx, x.RaftHasLeader.M(1)) } else { - ostats.Record(context.Background(), x.RaftHasLeader.M(0)) + ostats.Record(ctx, x.RaftHasLeader.M(0)) } if leader { - ostats.Record(context.Background(), x.RaftIsLeader.M(1)) + ostats.Record(ctx, x.RaftIsLeader.M(1)) } else { - ostats.Record(context.Background(), x.RaftIsLeader.M(0)) + ostats.Record(ctx, x.RaftIsLeader.M(0)) } } if leader { diff --git a/x/metrics.go b/x/metrics.go index b320a57bc78..a04f560a75b 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -111,9 +111,10 @@ var ( // PLCacheHitRatio records the hit ratio of posting list cache. PLCacheHitRatio = stats.Float64("hit_ratio_posting_cache", "Hit ratio of posting list cache", stats.UnitDimensionless) - // MaxAssignedTs records the latest max assigned timestamp. + // RaftHasLeader records whether this instance has a leader RaftHasLeader = stats.Int64("raft_has_leader", "Whether or not a leader exists for the group", stats.UnitDimensionless) + // RaftIsLeader records whether this instance is the leader RaftIsLeader = stats.Int64("raft_is_leader", "Whether or not this instance is the leader of the group", stats.UnitDimensionless) @@ -123,6 +124,9 @@ var ( // Tag keys. + // KeyGroup is the tag key used to record the group for Raft metrics. + KeyGroup, _ = tag.NewKey("group") + // KeyStatus is the tag key used to record the status of the server. KeyStatus, _ = tag.NewKey("status") // KeyMethod is the tag key used to record the method (e.g read or mutate). @@ -146,6 +150,8 @@ var ( KeyStatus, KeyMethod, } + allRaftKeys = []tag.Key{KeyGroup} + allViews = []*view.View{ { Name: LatencyMs.Name(), @@ -259,35 +265,35 @@ var ( Measure: RaftAppliedIndex, Description: RaftAppliedIndex.Description(), Aggregation: view.LastValue(), - TagKeys: allTagKeys, + TagKeys: allRaftKeys, }, { Name: RaftApplyCh.Name(), Measure: RaftApplyCh, Description: RaftApplyCh.Description(), Aggregation: view.LastValue(), - TagKeys: allTagKeys, + TagKeys: allRaftKeys, }, { Name: RaftPendingSize.Name(), Measure: RaftPendingSize, Description: RaftPendingSize.Description(), Aggregation: view.LastValue(), - TagKeys: allTagKeys, + TagKeys: allRaftKeys, }, { Name: RaftHasLeader.Name(), Measure: RaftHasLeader, Description: RaftHasLeader.Description(), Aggregation: view.LastValue(), - TagKeys: allTagKeys, + TagKeys: allRaftKeys, }, { Name: RaftIsLeader.Name(), Measure: RaftIsLeader, Description: RaftIsLeader.Description(), Aggregation: view.LastValue(), - TagKeys: allTagKeys, + TagKeys: allRaftKeys, }, { Name: MaxAssignedTs.Name(), From 002e18f9e9eba11dcb0dde89cfffd2ad50c6d715 Mon Sep 17 00:00:00 2001 From: Daniel Mai Date: Tue, 19 Jan 2021 14:39:02 -0800 Subject: [PATCH 08/17] chore: Use node context instead of context.Background(). --- worker/draft.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/draft.go b/worker/draft.go index 172ae193ffa..f133c4bd92a 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -1113,8 +1113,8 @@ func (n *node) Run() { if rd.SoftState != nil { groups().triggerMembershipSync() leader = rd.RaftState == raft.StateLeader - // ctx, _ := tag.New(context.Background(), tag.Upsert(x.KeyGroup, "10")) - ctx := context.Background() + // ctx, _ := tag.New(n.ctx, tag.Upsert(x.KeyGroup, "10")) + ctx := n.ctx if rd.SoftState.Lead != raft.None { ostats.Record(ctx, x.RaftHasLeader.M(1)) } else { From a9d9d45b26a20afee37b3a11c7ddccf3619f78ee Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Wed, 20 Jan 2021 17:23:21 -0500 Subject: [PATCH 09/17] add leader_changes_seen_total metric --- worker/draft.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/worker/draft.go b/worker/draft.go index f133c4bd92a..305d833c234 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "encoding/hex" "fmt" + "math" "sort" "sync" "sync/atomic" @@ -1059,6 +1060,9 @@ const tickDur = 100 * time.Millisecond func (n *node) Run() { defer n.closer.Done() // CLOSER:1 + // lastLead is for detecting leadership changes + lastLead := uint64(math.MaxUint64) + firstRun := true var leader bool // See also our configuration of HeartbeatTick and ElectionTick. @@ -1113,8 +1117,13 @@ func (n *node) Run() { if rd.SoftState != nil { groups().triggerMembershipSync() leader = rd.RaftState == raft.StateLeader - // ctx, _ := tag.New(n.ctx, tag.Upsert(x.KeyGroup, "10")) - ctx := n.ctx + // create context with group id + ctx, _ := tag.New(n.ctx, tag.Upsert(x.KeyGroup, fmt.Sprintf("%d", n.gid))) + // detect leadership changes + if rd.SoftState.Lead != lastLead { + lastLead = rd.SoftState.Lead + ostats.Record(ctx, x.RaftLeaderChangesSeenTotal.M(1)) + } if rd.SoftState.Lead != raft.None { ostats.Record(ctx, x.RaftHasLeader.M(1)) } else { From 24903c6998b6d7a1656c1847326398aa516b39de Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Wed, 20 Jan 2021 17:24:45 -0500 Subject: [PATCH 10/17] leader_changes_seen_total --- x/metrics.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x/metrics.go b/x/metrics.go index a04f560a75b..3a37885e3a9 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -117,6 +117,9 @@ var ( // RaftIsLeader records whether this instance is the leader RaftIsLeader = stats.Int64("raft_is_leader", "Whether or not this instance is the leader of the group", stats.UnitDimensionless) + // RaftLeaderChangesSeenTotal records the total number of leader changes seen. + RaftLeaderChangesSeenTotal = stats.Int64("raft_leader_changes_seen_total", + "Total number of leader changes seen", stats.UnitDimensionless) // Conf holds the metrics config. // TODO: Request statistics, latencies, 500, timeouts @@ -295,6 +298,13 @@ var ( Aggregation: view.LastValue(), TagKeys: allRaftKeys, }, + { + Name: RaftLeaderChangesSeenTotal.Name(), + Measure: RaftLeaderChangesSeenTotal, + Description: RaftLeaderChangesSeenTotal.Description(), + Aggregation: view.Count(), + TagKeys: nil, + }, { Name: MaxAssignedTs.Name(), Measure: MaxAssignedTs, From fca08e3dbc3e5d1060e33bf977080ebc191e709e Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Wed, 20 Jan 2021 17:29:43 -0500 Subject: [PATCH 11/17] raft keys --- x/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/metrics.go b/x/metrics.go index 3a37885e3a9..c03b5066a5c 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -303,7 +303,7 @@ var ( Measure: RaftLeaderChangesSeenTotal, Description: RaftLeaderChangesSeenTotal.Description(), Aggregation: view.Count(), - TagKeys: nil, + TagKeys: allRaftKeys, }, { Name: MaxAssignedTs.Name(), From 5e3742b687a062daa4384bffed14b71f24d79883 Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Thu, 21 Jan 2021 08:49:02 -0500 Subject: [PATCH 12/17] zero leader changes --- dgraph/cmd/zero/raft.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index aa9925f8708..58fdd0cc2cc 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -807,6 +807,9 @@ func (n *node) calculateAndProposeSnapshot() error { const tickDur = 100 * time.Millisecond func (n *node) Run() { + // lastLead is for detecting leadership changes + lastLead := uint64(math.MaxUint64) + var leader bool licenseApplied := false ticker := time.NewTicker(tickDur) @@ -859,6 +862,10 @@ func (n *node) Run() { n.server.updateLeases() } leader = rd.RaftState == raft.StateLeader + if rd.SoftState.Lead != lastLead { + lastLead = rd.SoftState.Lead + ostats.Record(context.Background(), x.RaftLeaderChangesSeenTotal.M(1)) + } if rd.SoftState.Lead != raft.None { ostats.Record(context.Background(), x.RaftHasLeader.M(1)) } else { From 29ffe7b61bd6e3a0a9dffb63164a2ddc5c182230 Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Thu, 21 Jan 2021 10:28:01 -0500 Subject: [PATCH 13/17] do we need tests? --- worker/draft_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/worker/draft_test.go b/worker/draft_test.go index 9d1fa72ec0f..72cab8147a7 100644 --- a/worker/draft_test.go +++ b/worker/draft_test.go @@ -114,3 +114,7 @@ func TestCalculateSnapshot(t *testing.T) { require.NoError(t, err) require.Nil(t, snap) } + +func TestRaftMetrics(t *testing.T) { + // TODO +} From 4c3cf9a72be519d3f966369dccb5c58e344f98dc Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Fri, 22 Jan 2021 13:28:47 -0500 Subject: [PATCH 14/17] hardcode 0 group for zero --- dgraph/cmd/zero/raft.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index 58fdd0cc2cc..ee6d3bf0781 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -38,6 +38,7 @@ import ( "go.etcd.io/etcd/raft" "go.etcd.io/etcd/raft/raftpb" ostats "go.opencensus.io/stats" + "go.opencensus.io/tag" otrace "go.opencensus.io/trace" ) @@ -862,19 +863,21 @@ func (n *node) Run() { n.server.updateLeases() } leader = rd.RaftState == raft.StateLeader + // group id hardcoded as 0 + ctx, _ := tag.New(n.ctx, tag.Upsert(x.KeyGroup, "0")) if rd.SoftState.Lead != lastLead { lastLead = rd.SoftState.Lead - ostats.Record(context.Background(), x.RaftLeaderChangesSeenTotal.M(1)) + ostats.Record(ctx, x.RaftLeaderChangesSeenTotal.M(1)) } if rd.SoftState.Lead != raft.None { - ostats.Record(context.Background(), x.RaftHasLeader.M(1)) + ostats.Record(ctx, x.RaftHasLeader.M(1)) } else { - ostats.Record(context.Background(), x.RaftHasLeader.M(0)) + ostats.Record(ctx, x.RaftHasLeader.M(0)) } if leader { - ostats.Record(context.Background(), x.RaftIsLeader.M(1)) + ostats.Record(ctx, x.RaftIsLeader.M(1)) } else { - ostats.Record(context.Background(), x.RaftIsLeader.M(0)) + ostats.Record(ctx, x.RaftIsLeader.M(0)) } // Oracle stream would close the stream once it steps down as leader // predicate move would cancel any in progress move on stepping down. From 17b0c1a2ee0d7db3695bf36bbc13ec3074ae7a10 Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Fri, 22 Jan 2021 14:09:36 -0500 Subject: [PATCH 15/17] documentation --- dgraph/cmd/zero/raft.go | 3 +++ worker/draft.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index ee6d3bf0781..c7f77daaf23 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -809,6 +809,9 @@ const tickDur = 100 * time.Millisecond func (n *node) Run() { // lastLead is for detecting leadership changes + // + // etcd has a similar mechanism for tracking leader changes, with their + // raftReadyHandler.getLead() function that returns the previous leader lastLead := uint64(math.MaxUint64) var leader bool diff --git a/worker/draft.go b/worker/draft.go index 305d833c234..00f3536c8db 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -1061,6 +1061,9 @@ func (n *node) Run() { defer n.closer.Done() // CLOSER:1 // lastLead is for detecting leadership changes + // + // etcd has a similar mechanism for tracking leader changes, with their + // raftReadyHandler.getLead() function that returns the previous leader lastLead := uint64(math.MaxUint64) firstRun := true From b8db763b5f46ca7e4a0bde62895d325ee298c92b Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Fri, 22 Jan 2021 14:13:08 -0500 Subject: [PATCH 16/17] require raft metrics and change LeaderChangesSeenTotal to LeaderChanges --- dgraph/cmd/alpha/metrics_test.go | 3 +++ dgraph/cmd/zero/raft.go | 2 +- worker/draft.go | 2 +- worker/draft_test.go | 4 ---- x/metrics.go | 4 ++-- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dgraph/cmd/alpha/metrics_test.go b/dgraph/cmd/alpha/metrics_test.go index 3edf8536d93..0a898b658be 100644 --- a/dgraph/cmd/alpha/metrics_test.go +++ b/dgraph/cmd/alpha/metrics_test.go @@ -115,6 +115,9 @@ func TestMetrics(t *testing.T) { "dgraph_active_mutations_total", "dgraph_pending_proposals_total", "dgraph_pending_queries_total", "dgraph_num_queries_total", "dgraph_alpha_health_status", + + // Raft metrics + "dgraph_raft_has_leader", "dgraph_raft_is_leader", "dgraph_raft_leader_changes_total", } for _, requiredM := range requiredMetrics { _, ok := metricsMap[requiredM] diff --git a/dgraph/cmd/zero/raft.go b/dgraph/cmd/zero/raft.go index c7f77daaf23..d95e99c0f23 100644 --- a/dgraph/cmd/zero/raft.go +++ b/dgraph/cmd/zero/raft.go @@ -870,7 +870,7 @@ func (n *node) Run() { ctx, _ := tag.New(n.ctx, tag.Upsert(x.KeyGroup, "0")) if rd.SoftState.Lead != lastLead { lastLead = rd.SoftState.Lead - ostats.Record(ctx, x.RaftLeaderChangesSeenTotal.M(1)) + ostats.Record(ctx, x.RaftLeaderChanges.M(1)) } if rd.SoftState.Lead != raft.None { ostats.Record(ctx, x.RaftHasLeader.M(1)) diff --git a/worker/draft.go b/worker/draft.go index 00f3536c8db..66f1c7c3436 100644 --- a/worker/draft.go +++ b/worker/draft.go @@ -1125,7 +1125,7 @@ func (n *node) Run() { // detect leadership changes if rd.SoftState.Lead != lastLead { lastLead = rd.SoftState.Lead - ostats.Record(ctx, x.RaftLeaderChangesSeenTotal.M(1)) + ostats.Record(ctx, x.RaftLeaderChanges.M(1)) } if rd.SoftState.Lead != raft.None { ostats.Record(ctx, x.RaftHasLeader.M(1)) diff --git a/worker/draft_test.go b/worker/draft_test.go index 72cab8147a7..9d1fa72ec0f 100644 --- a/worker/draft_test.go +++ b/worker/draft_test.go @@ -114,7 +114,3 @@ func TestCalculateSnapshot(t *testing.T) { require.NoError(t, err) require.Nil(t, snap) } - -func TestRaftMetrics(t *testing.T) { - // TODO -} diff --git a/x/metrics.go b/x/metrics.go index c03b5066a5c..6df3b0c2ad1 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -117,8 +117,8 @@ var ( // RaftIsLeader records whether this instance is the leader RaftIsLeader = stats.Int64("raft_is_leader", "Whether or not this instance is the leader of the group", stats.UnitDimensionless) - // RaftLeaderChangesSeenTotal records the total number of leader changes seen. - RaftLeaderChangesSeenTotal = stats.Int64("raft_leader_changes_seen_total", + // RaftLeaderChanges records the total number of leader changes seen. + RaftLeaderChanges = stats.Int64("raft_leader_changes_total", "Total number of leader changes seen", stats.UnitDimensionless) // Conf holds the metrics config. From d1c45d4ce3ccc9a7c4fe07ae7a4d771e97ffdf46 Mon Sep 17 00:00:00 2001 From: Karl McGuire Date: Fri, 22 Jan 2021 14:14:59 -0500 Subject: [PATCH 17/17] fix --- x/metrics.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x/metrics.go b/x/metrics.go index 6df3b0c2ad1..d9b97430100 100644 --- a/x/metrics.go +++ b/x/metrics.go @@ -299,9 +299,9 @@ var ( TagKeys: allRaftKeys, }, { - Name: RaftLeaderChangesSeenTotal.Name(), - Measure: RaftLeaderChangesSeenTotal, - Description: RaftLeaderChangesSeenTotal.Description(), + Name: RaftLeaderChanges.Name(), + Measure: RaftLeaderChanges, + Description: RaftLeaderChanges.Description(), Aggregation: view.Count(), TagKeys: allRaftKeys, },