From ef7eb708d95822ff570bb9bd04194fc656261a84 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 3 Sep 2020 08:35:02 +0300 Subject: [PATCH 01/12] vttablet: heartbeat always enabled -enable_heartbeat flag now always assumed to be enabled -heartbeat_interval is now always set to some value in the range (0..1] seconds, default 500ms Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../vttablet/tabletserver/tabletenv/config.go | 31 ++++++++++--------- .../tabletserver/tabletenv/config_test.go | 22 ++++++------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 1a25f8c7d01..c4e9eb37f03 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -41,6 +41,10 @@ const ( Heartbeat = "heartbeat" ) +var ( + defaultHeartbeatInterval = 500 * time.Millisecond +) + var ( currentConfig = TabletConfig{ DB: &dbconfigs.GlobalDBConfigs, @@ -72,7 +76,6 @@ var ( enableHotRowProtectionDryRun bool enableConsolidator bool enableConsolidatorReplicas bool - enableHeartbeat bool heartbeatInterval time.Duration healthCheckInterval time.Duration degradedThreshold time.Duration @@ -140,8 +143,9 @@ func init() { flag.BoolVar(¤tConfig.TransactionLimitByComponent, "transaction_limit_by_component", defaultConfig.TransactionLimitByComponent, "Include CallerID.component when considering who the user is for the purpose of transaction limit.") flag.BoolVar(¤tConfig.TransactionLimitBySubcomponent, "transaction_limit_by_subcomponent", defaultConfig.TransactionLimitBySubcomponent, "Include CallerID.subcomponent when considering who the user is for the purpose of transaction limit.") - flag.BoolVar(&enableHeartbeat, "heartbeat_enable", false, "If true, vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the table _vt.heartbeat. The result is used to inform the serving state of the vttablet via healthchecks.") - flag.DurationVar(&heartbeatInterval, "heartbeat_interval", 1*time.Second, "How frequently to read and write replication heartbeat.") + var dummyEnableHeartbeat bool + flag.BoolVar(&dummyEnableHeartbeat, "heartbeat_enable", true, "Always enabled, flag to be deprecated. vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the table _vt.heartbeat. The result is used to inform the serving state of the vttablet via healthchecks.") + flag.DurationVar(&heartbeatInterval, "heartbeat_interval", defaultHeartbeatInterval, "How frequently to read and write replication heartbeat. Maximum value: 1sec") flag.BoolVar(¤tConfig.EnforceStrictTransTables, "enforce_strict_trans_tables", defaultConfig.EnforceStrictTransTables, "If true, vttablet requires MySQL to run with STRICT_TRANS_TABLES or STRICT_ALL_TABLES on. It is recommended to not turn this flag off. Otherwise MySQL may alter your supplied values before saving them to the database.") flag.BoolVar(&enableConsolidator, "enable-consolidator", true, "This option enables the query consolidator.") @@ -182,17 +186,15 @@ func Init() { currentConfig.Consolidator = Disable } - switch { - case enableHeartbeat: - currentConfig.ReplicationTracker.Mode = Heartbeat - currentConfig.ReplicationTracker.HeartbeatIntervalSeconds.Set(heartbeatInterval) - case enableReplicationReporter: - currentConfig.ReplicationTracker.Mode = Polling - currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 - default: - currentConfig.ReplicationTracker.Mode = Disable - currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 + // We hard-code enable heartbeat, and force heartbeat interval onto a reasonable value + if heartbeatInterval == 0 { + heartbeatInterval = defaultHeartbeatInterval + } + if heartbeatInterval > time.Second { + heartbeatInterval = time.Second } + currentConfig.ReplicationTracker.Mode = Heartbeat + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds.Set(heartbeatInterval) currentConfig.Healthcheck.IntervalSeconds.Set(healthCheckInterval) currentConfig.Healthcheck.DegradedThresholdSeconds.Set(degradedThreshold) @@ -418,7 +420,8 @@ var defaultConfig = TabletConfig{ UnhealthyThresholdSeconds: 7200, }, ReplicationTracker: ReplicationTrackerConfig{ - Mode: Disable, + Mode: Heartbeat, + HeartbeatIntervalSeconds: 0.5, }, HotRowProtection: HotRowProtectionConfig{ Mode: Disable, diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index bcb5e684a30..081fcacfd23 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -131,7 +131,8 @@ oltpReadPool: size: 16 queryCacheSize: 5000 replicationTracker: - mode: disable + heartbeatIntervalSeconds: 0.5 + mode: heartbeat schemaReloadIntervalSeconds: 1800 streamBufferSize: 32768 txPool: @@ -215,7 +216,8 @@ func TestFlags(t *testing.T) { want.Healthcheck.IntervalSeconds = 20 want.Healthcheck.DegradedThresholdSeconds = 30 want.Healthcheck.UnhealthyThresholdSeconds = 7200 - want.ReplicationTracker.Mode = Disable + want.ReplicationTracker.Mode = Heartbeat + want.ReplicationTracker.HeartbeatIntervalSeconds = 0.5 assert.Equal(t, want.DB, currentConfig.DB) assert.Equal(t, want, currentConfig) @@ -267,31 +269,29 @@ func TestFlags(t *testing.T) { want.Consolidator = Disable assert.Equal(t, want, currentConfig) - enableHeartbeat = true heartbeatInterval = 1 * time.Second currentConfig.ReplicationTracker.Mode = "" - currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 + currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0.2 Init() want.ReplicationTracker.Mode = Heartbeat want.ReplicationTracker.HeartbeatIntervalSeconds = 1 assert.Equal(t, want, currentConfig) - enableHeartbeat = false - heartbeatInterval = 1 * time.Second + heartbeatInterval = 5 * time.Second currentConfig.ReplicationTracker.Mode = "" currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 Init() - want.ReplicationTracker.Mode = Disable - want.ReplicationTracker.HeartbeatIntervalSeconds = 0 + want.ReplicationTracker.Mode = Heartbeat + want.ReplicationTracker.HeartbeatIntervalSeconds = 1 assert.Equal(t, want, currentConfig) enableReplicationReporter = true - heartbeatInterval = 1 * time.Second + heartbeatInterval = 0 * time.Second currentConfig.ReplicationTracker.Mode = "" currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 Init() - want.ReplicationTracker.Mode = Polling - want.ReplicationTracker.HeartbeatIntervalSeconds = 0 + want.ReplicationTracker.Mode = Heartbeat + want.ReplicationTracker.HeartbeatIntervalSeconds = 0.5 assert.Equal(t, want, currentConfig) healthCheckInterval = 1 * time.Second From 9db9bc208980945c281d9e4758b93bd92fe2820f Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 3 Sep 2020 08:50:08 +0300 Subject: [PATCH 02/12] use defaultConfig.ReplicationTracker.HeartbeatIntervalSeconds Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/tabletenv/config.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index c4e9eb37f03..887bb54e90e 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -41,10 +41,6 @@ const ( Heartbeat = "heartbeat" ) -var ( - defaultHeartbeatInterval = 500 * time.Millisecond -) - var ( currentConfig = TabletConfig{ DB: &dbconfigs.GlobalDBConfigs, @@ -145,7 +141,7 @@ func init() { var dummyEnableHeartbeat bool flag.BoolVar(&dummyEnableHeartbeat, "heartbeat_enable", true, "Always enabled, flag to be deprecated. vttablet records (if master) or checks (if replica) the current time of a replication heartbeat in the table _vt.heartbeat. The result is used to inform the serving state of the vttablet via healthchecks.") - flag.DurationVar(&heartbeatInterval, "heartbeat_interval", defaultHeartbeatInterval, "How frequently to read and write replication heartbeat. Maximum value: 1sec") + flag.DurationVar(&heartbeatInterval, "heartbeat_interval", time.Duration(defaultConfig.ReplicationTracker.HeartbeatIntervalSeconds*1000)*time.Millisecond, "How frequently to read and write replication heartbeat. Maximum value: 1sec") flag.BoolVar(¤tConfig.EnforceStrictTransTables, "enforce_strict_trans_tables", defaultConfig.EnforceStrictTransTables, "If true, vttablet requires MySQL to run with STRICT_TRANS_TABLES or STRICT_ALL_TABLES on. It is recommended to not turn this flag off. Otherwise MySQL may alter your supplied values before saving them to the database.") flag.BoolVar(&enableConsolidator, "enable-consolidator", true, "This option enables the query consolidator.") @@ -188,7 +184,7 @@ func Init() { // We hard-code enable heartbeat, and force heartbeat interval onto a reasonable value if heartbeatInterval == 0 { - heartbeatInterval = defaultHeartbeatInterval + heartbeatInterval = time.Duration(defaultConfig.ReplicationTracker.HeartbeatIntervalSeconds*1000) * time.Millisecond } if heartbeatInterval > time.Second { heartbeatInterval = time.Second From 36ee6ba09d3af614c502a8ed6dcc8af44b5cb6a8 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 6 Sep 2020 13:41:25 +0300 Subject: [PATCH 03/12] interval changed to 0.25sec Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/tabletenv/config.go | 2 +- go/vt/vttablet/tabletserver/tabletenv/config_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/tabletenv/config.go b/go/vt/vttablet/tabletserver/tabletenv/config.go index 887bb54e90e..10bcdf526ba 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config.go @@ -417,7 +417,7 @@ var defaultConfig = TabletConfig{ }, ReplicationTracker: ReplicationTrackerConfig{ Mode: Heartbeat, - HeartbeatIntervalSeconds: 0.5, + HeartbeatIntervalSeconds: 0.25, }, HotRowProtection: HotRowProtectionConfig{ Mode: Disable, diff --git a/go/vt/vttablet/tabletserver/tabletenv/config_test.go b/go/vt/vttablet/tabletserver/tabletenv/config_test.go index 081fcacfd23..12b124acb70 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/config_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/config_test.go @@ -131,7 +131,7 @@ oltpReadPool: size: 16 queryCacheSize: 5000 replicationTracker: - heartbeatIntervalSeconds: 0.5 + heartbeatIntervalSeconds: 0.25 mode: heartbeat schemaReloadIntervalSeconds: 1800 streamBufferSize: 32768 @@ -217,7 +217,7 @@ func TestFlags(t *testing.T) { want.Healthcheck.DegradedThresholdSeconds = 30 want.Healthcheck.UnhealthyThresholdSeconds = 7200 want.ReplicationTracker.Mode = Heartbeat - want.ReplicationTracker.HeartbeatIntervalSeconds = 0.5 + want.ReplicationTracker.HeartbeatIntervalSeconds = 0.25 assert.Equal(t, want.DB, currentConfig.DB) assert.Equal(t, want, currentConfig) @@ -291,7 +291,7 @@ func TestFlags(t *testing.T) { currentConfig.ReplicationTracker.HeartbeatIntervalSeconds = 0 Init() want.ReplicationTracker.Mode = Heartbeat - want.ReplicationTracker.HeartbeatIntervalSeconds = 0.5 + want.ReplicationTracker.HeartbeatIntervalSeconds = 0.25 assert.Equal(t, want, currentConfig) healthCheckInterval = 1 * time.Second From df14fa237132e8bd5df66f361cc423ff2e1d2746 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 29 Sep 2020 13:43:53 +0300 Subject: [PATCH 04/12] support enable/disable heartbeat injection, used by tests Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/endtoend/vstreamer_test.go | 2 ++ .../tabletserver/repltracker/repltracker.go | 6 ++++++ go/vt/vttablet/tabletserver/repltracker/writer.go | 13 +++++++++++-- go/vt/vttablet/tabletserver/tabletserver.go | 6 ++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go index 24e3e8a6204..b6fbc09f779 100644 --- a/go/vt/vttablet/endtoend/vstreamer_test.go +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -58,6 +58,8 @@ func TestSchemaVersioning(t *testing.T) { tsv := framework.Server tsv.EnableHistorian(false) tsv.SetTracking(false) + tsv.EnableHeartbeat(false) + defer tsv.EnableHeartbeat(true) defer tsv.EnableHistorian(true) defer tsv.SetTracking(true) diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 7c3013bc67f..99e0b7444b4 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -128,3 +128,9 @@ func (rt *ReplTracker) Status() (time.Duration, error) { // rt.mode == tabletenv.Poller return rt.poller.Status() } + +// EnableHeartbeat enables or disables writes of heartbeat. This functionality +// is only used by tests. +func (rt *ReplTracker) EnableHeartbeat(enable bool) { + rt.hw.enableWrites(enable) +} diff --git a/go/vt/vttablet/tabletserver/repltracker/writer.go b/go/vt/vttablet/tabletserver/repltracker/writer.go index 249fd06efa3..18712eb715d 100644 --- a/go/vt/vttablet/tabletserver/repltracker/writer.go +++ b/go/vt/vttablet/tabletserver/repltracker/writer.go @@ -111,7 +111,7 @@ func (w *heartbeatWriter) Open() { log.Info("Hearbeat Writer: opening") w.pool.Open(w.env.Config().DB.AppWithDB(), w.env.Config().DB.DbaWithDB(), w.env.Config().DB.AppDebugWithDB()) - w.ticks.Start(w.writeHeartbeat) + w.enableWrites(true) w.isOpen = true } @@ -126,7 +126,7 @@ func (w *heartbeatWriter) Close() { return } - w.ticks.Stop() + w.enableWrites(false) w.pool.Close() w.isOpen = false log.Info("Hearbeat Writer: closed") @@ -182,3 +182,12 @@ func (w *heartbeatWriter) recordError(err error) { w.errorLog.Errorf("%v", err) writeErrors.Add(1) } + +// enableWrites actives or deactives heartbeat writes +func (w *heartbeatWriter) enableWrites(enable bool) { + if enable { + w.ticks.Start(w.writeHeartbeat) + } else { + w.ticks.Stop() + } +} diff --git a/go/vt/vttablet/tabletserver/tabletserver.go b/go/vt/vttablet/tabletserver/tabletserver.go index 6ef9abd7358..f34a96fe803 100644 --- a/go/vt/vttablet/tabletserver/tabletserver.go +++ b/go/vt/vttablet/tabletserver/tabletserver.go @@ -1455,6 +1455,12 @@ func (tsv *TabletServer) registerTwopczHandler() { }) } +// EnableHeartbeat forces heartbeat to be on or off. +// Only to be used for testing. +func (tsv *TabletServer) EnableHeartbeat(enabled bool) { + tsv.rt.EnableHeartbeat(enabled) +} + // SetTracking forces tracking to be on or off. // Only to be used for testing. func (tsv *TabletServer) SetTracking(enabled bool) { From 57d5fe3254b21d990f79e88f14d92ddbabb95f6c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 29 Sep 2020 13:45:35 +0300 Subject: [PATCH 05/12] applying EnableHeartbeat in tests Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/endtoend/vstreamer_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go index b6fbc09f779..126dd2d3bc2 100644 --- a/go/vt/vttablet/endtoend/vstreamer_test.go +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -67,6 +67,7 @@ func TestSchemaVersioning(t *testing.T) { defer cancel() tsv.EnableHistorian(true) tsv.SetTracking(true) + tsv.EnableHeartbeat(true) target := &querypb.Target{ Keyspace: "vttest", @@ -166,6 +167,7 @@ func TestSchemaVersioning(t *testing.T) { runCases(ctx, t, cases, eventCh) tsv.SetTracking(false) + tsv.EnableHeartbeat(false) cases = []test{ { //comment prefix required so we don't look for ddl in schema_version From 327f44bd6e4b8e1e4810829e1939a553b76fb76e Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Tue, 29 Sep 2020 14:06:36 +0300 Subject: [PATCH 06/12] do not enable heartbeat while inside test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/endtoend/vstreamer_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go index 126dd2d3bc2..b6fbc09f779 100644 --- a/go/vt/vttablet/endtoend/vstreamer_test.go +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -67,7 +67,6 @@ func TestSchemaVersioning(t *testing.T) { defer cancel() tsv.EnableHistorian(true) tsv.SetTracking(true) - tsv.EnableHeartbeat(true) target := &querypb.Target{ Keyspace: "vttest", @@ -167,7 +166,6 @@ func TestSchemaVersioning(t *testing.T) { runCases(ctx, t, cases, eventCh) tsv.SetTracking(false) - tsv.EnableHeartbeat(false) cases = []test{ { //comment prefix required so we don't look for ddl in schema_version From 19eaf4b1c34341e1e0539f2b3b4a3ad5aa243d0b Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 11:01:18 +0300 Subject: [PATCH 07/12] debug message Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/cluster/vtgate_process.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index ed2c287304d..6e3fda61be9 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -161,6 +161,7 @@ func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCou if key.String() == name { value := fmt.Sprintf("%v", object.MapIndex(key)) countStr := strconv.Itoa(endPointsCount) + fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, countStr=%+v\n", name, endPointsCount, countStr) return value == countStr } } From de0a7264d1d75ef95ebf9feeef1025b60746504d Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 11:23:31 +0300 Subject: [PATCH 08/12] debug message Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/cluster/vtgate_process.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index 6e3fda61be9..614cf14c3bf 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -147,6 +147,7 @@ func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCou if err != nil { return false } + fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, resp.StatusCode=%+v\n", name, endPointsCount, resp.StatusCode) if resp.StatusCode == 200 { resultMap := make(map[string]interface{}) respByte, _ := ioutil.ReadAll(resp.Body) @@ -157,6 +158,7 @@ func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCou object := reflect.ValueOf(resultMap["HealthcheckConnections"]) masterConnectionExist := false if object.Kind() == reflect.Map { + fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, object.MapKeys() =%+v\n", name, endPointsCount, object.MapKeys()) for _, key := range object.MapKeys() { if key.String() == name { value := fmt.Sprintf("%v", object.MapIndex(key)) From 35d8ddb0e49ff444f521ba8f5ece9eb0ea4b90ff Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 12:45:02 +0300 Subject: [PATCH 09/12] debug message Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/cluster/vtgate_process.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index 614cf14c3bf..e4f4defc4d6 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -143,6 +143,7 @@ func (vtgate *VtgateProcess) WaitForStatus() bool { // GetStatusForTabletOfShard function gets status for a specific tablet of a shard in keyspace // endPointsCount : number of endpoints func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCount int) bool { + fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, vtgate.VerifyURL=%+v\n", name, endPointsCount, vtgate.VerifyURL) resp, err := http.Get(vtgate.VerifyURL) if err != nil { return false @@ -158,12 +159,10 @@ func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCou object := reflect.ValueOf(resultMap["HealthcheckConnections"]) masterConnectionExist := false if object.Kind() == reflect.Map { - fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, object.MapKeys() =%+v\n", name, endPointsCount, object.MapKeys()) for _, key := range object.MapKeys() { if key.String() == name { value := fmt.Sprintf("%v", object.MapIndex(key)) countStr := strconv.Itoa(endPointsCount) - fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, countStr=%+v\n", name, endPointsCount, countStr) return value == countStr } } From d40db4a260b806c6563ecc3f541b671d69ce48f4 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 14:14:16 +0300 Subject: [PATCH 10/12] clenaup debug messages Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/cluster/vtgate_process.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index e4f4defc4d6..ed2c287304d 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -143,12 +143,10 @@ func (vtgate *VtgateProcess) WaitForStatus() bool { // GetStatusForTabletOfShard function gets status for a specific tablet of a shard in keyspace // endPointsCount : number of endpoints func (vtgate *VtgateProcess) GetStatusForTabletOfShard(name string, endPointsCount int) bool { - fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, vtgate.VerifyURL=%+v\n", name, endPointsCount, vtgate.VerifyURL) resp, err := http.Get(vtgate.VerifyURL) if err != nil { return false } - fmt.Printf("============= GetStatusForTabletOfShard name=%s, endPointsCount=%d, resp.StatusCode=%+v\n", name, endPointsCount, resp.StatusCode) if resp.StatusCode == 200 { resultMap := make(map[string]interface{}) respByte, _ := ioutil.ReadAll(resp.Body) From 00100d442e3ac08c81683c457168237d1b15fb01 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 15:07:21 +0300 Subject: [PATCH 11/12] experiment: repl status Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/repltracker/repltracker.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 99e0b7444b4..7e1de33d2c4 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -123,7 +123,10 @@ func (rt *ReplTracker) Status() (time.Duration, error) { case rt.isMaster || rt.mode == tabletenv.Disable: return 0, nil case rt.mode == tabletenv.Heartbeat: - return rt.hr.Status() + { + _, err := rt.hr.Status() + return 0, err + } } // rt.mode == tabletenv.Poller return rt.poller.Status() From f30a854ca67d1c329562ac8383ad5e8729e43ae2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 30 Sep 2020 15:31:01 +0300 Subject: [PATCH 12/12] undo experiment: same endtoend failures Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/vt/vttablet/tabletserver/repltracker/repltracker.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/go/vt/vttablet/tabletserver/repltracker/repltracker.go b/go/vt/vttablet/tabletserver/repltracker/repltracker.go index 7e1de33d2c4..99e0b7444b4 100644 --- a/go/vt/vttablet/tabletserver/repltracker/repltracker.go +++ b/go/vt/vttablet/tabletserver/repltracker/repltracker.go @@ -123,10 +123,7 @@ func (rt *ReplTracker) Status() (time.Duration, error) { case rt.isMaster || rt.mode == tabletenv.Disable: return 0, nil case rt.mode == tabletenv.Heartbeat: - { - _, err := rt.hr.Status() - return 0, err - } + return rt.hr.Status() } // rt.mode == tabletenv.Poller return rt.poller.Status()